Adjust UrlSource behavior when reading non-range responses, it now honors the cache size and behaves like a ReadableStreamSource (fixes #457)

test/node/url-source.test.ts

@@ -0,0 +1,309 @@
+import { expect, test } from 'vitest';
+import http from 'node:http';
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+	ALL_FORMATS,
+	EncodedPacket,
+	EncodedPacketSink,
+	FilePathSource,
+	Input,
+	Logging,
+	LogLevel,
+	UrlSource,
+} from '../../src/index.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+const videoFilePath = path.join(__dirname, '..', 'public/video.mp4');
+
+test('UrlSource works against a server without range request support', async () => {
+	const server = await startRangelessServer();
+	const logs = captureLogs();
+
+	try {
+		using input = new Input({
+			source: new UrlSource(server.url),
+			formats: ALL_FORMATS,
+		});
+
+		const track = await input.getPrimaryVideoTrack();
+		if (!track) throw new Error('No video track found');
+
+		const sink = new EncodedPacketSink(track);
+
+		const timestamps: number[] = [];
+		for await (const packet of sink.packets()) {
+			timestamps.push(packet.timestamp);
+		}
+
+		expect(timestamps).toHaveLength(125);
+
+		// The default cache size exceeds the file size, so random access back to the start of the file still works
+		const firstPacket = await sink.getFirstPacket();
+		if (!firstPacket) throw new Error('No first packet found');
+
+		expect(firstPacket.timestamp).toBe(0);
+		expect(firstPacket.data.byteLength).toBeGreaterThan(0);
+
+		expect(logs.warnings.filter(
+			x => x.includes('did not respond to a range request with 206 Partial Content'),
+		)).toHaveLength(1);
+	} finally {
+		logs.stop();
+		server.close();
+	}
+});
+
+test('UrlSource throws when reading from an evicted region in sequential mode', async () => {
+	const server = await startRangelessServer();
+	const logs = captureLogs();
+
+	try {
+		using input = new Input({
+			// Much smaller than the file, so the start of the file will get evicted during full iteration
+			source: new UrlSource(server.url, { maxCacheSize: 2 ** 20 /* 1 MiB */ }),
+			formats: ALL_FORMATS,
+		});
+
+		const track = await input.getPrimaryVideoTrack();
+		if (!track) throw new Error('No video track found');
+
+		const sink = new EncodedPacketSink(track);
+
+		const timestamps: number[] = [];
+		for await (const packet of sink.packets()) {
+			timestamps.push(packet.timestamp);
+		}
+
+		expect(timestamps).toHaveLength(125);
+
+		await expect(sink.getFirstPacket()).rejects.toThrow(/already-evicted part of the cache/);
+
+		expect(logs.warnings.filter(
+			x => x.includes('did not respond to a range request with 206 Partial Content'),
+		)).toHaveLength(1);
+	} finally {
+		logs.stop();
+		server.close();
+	}
+});
+
+test('UrlSource resumes with correct data when the connection dies in sequential mode', async () => {
+	// The server kills the connection partway through the first two responses. Since it doesn't support range
+	// requests, each resume response starts back at byte 0 and the already-delivered prefix must be skipped over
+	// exactly; any off-by-one would corrupt the packet data.
+	const server = await startRangelessServer({ responseByteLimits: [700_000, 1_500_000] });
+	const logs = captureLogs();
+
+	try {
+		using referenceInput = new Input({
+			source: new FilePathSource(videoFilePath),
+			formats: ALL_FORMATS,
+		});
+
+		const referenceTrack = await referenceInput.getPrimaryVideoTrack();
+		if (!referenceTrack) throw new Error('No video track found');
+
+		const referencePackets: EncodedPacket[] = [];
+		for await (const packet of new EncodedPacketSink(referenceTrack).packets()) {
+			referencePackets.push(packet);
+		}
+
+		using input = new Input({
+			source: new UrlSource(server.url, { getRetryDelay: () => 0 }),
+			formats: ALL_FORMATS,
+		});
+
+		const track = await input.getPrimaryVideoTrack();
+		if (!track) throw new Error('No video track found');
+
+		let packetIndex = 0;
+		for await (const packet of new EncodedPacketSink(track).packets()) {
+			const referencePacket = referencePackets[packetIndex]!;
+
+			expect(packet.timestamp).toBe(referencePacket.timestamp);
+			expect(Buffer.from(packet.data).equals(Buffer.from(referencePacket.data))).toBe(true);
+
+			packetIndex++;
+		}
+
+		expect(packetIndex).toBe(referencePackets.length);
+
+		// Initial request plus one resume per killed response
+		expect(server.requestCount()).toBe(3);
+		expect(logs.errors.filter(x => x.includes('Attempting to resume'))).toHaveLength(2);
+	} finally {
+		logs.stop();
+		server.close();
+	}
+});
+
+test('UrlSource with maxCacheSize: Infinity allows random access against a rangeless server', async () => {
+	const server = await startRangelessServer();
+	const logs = captureLogs();
+
+	try {
+		using input = new Input({
+			// This is the escape hatch recommended by the eviction error message: with an infinite cache, nothing
+			// is ever evicted and random access keeps working
+			source: new UrlSource(server.url, { maxCacheSize: Infinity }),
+			formats: ALL_FORMATS,
+		});
+
+		const track = await input.getPrimaryVideoTrack();
+		if (!track) throw new Error('No video track found');
+
+		const sink = new EncodedPacketSink(track);
+
+		const timestamps: number[] = [];
+		for await (const packet of sink.packets()) {
+			timestamps.push(packet.timestamp);
+		}
+
+		expect(timestamps).toHaveLength(125);
+
+		const firstPacket = await sink.getFirstPacket();
+		if (!firstPacket) throw new Error('No first packet found');
+
+		expect(firstPacket.timestamp).toBe(0);
+		expect(firstPacket.data.byteLength).toBeGreaterThan(0);
+
+		expect(logs.warnings.filter(
+			x => x.includes('did not respond to a range request with 206 Partial Content'),
+		)).toHaveLength(1);
+	} finally {
+		logs.stop();
+		server.close();
+	}
+});
+
+test('UrlSource in sequential mode downloads lazily and aborts the response on dispose', async () => {
+	const fileSize = fs.statSync(videoFilePath).size;
+	const server = await startRangelessServer();
+	const logs = captureLogs();
+
+	const input = new Input({
+		source: new UrlSource(server.url),
+		formats: ALL_FORMATS,
+	});
+
+	try {
+		const track = await input.getPrimaryVideoTrack();
+		if (!track) throw new Error('No video track found');
+
+		// If the client were downloading eagerly, the entire file would easily arrive during this window. Instead,
+		// the server is expected to stall, since data is only pulled down when reads demand it.
+		await new Promise(resolve => setTimeout(resolve, 3000));
+
+		expect(server.bytesSent()).toBeLessThan(fileSize / 2);
+
+		// The response is merely suspended, not dead: reading still works
+		const sink = new EncodedPacketSink(track);
+		const firstPacket = await sink.getFirstPacket();
+		if (!firstPacket) throw new Error('No first packet found');
+
+		expect(firstPacket.timestamp).toBe(0);
+		expect(firstPacket.data.byteLength).toBeGreaterThan(0);
+
+		input.dispose();
+
+		// Give the abort a moment to propagate to the server
+		await new Promise(resolve => setTimeout(resolve, 250));
+
+		// Disposal terminated the response early, long before the entire file was sent
+		expect(server.abortedResponses()).toBe(1);
+		expect(server.bytesSent()).toBeLessThan(fileSize / 2);
+	} finally {
+		input.dispose();
+		logs.stop();
+		server.close();
+	}
+}, 10_000);
+
+/**
+ * Spins up a server that ignores Range headers, always responding with 200 and the full file. If responseByteLimits
+ * is provided, the n-th response gets its connection killed after roughly that many bytes.
+ */
+const startRangelessServer = async (options: { responseByteLimits?: number[] } = {}) => {
+	const fileSize = fs.statSync(videoFilePath).size;
+	let requestCount = 0;
+	let bytesSent = 0;
+	let abortedResponses = 0;
+
+	const server = http.createServer((req, res) => {
+		const byteLimit = options.responseByteLimits?.[requestCount] ?? Infinity;
+		requestCount++;
+
+		res.on('error', () => {}); // The client may abort the connection at any time
+		res.on('close', () => {
+			if (!res.writableFinished) {
+				abortedResponses++;
+			}
+		});
+		res.writeHead(200, {
+			'Content-Type': 'video/mp4',
+			'Content-Length': fileSize,
+		});
+
+		// Stream the file with backpressure and a small chunk size, so that the amount of sent bytes closely
+		// tracks how much the client actually consumes
+		const stream = fs.createReadStream(videoFilePath, { highWaterMark: 2 ** 14 });
+		let bytesSentThisResponse = 0;
+
+		stream.on('data', (chunk) => {
+			const canContinue = res.write(chunk);
+			bytesSent += chunk.length;
+			bytesSentThisResponse += chunk.length;
+
+			if (bytesSentThisResponse >= byteLimit) {
+				stream.destroy();
+				res.destroy(); // Kill the connection mid-response
+
+				return;
+			}
+
+			if (!canContinue) {
+				stream.pause();
+				res.once('drain', () => stream.resume());
+			}
+		});
+		stream.on('end', () => res.end());
+	});
+
+	await new Promise<void>(resolve => server.listen(0, resolve));
+
+	const address = server.address();
+	if (!address || typeof address === 'string') throw new Error('Unexpected server address');
+
+	return {
+		url: `http://localhost:${address.port}/video.mp4`,
+		requestCount: () => requestCount,
+		bytesSent: () => bytesSent,
+		abortedResponses: () => abortedResponses,
+		close: () => {
+			server.closeAllConnections();
+			server.close();
+		},
+	};
+};
+
+/** Collects logged warnings and errors, keeping the console clean while doing so. */
+const captureLogs = () => {
+	const warnings: string[] = [];
+	const errors: string[] = [];
+	const unsubscribeWarn = Logging.on('warn', args => warnings.push(args.map(String).join(' ')));
+	const unsubscribeError = Logging.on('error', args => errors.push(args.map(String).join(' ')));
+
+	const previousLogLevel = Logging.level;
+	Logging.level = LogLevel.Silent;
+
+	return {
+		warnings,
+		errors,
+		stop: () => {
+			unsubscribeWarn();
+			unsubscribeError();
+			Logging.level = previousLogLevel;
+		},
+	};
+};