feat: add docker pipeline and preview fixes
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { decodeOpaqueId, encodeOpaqueId } from '../src/lib/fileId.js';
|
||||
|
||||
test('opaque ids round-trip metadata', () => {
|
||||
const token = encodeOpaqueId({
|
||||
kind: 'file',
|
||||
filesetDbId: 12,
|
||||
dbPath: 'C:\\Data\\demo.mp4'
|
||||
});
|
||||
|
||||
assert.deepEqual(decodeOpaqueId(token), {
|
||||
v: 3,
|
||||
kind: 'file',
|
||||
filesetDbId: 12,
|
||||
dbPath: 'C:\\Data\\demo.mp4'
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
buildResolvedFilesetId,
|
||||
computeCanonicalBlockSize,
|
||||
normalizeCompressionMethod,
|
||||
toZipEntryName
|
||||
} from '../src/lib/duplicati.js';
|
||||
import { normalizeApiPath, toApiPath } from '../src/lib/paths.js';
|
||||
import { duplicatiTicksToIso, unixSecondsToIso } from '../src/lib/time.js';
|
||||
|
||||
test('db paths normalize to API paths', () => {
|
||||
assert.equal(toApiPath('C:\\Users\\me\\movie.mp4'), '/C:/Users/me/movie.mp4');
|
||||
assert.equal(normalizeApiPath('/movies/demo.mp4/'), '/movies/demo.mp4');
|
||||
});
|
||||
|
||||
test('hashes convert to duplicati zip entry names', () => {
|
||||
assert.equal(toZipEntryName('abc+/def='), 'abc-_def=');
|
||||
});
|
||||
|
||||
test('compression methods normalize from zip numeric codes', () => {
|
||||
assert.equal(normalizeCompressionMethod(8), 'deflate');
|
||||
assert.equal(normalizeCompressionMethod('0'), 'store');
|
||||
});
|
||||
|
||||
test('resolved fileset id is deterministic', () => {
|
||||
assert.equal(buildResolvedFilesetId('2026-05-05T08:00:00.000Z'), 'fs_2026-05-05T08:00:00.000Z');
|
||||
});
|
||||
|
||||
test('canonical block size chooses the dominant chunk size', () => {
|
||||
assert.equal(
|
||||
computeCanonicalBlockSize([
|
||||
{ logicalSize: 102400 },
|
||||
{ logicalSize: 102400 },
|
||||
{ logicalSize: 4096 }
|
||||
]),
|
||||
102400
|
||||
);
|
||||
});
|
||||
|
||||
test('timestamps convert from unix seconds and .NET ticks', () => {
|
||||
assert.equal(unixSecondsToIso(0), '1970-01-01T00:00:00.000Z');
|
||||
assert.equal(duplicatiTicksToIso('621355968000000000'), '1970-01-01T00:00:00.000Z');
|
||||
});
|
||||
@@ -0,0 +1,358 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
clearAllCachedVolumes,
|
||||
clearCachedVolumesForSource,
|
||||
decryptAesCryptV2Bytes,
|
||||
decryptAesCryptV2StreamToWriter,
|
||||
extractZipEntryBytesFromBuffer,
|
||||
getCachedVolumeStats,
|
||||
getGlobalCacheStats,
|
||||
listCachedVolumeNames,
|
||||
parseSingleRange,
|
||||
selectSegmentsForRange
|
||||
} from '../public/modules/media-core.js';
|
||||
import { scanZipEntries } from '../src/lib/zip.js';
|
||||
import { createDuplicatiFixture } from './support/duplicatiFixture.js';
|
||||
|
||||
function createNotFoundError(message = 'Not found.') {
|
||||
const error = new Error(message);
|
||||
error.name = 'NotFoundError';
|
||||
return error;
|
||||
}
|
||||
|
||||
class FakeFile {
|
||||
constructor(bytes) {
|
||||
this.bytes = Uint8Array.from(bytes);
|
||||
this.size = this.bytes.length;
|
||||
}
|
||||
|
||||
slice(start = 0, end = this.bytes.length) {
|
||||
const bytes = this.bytes.slice(start, end);
|
||||
return {
|
||||
async arrayBuffer() {
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async arrayBuffer() {
|
||||
return this.bytes.buffer.slice(this.bytes.byteOffset, this.bytes.byteOffset + this.bytes.byteLength);
|
||||
}
|
||||
}
|
||||
|
||||
class FakeFileHandle {
|
||||
constructor(name, bytes) {
|
||||
this.kind = 'file';
|
||||
this.name = name;
|
||||
this.file = new FakeFile(bytes);
|
||||
}
|
||||
|
||||
async getFile() {
|
||||
return this.file;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeDirectoryHandle {
|
||||
constructor(name = '') {
|
||||
this.kind = 'directory';
|
||||
this.name = name;
|
||||
this.children = new Map();
|
||||
}
|
||||
|
||||
setFile(name, bytes) {
|
||||
this.children.set(name, new FakeFileHandle(name, bytes));
|
||||
return this;
|
||||
}
|
||||
|
||||
setDirectory(name, directoryHandle) {
|
||||
this.children.set(name, directoryHandle);
|
||||
return this;
|
||||
}
|
||||
|
||||
async getDirectoryHandle(name, options = {}) {
|
||||
const existing = this.children.get(name);
|
||||
if (existing?.kind === 'directory') {
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (options.create) {
|
||||
const directory = new FakeDirectoryHandle(name);
|
||||
this.children.set(name, directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
throw createNotFoundError(`Directory ${name} was not found.`);
|
||||
}
|
||||
|
||||
async getFileHandle(name, options = {}) {
|
||||
const existing = this.children.get(name);
|
||||
if (existing?.kind === 'file') {
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (options.create) {
|
||||
const fileHandle = new FakeFileHandle(name, new Uint8Array(0));
|
||||
this.children.set(name, fileHandle);
|
||||
return fileHandle;
|
||||
}
|
||||
|
||||
throw createNotFoundError(`File ${name} was not found.`);
|
||||
}
|
||||
|
||||
async removeEntry(name) {
|
||||
if (!this.children.has(name)) {
|
||||
throw createNotFoundError(`Entry ${name} was not found.`);
|
||||
}
|
||||
|
||||
this.children.delete(name);
|
||||
}
|
||||
|
||||
async *entries() {
|
||||
for (const entry of this.children.entries()) {
|
||||
yield entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('parseSingleRange supports explicit, open, and suffix byte ranges', () => {
|
||||
assert.deepEqual(parseSingleRange('bytes=0-99', 1000), {
|
||||
start: 0,
|
||||
endInclusive: 99,
|
||||
endExclusive: 100,
|
||||
contentLength: 100,
|
||||
partial: true
|
||||
});
|
||||
|
||||
assert.deepEqual(parseSingleRange('bytes=200-', 1000), {
|
||||
start: 200,
|
||||
endInclusive: 999,
|
||||
endExclusive: 1000,
|
||||
contentLength: 800,
|
||||
partial: true
|
||||
});
|
||||
|
||||
assert.deepEqual(parseSingleRange('bytes=-128', 1000), {
|
||||
start: 872,
|
||||
endInclusive: 999,
|
||||
endExclusive: 1000,
|
||||
contentLength: 128,
|
||||
partial: true
|
||||
});
|
||||
});
|
||||
|
||||
test('selectSegmentsForRange preserves cross-block slicing metadata', () => {
|
||||
const overlaps = selectSegmentsForRange(
|
||||
[
|
||||
{ segmentIndex: 0, logicalOffset: 0, logicalSize: 8 },
|
||||
{ segmentIndex: 1, logicalOffset: 8, logicalSize: 8 },
|
||||
{ segmentIndex: 2, logicalOffset: 16, logicalSize: 8 }
|
||||
],
|
||||
4,
|
||||
20
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
overlaps.map((entry) => ({
|
||||
segmentIndex: entry.segment.segmentIndex,
|
||||
offsetWithinSegmentStart: entry.offsetWithinSegmentStart,
|
||||
offsetWithinSegmentEnd: entry.offsetWithinSegmentEnd
|
||||
})),
|
||||
[
|
||||
{ segmentIndex: 0, offsetWithinSegmentStart: 4, offsetWithinSegmentEnd: 8 },
|
||||
{ segmentIndex: 1, offsetWithinSegmentStart: 0, offsetWithinSegmentEnd: 8 },
|
||||
{ segmentIndex: 2, offsetWithinSegmentStart: 0, offsetWithinSegmentEnd: 4 }
|
||||
]
|
||||
);
|
||||
});
|
||||
|
||||
test('browser media helpers decrypt AES v2 volumes and restore zip entries', async () => {
|
||||
const fixture = createDuplicatiFixture({
|
||||
withRemoteVolumes: true,
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
const tempDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'media-core-'));
|
||||
const plainZipPath = path.join(tempDirectory, 'demo.zip');
|
||||
|
||||
try {
|
||||
const encryptedDemoBytes = await fs.readFile(fixture.remoteFilesByName['duplicati-demo.dblock.zip.aes']);
|
||||
const decryptedDemo = await decryptAesCryptV2Bytes(encryptedDemoBytes, fixture.passphrase);
|
||||
await fs.writeFile(plainZipPath, decryptedDemo.plainBytes);
|
||||
|
||||
const scanResult = await scanZipEntries(plainZipPath);
|
||||
const firstEntry = scanResult.entries.find(
|
||||
(entry) => entry.entryName === '0td8NEaS7SMrQc5Gs0Sdxjb_1MXEEuwkyxRpguDiWsY='
|
||||
);
|
||||
assert.ok(firstEntry);
|
||||
|
||||
const firstBlock = await extractZipEntryBytesFromBuffer(decryptedDemo.plainBytes, {
|
||||
dataOffsetPlain: firstEntry.dataOffset,
|
||||
compressedSize: firstEntry.compressedSize,
|
||||
uncompressedSize: firstEntry.uncompressedSize,
|
||||
compressionMethod: firstEntry.compressionMethod
|
||||
});
|
||||
assert.equal(firstBlock.length, 102400);
|
||||
assert.equal(firstBlock[0], 0x11);
|
||||
assert.equal(firstBlock[firstBlock.length - 1], 0x11);
|
||||
|
||||
const encryptedTextBytes = await fs.readFile(fixture.remoteFilesByName['duplicati-text.dblock.zip.aes']);
|
||||
const decryptedText = await decryptAesCryptV2Bytes(encryptedTextBytes, fixture.passphrase);
|
||||
await fs.writeFile(plainZipPath, decryptedText.plainBytes);
|
||||
|
||||
const textScanResult = await scanZipEntries(plainZipPath);
|
||||
const textEntry = textScanResult.entries.find((entry) => entry.entryName === 'ySjwbaRrk-rm6Vx0W0xP8A==');
|
||||
assert.ok(textEntry);
|
||||
|
||||
const textBytes = await extractZipEntryBytesFromBuffer(decryptedText.plainBytes, {
|
||||
dataOffsetPlain: textEntry.dataOffset,
|
||||
compressedSize: textEntry.compressedSize,
|
||||
uncompressedSize: textEntry.uncompressedSize,
|
||||
compressionMethod: textEntry.compressionMethod
|
||||
});
|
||||
assert.equal(new TextDecoder().decode(textBytes), 'hello');
|
||||
} finally {
|
||||
fixture.cleanup();
|
||||
await fs.rm(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('streaming AES v2 decrypt writes plain zip bytes without buffering the whole response first', async () => {
|
||||
const fixture = createDuplicatiFixture({
|
||||
withRemoteVolumes: true,
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
|
||||
try {
|
||||
const encryptedBytes = await fs.readFile(fixture.remoteFilesByName['duplicati-demo.dblock.zip.aes']);
|
||||
const expected = await decryptAesCryptV2Bytes(encryptedBytes, fixture.passphrase);
|
||||
|
||||
const writtenChunks = [];
|
||||
const writer = {
|
||||
async write(bytes) {
|
||||
writtenChunks.push(Uint8Array.from(bytes));
|
||||
},
|
||||
async close() {},
|
||||
async abort() {
|
||||
writtenChunks.length = 0;
|
||||
}
|
||||
};
|
||||
|
||||
const streamed = await decryptAesCryptV2StreamToWriter(
|
||||
new Blob([encryptedBytes]).stream(),
|
||||
fixture.passphrase,
|
||||
writer
|
||||
);
|
||||
|
||||
const actual = Uint8Array.from(writtenChunks.flatMap((chunk) => [...chunk]));
|
||||
assert.deepEqual(actual, expected.plainBytes);
|
||||
assert.deepEqual(streamed.meta, expected.meta);
|
||||
} finally {
|
||||
fixture.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('streaming AES v2 decrypt stops when the caller aborts the active request', async () => {
|
||||
const fixture = createDuplicatiFixture({
|
||||
withRemoteVolumes: true,
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
|
||||
try {
|
||||
const encryptedBytes = await fs.readFile(fixture.remoteFilesByName['duplicati-demo.dblock.zip.aes']);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
decryptAesCryptV2StreamToWriter(
|
||||
new Blob([encryptedBytes]).stream(),
|
||||
fixture.passphrase,
|
||||
{
|
||||
async write() {},
|
||||
async close() {},
|
||||
async abort() {}
|
||||
},
|
||||
() => {},
|
||||
controller.signal
|
||||
),
|
||||
(error) => error?.name === 'AbortError' && error?.code === 'MEDIA_REQUEST_ABORTED'
|
||||
);
|
||||
} finally {
|
||||
fixture.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('browser media cache helpers summarize and clear source-scoped OPFS entries', async () => {
|
||||
const root = new FakeDirectoryHandle('root');
|
||||
const cacheRoot = new FakeDirectoryHandle('duplicati-media-cache');
|
||||
const sourceA = new FakeDirectoryHandle('source-a')
|
||||
.setFile('vol-1.zip', new Uint8Array(10))
|
||||
.setFile('vol-2.zip', new Uint8Array(22));
|
||||
const sourceB = new FakeDirectoryHandle('source-b')
|
||||
.setFile('vol-3.zip', new Uint8Array(5));
|
||||
cacheRoot.setDirectory('source-a', sourceA);
|
||||
cacheRoot.setDirectory('source-b', sourceB);
|
||||
root.setDirectory('duplicati-media-cache', cacheRoot);
|
||||
|
||||
const originalNavigator = globalThis.navigator;
|
||||
Object.defineProperty(globalThis, 'navigator', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: {
|
||||
storage: {
|
||||
async getDirectory() {
|
||||
return root;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
assert.deepEqual(await listCachedVolumeNames('source-a'), ['vol-1.zip', 'vol-2.zip']);
|
||||
assert.deepEqual(await getCachedVolumeStats('source-a'), {
|
||||
sourceCount: 0,
|
||||
volumeCount: 2,
|
||||
totalBytes: 32
|
||||
});
|
||||
assert.deepEqual(await getGlobalCacheStats(), {
|
||||
sourceCount: 2,
|
||||
volumeCount: 3,
|
||||
totalBytes: 37
|
||||
});
|
||||
|
||||
assert.deepEqual(await clearCachedVolumesForSource('source-a'), {
|
||||
removedSources: 1,
|
||||
removedVolumes: 2,
|
||||
removedBytes: 32
|
||||
});
|
||||
assert.deepEqual(await getGlobalCacheStats(), {
|
||||
sourceCount: 1,
|
||||
volumeCount: 1,
|
||||
totalBytes: 5
|
||||
});
|
||||
|
||||
assert.deepEqual(await clearAllCachedVolumes(), {
|
||||
removedSources: 1,
|
||||
removedVolumes: 1,
|
||||
removedBytes: 5
|
||||
});
|
||||
assert.deepEqual(await getGlobalCacheStats(), {
|
||||
sourceCount: 0,
|
||||
volumeCount: 0,
|
||||
totalBytes: 0
|
||||
});
|
||||
} finally {
|
||||
Object.defineProperty(globalThis, 'navigator', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalNavigator
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { DuplicatiRepository } from '../src/db/duplicatiRepository.js';
|
||||
import { openWritableSqlite } from '../src/db/sqlite.js';
|
||||
import { createDuplicatiFixture } from './support/duplicatiFixture.js';
|
||||
|
||||
function createAppDatabasePath() {
|
||||
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-app-'));
|
||||
return {
|
||||
appDbPath: path.join(tempDirectory, 'app.sqlite'),
|
||||
cleanup() {
|
||||
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function openAttachedRepository(sourceOverrides = {}) {
|
||||
const fixture = createDuplicatiFixture(sourceOverrides);
|
||||
const appDb = createAppDatabasePath();
|
||||
const database = await openWritableSqlite(appDb.appDbPath);
|
||||
await database.attachDatabase('source', fixture.databasePath, { readOnly: true });
|
||||
const sourceLayout = sourceOverrides.layout ?? 'file';
|
||||
|
||||
const repository = new DuplicatiRepository({
|
||||
database,
|
||||
previewMaxBytes: 1024,
|
||||
source: {
|
||||
id: 'source-1',
|
||||
storagePath: fixture.databasePath,
|
||||
sha256: 'sha-demo',
|
||||
capabilities: {
|
||||
archiveEntryIndex: true,
|
||||
volumeCryptoCache: true
|
||||
}
|
||||
},
|
||||
sourceLayout
|
||||
});
|
||||
|
||||
return {
|
||||
fixture,
|
||||
appDb,
|
||||
repository,
|
||||
database
|
||||
};
|
||||
}
|
||||
|
||||
test('listDirectory returns immediate directory and file children', async () => {
|
||||
const session = await openAttachedRepository();
|
||||
|
||||
try {
|
||||
const result = await session.repository.listDirectory({
|
||||
apiPath: '/C:/media',
|
||||
snapshotId: 'latest'
|
||||
});
|
||||
|
||||
assert.equal(result.path, '/C:/media');
|
||||
assert.equal(result.entries.length, 2);
|
||||
assert.deepEqual(
|
||||
result.entries.map((entry) => ({
|
||||
type: entry.type,
|
||||
path: entry.path
|
||||
})),
|
||||
[
|
||||
{ type: 'dir', path: '/C:/media/movies' },
|
||||
{ type: 'file', path: '/C:/media/readme.txt' }
|
||||
]
|
||||
);
|
||||
} finally {
|
||||
await session.database.detachDatabase('source');
|
||||
await session.database.close();
|
||||
session.fixture.cleanup();
|
||||
session.appDb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('getFileInfo expands ordered segments and required dblocks', async () => {
|
||||
const session = await openAttachedRepository();
|
||||
|
||||
try {
|
||||
const listing = await session.repository.listDirectory({
|
||||
apiPath: '/C:/media/movies',
|
||||
snapshotId: 'latest'
|
||||
});
|
||||
const fileId = listing.entries.find((entry) => entry.type === 'file')?.id;
|
||||
|
||||
assert.ok(fileId);
|
||||
|
||||
const result = await session.repository.getFileInfo(fileId);
|
||||
assert.equal(result.file.path, '/C:/media/movies/demo.mp4');
|
||||
assert.equal(result.restorePlan.segmentCount, 3);
|
||||
assert.equal(result.restorePlan.blockSize, 102400);
|
||||
assert.equal(result.requiredDblocks.length, 1);
|
||||
assert.equal(result.requiredDblocks[0], 'duplicati-demo.dblock.zip.aes');
|
||||
assert.equal(result.volumes[0].encryption.streamFormat, 'v2');
|
||||
assert.equal(result.segments[0].logicalOffset, 0);
|
||||
assert.equal(result.segments[1].logicalOffset, 102400);
|
||||
assert.equal(result.segments[2].zip.compressionMethod, 'deflate');
|
||||
} finally {
|
||||
await session.database.detachDatabase('source');
|
||||
await session.database.close();
|
||||
session.fixture.cleanup();
|
||||
session.appDb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('listDirectory supports FileLookup plus PathPrefix based databases', async () => {
|
||||
const session = await openAttachedRepository({ layout: 'file_lookup' });
|
||||
|
||||
try {
|
||||
const result = await session.repository.listDirectory({
|
||||
apiPath: '/source/media',
|
||||
snapshotId: 'latest'
|
||||
});
|
||||
|
||||
assert.equal(result.path, '/source/media');
|
||||
assert.equal(result.entries.length, 2);
|
||||
assert.deepEqual(
|
||||
result.entries.map((entry) => ({
|
||||
type: entry.type,
|
||||
path: entry.path
|
||||
})),
|
||||
[
|
||||
{ type: 'dir', path: '/source/media/movies' },
|
||||
{ type: 'file', path: '/source/media/readme.txt' }
|
||||
]
|
||||
);
|
||||
} finally {
|
||||
await session.database.detachDatabase('source');
|
||||
await session.database.close();
|
||||
session.fixture.cleanup();
|
||||
session.appDb.cleanup();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,743 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { createServer } from 'node:http';
|
||||
import { rm } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { createServerApp } from '../src/server.js';
|
||||
import {
|
||||
createDuplicatiFixture,
|
||||
createDuplicatiServerDbFixture
|
||||
} from './support/duplicatiFixture.js';
|
||||
|
||||
async function startTestServer() {
|
||||
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-server-'));
|
||||
const appDbPath = path.join(tempDirectory, 'app.sqlite');
|
||||
const uploadDir = path.join(tempDirectory, 'sources');
|
||||
const runtime = await createServerApp({
|
||||
appDbPath,
|
||||
uploadDir,
|
||||
previewMaxBytes: 1024
|
||||
});
|
||||
|
||||
const server = await new Promise((resolve) => {
|
||||
const instance = runtime.app.listen(0, () => resolve(instance));
|
||||
});
|
||||
const address = server.address();
|
||||
const baseUrl = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
return {
|
||||
baseUrl,
|
||||
appDbPath,
|
||||
uploadDir,
|
||||
runtime,
|
||||
server,
|
||||
async close() {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
await runtime.close();
|
||||
await rm(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadFixture(baseUrl, fixture) {
|
||||
return uploadDatabase(baseUrl, '/api/sources/upload', fixture);
|
||||
}
|
||||
|
||||
async function uploadServerDbFixture(baseUrl, fixture) {
|
||||
return uploadDatabase(baseUrl, '/api/server-db/upload', fixture);
|
||||
}
|
||||
|
||||
async function uploadDatabase(baseUrl, endpoint, fixture) {
|
||||
const form = new FormData();
|
||||
const bytes = await fs.promises.readFile(fixture.databasePath);
|
||||
form.set('database', new File([bytes], path.basename(fixture.databasePath), { type: 'application/octet-stream' }));
|
||||
|
||||
const response = await fetch(`${baseUrl}${endpoint}`, {
|
||||
method: 'POST',
|
||||
body: form
|
||||
});
|
||||
const payload = await response.json();
|
||||
return { response, payload };
|
||||
}
|
||||
|
||||
function createBasicHeader(username, password) {
|
||||
return `Basic ${Buffer.from(`${username}:${password}`, 'utf8').toString('base64')}`;
|
||||
}
|
||||
|
||||
async function startRemoteVolumeServer({ remoteDir, username, password }) {
|
||||
const expectedAuth = createBasicHeader(username, password);
|
||||
const server = createServer(async (request, response) => {
|
||||
if ((request.headers.authorization ?? '') !== expectedAuth) {
|
||||
response.writeHead(401, { 'WWW-Authenticate': 'Basic realm="test"' });
|
||||
response.end('unauthorized');
|
||||
return;
|
||||
}
|
||||
|
||||
const filename = decodeURIComponent(new URL(request.url, 'http://127.0.0.1').pathname.slice(1));
|
||||
const targetPath = path.join(remoteDir, filename);
|
||||
|
||||
try {
|
||||
const stat = await fs.promises.stat(targetPath);
|
||||
response.writeHead(200, {
|
||||
'Content-Length': stat.size,
|
||||
'Content-Type': 'application/octet-stream'
|
||||
});
|
||||
fs.createReadStream(targetPath).pipe(response);
|
||||
} catch (error) {
|
||||
response.writeHead(404);
|
||||
response.end('missing');
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve) => server.listen(0, resolve));
|
||||
const address = server.address();
|
||||
const baseUrl = `http://127.0.0.1:${address.port}/`;
|
||||
|
||||
return {
|
||||
baseUrl,
|
||||
server,
|
||||
async close() {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForEnhancementStatus(baseUrl, sourceId, allowedStatuses) {
|
||||
for (let attempt = 0; attempt < 120; attempt += 1) {
|
||||
const payload = await fetch(`${baseUrl}/api/sources/${sourceId}/enhance/status`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
if (allowedStatuses.includes(payload.enhancement.status)) {
|
||||
return payload.enhancement;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
throw new Error(`Timed out waiting for enhancement status ${allowedStatuses.join(', ')}`);
|
||||
}
|
||||
|
||||
test('multi-source uploads coexist, duplicate uploads reuse the existing source, and sourceId becomes required', async () => {
|
||||
const server = await startTestServer();
|
||||
const firstFixture = createDuplicatiFixture({
|
||||
filename: 'first.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
const secondFixture = createDuplicatiFixture({
|
||||
filename: 'second.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
demoFileName: 'demo-v2.mp4',
|
||||
demoFileHash: 'filehash-demo-v2=='
|
||||
});
|
||||
|
||||
try {
|
||||
const emptySources = await fetch(`${server.baseUrl}/api/sources`).then((response) => response.json());
|
||||
assert.equal(emptySources.sources.length, 0);
|
||||
|
||||
const firstUpload = await uploadFixture(server.baseUrl, firstFixture);
|
||||
assert.equal(firstUpload.response.status, 201);
|
||||
assert.equal(firstUpload.payload.reused, false);
|
||||
|
||||
const duplicateUpload = await uploadFixture(server.baseUrl, firstFixture);
|
||||
assert.equal(duplicateUpload.response.status, 200);
|
||||
assert.equal(duplicateUpload.payload.reused, true);
|
||||
assert.equal(duplicateUpload.payload.source.id, firstUpload.payload.source.id);
|
||||
|
||||
const secondUpload = await uploadFixture(server.baseUrl, secondFixture);
|
||||
assert.equal(secondUpload.response.status, 201);
|
||||
assert.notEqual(secondUpload.payload.source.id, firstUpload.payload.source.id);
|
||||
|
||||
const compatCurrent = await fetch(`${server.baseUrl}/api/source/current`);
|
||||
const compatPayload = await compatCurrent.json();
|
||||
assert.equal(compatCurrent.status, 409);
|
||||
assert.equal(compatPayload.error.code, 'SOURCE_ID_REQUIRED');
|
||||
|
||||
const missingSourceId = await fetch(`${server.baseUrl}/api/ls?path=/source/media/movies`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(missingSourceId.error.code, 'SOURCE_ID_REQUIRED');
|
||||
|
||||
const firstListing = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${firstUpload.payload.source.id}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
assert.equal(firstListing.sourceId, firstUpload.payload.source.id);
|
||||
|
||||
const secondListing = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${secondUpload.payload.source.id}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
assert.equal(secondListing.sourceId, secondUpload.payload.source.id);
|
||||
|
||||
const filesInSourceRoot = await fs.promises.readdir(server.uploadDir, { withFileTypes: true });
|
||||
const sourceDirectories = filesInSourceRoot.filter(
|
||||
(entry) => entry.isDirectory() && entry.name !== '_staging' && entry.name !== '_server'
|
||||
);
|
||||
assert.equal(sourceDirectories.length, 2);
|
||||
} finally {
|
||||
firstFixture.cleanup();
|
||||
secondFixture.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting a single source removes only that source and keeps the server database intact', async () => {
|
||||
const server = await startTestServer();
|
||||
const sourceA = createDuplicatiFixture({
|
||||
filename: 'delete-a.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
const sourceB = createDuplicatiFixture({
|
||||
filename: 'delete-b.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
demoFileName: 'delete-b.mp4',
|
||||
demoFileHash: 'delete-b-hash=='
|
||||
});
|
||||
const serverDb = createDuplicatiServerDbFixture({
|
||||
backups: [
|
||||
{ id: 21, name: 'Delete A', dbPath: '/data/Duplicati/delete-a.sqlite' },
|
||||
{ id: 22, name: 'Delete B', dbPath: '/data/Duplicati/delete-b.sqlite' }
|
||||
]
|
||||
});
|
||||
|
||||
try {
|
||||
const uploadA = await uploadFixture(server.baseUrl, sourceA);
|
||||
const uploadB = await uploadFixture(server.baseUrl, sourceB);
|
||||
const deleteSourceDir = path.join(server.uploadDir, uploadA.payload.source.id);
|
||||
const keepSourceDir = path.join(server.uploadDir, uploadB.payload.source.id);
|
||||
|
||||
const serverDbUpload = await uploadServerDbFixture(server.baseUrl, serverDb);
|
||||
assert.equal(serverDbUpload.response.status, 201);
|
||||
|
||||
const deleteResponse = await fetch(`${server.baseUrl}/api/sources/${uploadA.payload.source.id}`, {
|
||||
method: 'DELETE'
|
||||
});
|
||||
const deletePayload = await deleteResponse.json();
|
||||
assert.equal(deleteResponse.status, 200);
|
||||
assert.equal(deletePayload.deleted.id, uploadA.payload.source.id);
|
||||
|
||||
const remainingSources = await fetch(`${server.baseUrl}/api/sources`).then((response) => response.json());
|
||||
assert.equal(remainingSources.sources.length, 1);
|
||||
assert.equal(remainingSources.sources[0].id, uploadB.payload.source.id);
|
||||
|
||||
const deletedLookup = await fetch(`${server.baseUrl}/api/sources/${uploadA.payload.source.id}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(deletedLookup.error.code, 'SOURCE_NOT_FOUND');
|
||||
|
||||
await assert.rejects(() => fs.promises.stat(deleteSourceDir));
|
||||
const keptStat = await fs.promises.stat(keepSourceDir);
|
||||
assert.equal(keptStat.isDirectory(), true);
|
||||
|
||||
const serverDbSummary = await fetch(`${server.baseUrl}/api/server-db`).then((response) => response.json());
|
||||
assert.equal(serverDbSummary.serverDb.available, true);
|
||||
assert.equal(serverDbSummary.serverDb.backupCount, 2);
|
||||
} finally {
|
||||
sourceA.cleanup();
|
||||
sourceB.cleanup();
|
||||
serverDb.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('server db upload maps task names onto existing sources and replacement remaps them', async () => {
|
||||
const server = await startTestServer();
|
||||
const sourceA = createDuplicatiFixture({
|
||||
filename: 'TMQRJYNADS.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
const sourceB = createDuplicatiFixture({
|
||||
filename: 'ENVPLWAIWJ.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
demoFileName: 'env-demo.mp4',
|
||||
demoFileHash: 'env-demo-hash=='
|
||||
});
|
||||
const firstServerDb = createDuplicatiServerDbFixture({
|
||||
backups: [
|
||||
{ id: 2, name: '四川农场主', dbPath: '/data/Duplicati/TMQRJYNADS.sqlite' },
|
||||
{ id: 3, name: '小白', dbPath: '/data/Duplicati/ENVPLWAIWJ.sqlite' }
|
||||
]
|
||||
});
|
||||
const replacementServerDb = createDuplicatiServerDbFixture({
|
||||
backups: [
|
||||
{ id: 2, name: '四川农场主-新', dbPath: '/data/Duplicati/TMQRJYNADS.sqlite' },
|
||||
{ id: 3, name: '小白-新', dbPath: '/data/Duplicati/ENVPLWAIWJ.sqlite' }
|
||||
]
|
||||
});
|
||||
|
||||
try {
|
||||
const invalidServerDbUpload = await uploadDatabase(server.baseUrl, '/api/sources/upload', firstServerDb);
|
||||
assert.equal(invalidServerDbUpload.response.status, 422);
|
||||
assert.equal(invalidServerDbUpload.payload.error.code, 'INVALID_SOURCE_SCHEMA');
|
||||
|
||||
const uploadA = await uploadFixture(server.baseUrl, sourceA);
|
||||
const uploadB = await uploadFixture(server.baseUrl, sourceB);
|
||||
assert.equal(uploadA.payload.source.displayName, 'TMQRJYNADS.sqlite');
|
||||
assert.equal(uploadA.payload.source.displayNameSource, 'filename');
|
||||
|
||||
const firstServerDbUpload = await uploadServerDbFixture(server.baseUrl, firstServerDb);
|
||||
assert.equal(firstServerDbUpload.response.status, 201);
|
||||
assert.equal(firstServerDbUpload.payload.serverDb.available, true);
|
||||
assert.equal(firstServerDbUpload.payload.serverDb.backupCount, 2);
|
||||
|
||||
const sourcesAfterMapping = await fetch(`${server.baseUrl}/api/sources`).then((response) => response.json());
|
||||
const mappedA = sourcesAfterMapping.sources.find((source) => source.id === uploadA.payload.source.id);
|
||||
const mappedB = sourcesAfterMapping.sources.find((source) => source.id === uploadB.payload.source.id);
|
||||
assert.equal(mappedA.displayName, '四川农场主');
|
||||
assert.equal(mappedA.displayNameSource, 'server-db');
|
||||
assert.equal(mappedA.matchedBackupName, '四川农场主');
|
||||
assert.equal(mappedB.displayName, '小白');
|
||||
|
||||
const serverDbSummary = await fetch(`${server.baseUrl}/api/server-db`).then((response) => response.json());
|
||||
assert.equal(serverDbSummary.serverDb.available, true);
|
||||
assert.equal(serverDbSummary.serverDb.backupCount, 2);
|
||||
|
||||
const replacementUpload = await uploadServerDbFixture(server.baseUrl, replacementServerDb);
|
||||
assert.equal(replacementUpload.response.status, 201);
|
||||
|
||||
const sourcesAfterReplacement = await fetch(`${server.baseUrl}/api/sources`).then((response) => response.json());
|
||||
const remappedA = sourcesAfterReplacement.sources.find((source) => source.id === uploadA.payload.source.id);
|
||||
const remappedB = sourcesAfterReplacement.sources.find((source) => source.id === uploadB.payload.source.id);
|
||||
assert.equal(remappedA.displayName, '四川农场主-新');
|
||||
assert.equal(remappedB.displayName, '小白-新');
|
||||
} finally {
|
||||
sourceA.cleanup();
|
||||
sourceB.cleanup();
|
||||
firstServerDb.cleanup();
|
||||
replacementServerDb.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('global WebDAV defaults plus server-db target URL allow enhancement without per-source credential re-entry', async () => {
|
||||
const server = await startTestServer();
|
||||
const fixture = createDuplicatiFixture({
|
||||
filename: 'global-defaults.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
withRemoteVolumes: true
|
||||
});
|
||||
const remote = await startRemoteVolumeServer({
|
||||
remoteDir: fixture.remoteDir,
|
||||
username: 'demo',
|
||||
password: 'secret'
|
||||
});
|
||||
const serverDb = createDuplicatiServerDbFixture({
|
||||
backups: [
|
||||
{
|
||||
id: 7,
|
||||
name: 'Global Defaults Demo',
|
||||
dbPath: '/data/Duplicati/global-defaults.sqlite',
|
||||
targetUrl: remote.baseUrl.replace('http://', 'webdav://')
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
try {
|
||||
const upload = await uploadFixture(server.baseUrl, fixture);
|
||||
const sourceId = upload.payload.source.id;
|
||||
|
||||
const defaultsResponse = await fetch(`${server.baseUrl}/api/webdav-defaults`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
authMode: 'basic',
|
||||
username: 'demo',
|
||||
password: 'secret',
|
||||
passphrase: fixture.passphrase
|
||||
})
|
||||
});
|
||||
const defaultsPayload = await defaultsResponse.json();
|
||||
assert.equal(defaultsResponse.status, 200);
|
||||
assert.equal(defaultsPayload.defaults.configured, true);
|
||||
assert.equal(defaultsPayload.defaults.hasPassphrase, true);
|
||||
|
||||
const serverDbUpload = await uploadServerDbFixture(server.baseUrl, serverDb);
|
||||
assert.equal(serverDbUpload.response.status, 201);
|
||||
|
||||
const sourceDetail = await fetch(`${server.baseUrl}/api/sources/${sourceId}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(sourceDetail.source.webdav.derivedWebdavBaseUrl, remote.baseUrl);
|
||||
assert.equal(sourceDetail.source.webdav.effectiveWebdavBaseUrl, remote.baseUrl);
|
||||
assert.equal(sourceDetail.source.webdav.effectiveWebdavBaseUrlSource, 'server-db');
|
||||
|
||||
const browserSecrets = await fetch(`${server.baseUrl}/api/sources/${sourceId}/browser-secrets`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(browserSecrets.secrets.webdavBaseUrl, remote.baseUrl);
|
||||
assert.equal(browserSecrets.secrets.username, 'demo');
|
||||
assert.equal(browserSecrets.secrets.password, 'secret');
|
||||
assert.equal(browserSecrets.secrets.passphrase, fixture.passphrase);
|
||||
assert.equal(browserSecrets.secrets.authMode, 'basic');
|
||||
|
||||
const enhanceResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/enhance`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({})
|
||||
});
|
||||
assert.equal(enhanceResponse.status, 202);
|
||||
|
||||
const enhancement = await waitForEnhancementStatus(server.baseUrl, sourceId, ['ready', 'failed']);
|
||||
assert.equal(enhancement.status, 'ready');
|
||||
} finally {
|
||||
await remote.close();
|
||||
fixture.cleanup();
|
||||
serverDb.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('raw source keeps browsing available, gates file-info, and does not leak saved secrets', async () => {
|
||||
const server = await startTestServer();
|
||||
const rawFixture = createDuplicatiFixture({
|
||||
filename: 'raw-job.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false
|
||||
});
|
||||
|
||||
try {
|
||||
const upload = await uploadFixture(server.baseUrl, rawFixture);
|
||||
assert.equal(upload.response.status, 201);
|
||||
|
||||
const sourceId = upload.payload.source.id;
|
||||
const listing = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const fileId = listing.entries.find((entry) => entry.type === 'file')?.id;
|
||||
assert.ok(fileId);
|
||||
|
||||
const fileInfoBeforeEnhance = await fetch(
|
||||
`${server.baseUrl}/api/file-info?sourceId=${sourceId}&id=${encodeURIComponent(fileId)}`
|
||||
).then((response) => response.json());
|
||||
assert.equal(fileInfoBeforeEnhance.error.code, 'ENHANCEMENT_NOT_READY');
|
||||
|
||||
const saveSecretsResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/secrets`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webdavBaseUrl: 'http://127.0.0.1:9999/remote/',
|
||||
username: 'demo',
|
||||
password: 'secret',
|
||||
passphrase: 'passphrase',
|
||||
authMode: 'basic'
|
||||
})
|
||||
});
|
||||
const saveSecretsPayload = await saveSecretsResponse.json();
|
||||
assert.equal(saveSecretsResponse.status, 200);
|
||||
assert.equal(saveSecretsPayload.source.credentialsSaved, true);
|
||||
assert.equal(saveSecretsPayload.source.password, undefined);
|
||||
assert.equal(saveSecretsPayload.source.passphrase, undefined);
|
||||
|
||||
const sourceDetail = await fetch(`${server.baseUrl}/api/sources/${sourceId}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(sourceDetail.source.credentialsSaved, true);
|
||||
assert.equal(sourceDetail.source.username, undefined);
|
||||
assert.equal(sourceDetail.source.password, undefined);
|
||||
} finally {
|
||||
rawFixture.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('enhancement job builds archive indexes from Basic-auth WebDAV volumes and old file ids stay valid', async () => {
|
||||
const server = await startTestServer();
|
||||
const fixture = createDuplicatiFixture({
|
||||
filename: 'enhance.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
withRemoteVolumes: true,
|
||||
includeAesExtensions: true
|
||||
});
|
||||
const remote = await startRemoteVolumeServer({
|
||||
remoteDir: fixture.remoteDir,
|
||||
username: 'demo',
|
||||
password: 'secret'
|
||||
});
|
||||
|
||||
try {
|
||||
const upload = await uploadFixture(server.baseUrl, fixture);
|
||||
const sourceId = upload.payload.source.id;
|
||||
|
||||
const listing = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const fileId = listing.entries.find((entry) => entry.type === 'file')?.id;
|
||||
assert.ok(fileId);
|
||||
|
||||
const enhanceResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/enhance`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webdavBaseUrl: remote.baseUrl,
|
||||
username: 'demo',
|
||||
password: 'secret',
|
||||
passphrase: fixture.passphrase,
|
||||
authMode: 'basic'
|
||||
})
|
||||
});
|
||||
assert.equal(enhanceResponse.status, 202);
|
||||
|
||||
const enhancement = await waitForEnhancementStatus(server.baseUrl, sourceId, ['ready', 'failed']);
|
||||
assert.equal(enhancement.status, 'ready');
|
||||
assert.equal(enhancement.processedVolumes, 2);
|
||||
assert.equal(enhancement.totalVolumes, 2);
|
||||
|
||||
const sourceDetail = await fetch(`${server.baseUrl}/api/sources/${sourceId}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(sourceDetail.source.capabilities.archiveEntryIndex, true);
|
||||
assert.equal(sourceDetail.source.capabilities.volumeCryptoCache, true);
|
||||
|
||||
const fileInfo = await fetch(
|
||||
`${server.baseUrl}/api/file-info?sourceId=${sourceId}&id=${encodeURIComponent(fileId)}`
|
||||
).then((response) => response.json());
|
||||
assert.equal(fileInfo.file.path, '/source/media/movies/demo.mp4');
|
||||
assert.equal(fileInfo.restorePlan.segmentCount, 3);
|
||||
assert.equal(fileInfo.requiredDblocks[0], 'duplicati-demo.dblock.zip.aes');
|
||||
assert.equal(fileInfo.volumes[0].encryption.streamFormat, 'v2');
|
||||
} finally {
|
||||
await remote.close();
|
||||
fixture.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('video thumbnails can be uploaded, listed, fetched, deleted, and cleared per source', async () => {
|
||||
const server = await startTestServer();
|
||||
const fixture = createDuplicatiFixture({
|
||||
filename: 'thumbs.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
withRemoteVolumes: true
|
||||
});
|
||||
const remote = await startRemoteVolumeServer({
|
||||
remoteDir: fixture.remoteDir,
|
||||
username: 'demo',
|
||||
password: 'secret'
|
||||
});
|
||||
|
||||
try {
|
||||
const upload = await uploadFixture(server.baseUrl, fixture);
|
||||
const sourceId = upload.payload.source.id;
|
||||
|
||||
const enhanceResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/enhance`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webdavBaseUrl: remote.baseUrl,
|
||||
username: 'demo',
|
||||
password: 'secret',
|
||||
passphrase: fixture.passphrase,
|
||||
authMode: 'basic'
|
||||
})
|
||||
});
|
||||
assert.equal(enhanceResponse.status, 202);
|
||||
|
||||
const enhancement = await waitForEnhancementStatus(server.baseUrl, sourceId, ['ready', 'failed']);
|
||||
assert.equal(enhancement.status, 'ready');
|
||||
|
||||
const listingBefore = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const fileEntry = listingBefore.entries.find((entry) => entry.type === 'file');
|
||||
assert.ok(fileEntry);
|
||||
assert.equal(fileEntry.thumbnail.available, false);
|
||||
|
||||
const fileInfoBefore = await fetch(
|
||||
`${server.baseUrl}/api/file-info?sourceId=${sourceId}&id=${encodeURIComponent(fileEntry.id)}`
|
||||
).then((response) => response.json());
|
||||
assert.equal(fileInfoBefore.file.thumbnail.available, false);
|
||||
|
||||
const thumbnailForm = new FormData();
|
||||
thumbnailForm.set('fileId', fileEntry.id);
|
||||
thumbnailForm.set(
|
||||
'image',
|
||||
new File([Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0x10, 0x00, 0x00, 0x00])], 'preview.webp', {
|
||||
type: 'image/webp'
|
||||
})
|
||||
);
|
||||
|
||||
const thumbnailUploadResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/thumbnails`, {
|
||||
method: 'POST',
|
||||
body: thumbnailForm
|
||||
});
|
||||
const thumbnailUploadPayload = await thumbnailUploadResponse.json();
|
||||
assert.equal(thumbnailUploadResponse.status, 201);
|
||||
assert.equal(thumbnailUploadPayload.thumbnail.available, true);
|
||||
assert.ok(thumbnailUploadPayload.thumbnail.thumbnailId);
|
||||
|
||||
const thumbnailFetchResponse = await fetch(`${server.baseUrl}${thumbnailUploadPayload.thumbnail.thumbnailUrl}`);
|
||||
assert.equal(thumbnailFetchResponse.status, 200);
|
||||
assert.equal(thumbnailFetchResponse.headers.get('content-type'), 'image/webp');
|
||||
assert.deepEqual(new Uint8Array(await thumbnailFetchResponse.arrayBuffer()), Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0x10, 0x00, 0x00, 0x00]));
|
||||
|
||||
const listingAfterUpload = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const listedFileAfterUpload = listingAfterUpload.entries.find((entry) => entry.id === fileEntry.id);
|
||||
assert.equal(listedFileAfterUpload.thumbnail.available, true);
|
||||
assert.equal(listedFileAfterUpload.thumbnail.thumbnailId, thumbnailUploadPayload.thumbnail.thumbnailId);
|
||||
|
||||
const fileInfoAfterUpload = await fetch(
|
||||
`${server.baseUrl}/api/file-info?sourceId=${sourceId}&id=${encodeURIComponent(fileEntry.id)}`
|
||||
).then((response) => response.json());
|
||||
assert.equal(fileInfoAfterUpload.file.thumbnail.available, true);
|
||||
|
||||
const sourceDetailAfterUpload = await fetch(`${server.baseUrl}/api/sources/${sourceId}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(sourceDetailAfterUpload.source.thumbnailCache.count, 1);
|
||||
|
||||
const singleDeleteResponse = await fetch(
|
||||
`${server.baseUrl}/api/sources/${sourceId}/thumbnails/${encodeURIComponent(thumbnailUploadPayload.thumbnail.thumbnailId)}`,
|
||||
{
|
||||
method: 'DELETE'
|
||||
}
|
||||
);
|
||||
const singleDeletePayload = await singleDeleteResponse.json();
|
||||
assert.equal(singleDeleteResponse.status, 200);
|
||||
assert.equal(singleDeletePayload.deleted.fileId, fileEntry.id);
|
||||
|
||||
const listingAfterSingleDelete = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const listedFileAfterDelete = listingAfterSingleDelete.entries.find((entry) => entry.id === fileEntry.id);
|
||||
assert.equal(listedFileAfterDelete.thumbnail.available, false);
|
||||
|
||||
const secondThumbnailForm = new FormData();
|
||||
secondThumbnailForm.set('fileId', fileEntry.id);
|
||||
secondThumbnailForm.set(
|
||||
'image',
|
||||
new File([Uint8Array.from([0x57, 0x45, 0x42, 0x50])], 'preview-2.webp', {
|
||||
type: 'image/webp'
|
||||
})
|
||||
);
|
||||
const secondUploadResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/thumbnails`, {
|
||||
method: 'POST',
|
||||
body: secondThumbnailForm
|
||||
});
|
||||
assert.equal(secondUploadResponse.status, 201);
|
||||
|
||||
const clearResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/thumbnails`, {
|
||||
method: 'DELETE'
|
||||
});
|
||||
const clearPayload = await clearResponse.json();
|
||||
assert.equal(clearResponse.status, 200);
|
||||
assert.equal(clearPayload.cleared.removedCount, 1);
|
||||
|
||||
const sourceDetailAfterClear = await fetch(`${server.baseUrl}/api/sources/${sourceId}`).then((response) =>
|
||||
response.json()
|
||||
);
|
||||
assert.equal(sourceDetailAfterClear.source.thumbnailCache.count, 0);
|
||||
} finally {
|
||||
await remote.close();
|
||||
fixture.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('failed enhancement keeps directory browsing available and surfaces ENHANCEMENT_FAILED', async () => {
|
||||
const server = await startTestServer();
|
||||
const fixture = createDuplicatiFixture({
|
||||
filename: 'enhance-fail.sqlite',
|
||||
layout: 'file_lookup',
|
||||
includeArchiveEntryIndex: false,
|
||||
includeVolumeCryptoCache: false,
|
||||
withRemoteVolumes: true
|
||||
});
|
||||
const remote = await startRemoteVolumeServer({
|
||||
remoteDir: fixture.remoteDir,
|
||||
username: 'demo',
|
||||
password: 'secret'
|
||||
});
|
||||
|
||||
try {
|
||||
const upload = await uploadFixture(server.baseUrl, fixture);
|
||||
const sourceId = upload.payload.source.id;
|
||||
|
||||
const listingBefore = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
const fileId = listingBefore.entries.find((entry) => entry.type === 'file')?.id;
|
||||
assert.ok(fileId);
|
||||
|
||||
const enhanceResponse = await fetch(`${server.baseUrl}/api/sources/${sourceId}/enhance`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webdavBaseUrl: remote.baseUrl,
|
||||
username: 'demo',
|
||||
password: 'secret',
|
||||
passphrase: 'wrong-passphrase',
|
||||
authMode: 'basic'
|
||||
})
|
||||
});
|
||||
assert.equal(enhanceResponse.status, 202);
|
||||
|
||||
const enhancement = await waitForEnhancementStatus(server.baseUrl, sourceId, ['ready', 'failed']);
|
||||
assert.equal(enhancement.status, 'failed');
|
||||
|
||||
const listingAfter = await fetch(
|
||||
`${server.baseUrl}/api/ls?sourceId=${sourceId}&path=/source/media/movies`
|
||||
).then((response) => response.json());
|
||||
assert.equal(listingAfter.entries.length, listingBefore.entries.length);
|
||||
|
||||
const fileInfo = await fetch(
|
||||
`${server.baseUrl}/api/file-info?sourceId=${sourceId}&id=${encodeURIComponent(fileId)}`
|
||||
).then((response) => response.json());
|
||||
assert.equal(fileInfo.error.code, 'ENHANCEMENT_FAILED');
|
||||
} finally {
|
||||
await remote.close();
|
||||
fixture.cleanup();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,429 @@
|
||||
import { createCipheriv, createHash, createHmac, randomBytes } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { deflateRawSync } from 'node:zlib';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
function createCrc32Table() {
|
||||
const table = new Uint32Array(256);
|
||||
for (let index = 0; index < 256; index += 1) {
|
||||
let value = index;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
||||
}
|
||||
table[index] = value >>> 0;
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
|
||||
const CRC32_TABLE = createCrc32Table();
|
||||
|
||||
function crc32(buffer) {
|
||||
let value = 0xffffffff;
|
||||
for (const byte of buffer) {
|
||||
value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
|
||||
}
|
||||
|
||||
return (value ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function stretchPasswordV2(passphrase, externalIv) {
|
||||
const passwordBytes = Buffer.from(String(passphrase), 'utf16le');
|
||||
let digest = Buffer.concat([externalIv, Buffer.alloc(16, 0)]);
|
||||
|
||||
for (let index = 0; index < 8192; index += 1) {
|
||||
const hash = createHash('sha256');
|
||||
hash.update(digest);
|
||||
hash.update(passwordBytes);
|
||||
digest = hash.digest();
|
||||
}
|
||||
|
||||
return digest;
|
||||
}
|
||||
|
||||
function createAesCryptV2Buffer(plainBuffer, passphrase, options = {}) {
|
||||
const externalIv = randomBytes(16);
|
||||
const internalIv = randomBytes(16);
|
||||
const internalKey = randomBytes(32);
|
||||
const stretchedKey = stretchPasswordV2(passphrase, externalIv);
|
||||
const sessionBytes = Buffer.concat([internalIv, internalKey]);
|
||||
|
||||
const headerCipher = createCipheriv('aes-256-cbc', stretchedKey, externalIv);
|
||||
headerCipher.setAutoPadding(false);
|
||||
const encryptedSession = Buffer.concat([
|
||||
headerCipher.update(sessionBytes),
|
||||
headerCipher.final()
|
||||
]);
|
||||
|
||||
const headerHmac = createHmac('sha256', stretchedKey).update(encryptedSession).digest();
|
||||
|
||||
const fileSizeModulo = plainBuffer.length % 16;
|
||||
const padLength = fileSizeModulo === 0 ? 0 : 16 - fileSizeModulo;
|
||||
const paddedPlain = padLength > 0
|
||||
? Buffer.concat([plainBuffer, Buffer.alloc(padLength, padLength)])
|
||||
: plainBuffer;
|
||||
|
||||
const payloadCipher = createCipheriv('aes-256-cbc', internalKey, internalIv);
|
||||
payloadCipher.setAutoPadding(false);
|
||||
const ciphertext = Buffer.concat([
|
||||
payloadCipher.update(paddedPlain),
|
||||
payloadCipher.final()
|
||||
]);
|
||||
const payloadHmac = createHmac('sha256', internalKey).update(ciphertext).digest();
|
||||
|
||||
const extensionRecords = options.extensionRecords ?? [];
|
||||
const extensionBytes = extensionRecords.flatMap((record) => {
|
||||
const id = Buffer.from(record.id);
|
||||
const data = Buffer.from(record.data ?? []);
|
||||
const payload = Buffer.concat([id, data]);
|
||||
const length = Buffer.alloc(2);
|
||||
length.writeUInt16BE(payload.length, 0);
|
||||
return [length, payload];
|
||||
});
|
||||
|
||||
return Buffer.concat([
|
||||
Buffer.from('AES', 'utf8'),
|
||||
Buffer.from([2, 0]),
|
||||
...extensionBytes,
|
||||
Buffer.from([0, 0]),
|
||||
externalIv,
|
||||
encryptedSession,
|
||||
headerHmac,
|
||||
ciphertext,
|
||||
Buffer.from([fileSizeModulo]),
|
||||
payloadHmac
|
||||
]);
|
||||
}
|
||||
|
||||
function createZipBuffer(entries) {
|
||||
const localParts = [];
|
||||
const centralParts = [];
|
||||
let localOffset = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
const nameBuffer = Buffer.from(entry.name, 'utf8');
|
||||
const input = Buffer.from(entry.data);
|
||||
const compressedData =
|
||||
entry.compression === 'store' ? input : deflateRawSync(input);
|
||||
const compressionMethod = entry.compression === 'store' ? 0 : 8;
|
||||
const crc = crc32(input);
|
||||
|
||||
const localHeader = Buffer.alloc(30);
|
||||
localHeader.writeUInt32LE(0x04034b50, 0);
|
||||
localHeader.writeUInt16LE(20, 4);
|
||||
localHeader.writeUInt16LE(0, 6);
|
||||
localHeader.writeUInt16LE(compressionMethod, 8);
|
||||
localHeader.writeUInt16LE(0, 10);
|
||||
localHeader.writeUInt16LE(0, 12);
|
||||
localHeader.writeUInt32LE(crc, 14);
|
||||
localHeader.writeUInt32LE(compressedData.length, 18);
|
||||
localHeader.writeUInt32LE(input.length, 22);
|
||||
localHeader.writeUInt16LE(nameBuffer.length, 26);
|
||||
localHeader.writeUInt16LE(0, 28);
|
||||
|
||||
localParts.push(localHeader, nameBuffer, compressedData);
|
||||
|
||||
const centralHeader = Buffer.alloc(46);
|
||||
centralHeader.writeUInt32LE(0x02014b50, 0);
|
||||
centralHeader.writeUInt16LE(20, 4);
|
||||
centralHeader.writeUInt16LE(20, 6);
|
||||
centralHeader.writeUInt16LE(0, 8);
|
||||
centralHeader.writeUInt16LE(compressionMethod, 10);
|
||||
centralHeader.writeUInt16LE(0, 12);
|
||||
centralHeader.writeUInt16LE(0, 14);
|
||||
centralHeader.writeUInt32LE(crc, 16);
|
||||
centralHeader.writeUInt32LE(compressedData.length, 20);
|
||||
centralHeader.writeUInt32LE(input.length, 24);
|
||||
centralHeader.writeUInt16LE(nameBuffer.length, 28);
|
||||
centralHeader.writeUInt16LE(0, 30);
|
||||
centralHeader.writeUInt16LE(0, 32);
|
||||
centralHeader.writeUInt16LE(0, 34);
|
||||
centralHeader.writeUInt16LE(0, 36);
|
||||
centralHeader.writeUInt32LE(0, 38);
|
||||
centralHeader.writeUInt32LE(localOffset, 42);
|
||||
|
||||
centralParts.push(centralHeader, nameBuffer);
|
||||
localOffset += localHeader.length + nameBuffer.length + compressedData.length;
|
||||
}
|
||||
|
||||
const centralDirectory = Buffer.concat(centralParts);
|
||||
const localData = Buffer.concat(localParts);
|
||||
const eocd = Buffer.alloc(22);
|
||||
eocd.writeUInt32LE(0x06054b50, 0);
|
||||
eocd.writeUInt16LE(0, 4);
|
||||
eocd.writeUInt16LE(0, 6);
|
||||
eocd.writeUInt16LE(entries.length, 8);
|
||||
eocd.writeUInt16LE(entries.length, 10);
|
||||
eocd.writeUInt32LE(centralDirectory.length, 12);
|
||||
eocd.writeUInt32LE(localData.length, 16);
|
||||
eocd.writeUInt16LE(0, 20);
|
||||
|
||||
return Buffer.concat([localData, centralDirectory, eocd]);
|
||||
}
|
||||
|
||||
function buildRemoteVolumeBuffers() {
|
||||
const demoEntries = [
|
||||
{
|
||||
name: '0td8NEaS7SMrQc5Gs0Sdxjb_1MXEEuwkyxRpguDiWsY=',
|
||||
data: Buffer.alloc(102400, 0x11),
|
||||
compression: 'deflate'
|
||||
},
|
||||
{
|
||||
name: 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=',
|
||||
data: Buffer.alloc(102400, 0x22),
|
||||
compression: 'deflate'
|
||||
},
|
||||
{
|
||||
name: 'uS_2KMSmm2IWlZ77JiHH1p_yp7Cvhr8CKmRHJNMRqwA=',
|
||||
data: Buffer.alloc(10240, 0x33),
|
||||
compression: 'deflate'
|
||||
}
|
||||
];
|
||||
const textEntries = [
|
||||
{
|
||||
name: 'ySjwbaRrk-rm6Vx0W0xP8A==',
|
||||
data: Buffer.from('hello', 'utf8'),
|
||||
compression: 'store'
|
||||
}
|
||||
];
|
||||
|
||||
return {
|
||||
'duplicati-demo.dblock.zip.aes': createZipBuffer(demoEntries),
|
||||
'duplicati-text.dblock.zip.aes': createZipBuffer(textEntries)
|
||||
};
|
||||
}
|
||||
|
||||
export function createDuplicatiFixture(options = {}) {
|
||||
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-api-'));
|
||||
const databasePath = path.join(tempDirectory, options.filename ?? 'fixture.sqlite');
|
||||
const database = new DatabaseSync(databasePath);
|
||||
const includeArchiveEntryIndex = options.includeArchiveEntryIndex !== false;
|
||||
const includeVolumeCryptoCache = options.includeVolumeCryptoCache !== false;
|
||||
const layout = options.layout ?? 'file';
|
||||
const passphrase = options.passphrase ?? 'test-passphrase';
|
||||
const withRemoteVolumes = options.withRemoteVolumes === true;
|
||||
const includeAesExtensions = options.includeAesExtensions === true;
|
||||
|
||||
database.exec(`
|
||||
CREATE TABLE "Fileset" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Timestamp" INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE "Configuration" (
|
||||
"Key" TEXT PRIMARY KEY,
|
||||
"Value" TEXT NOT NULL
|
||||
);
|
||||
${layout === 'file'
|
||||
? `CREATE TABLE "File" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Path" TEXT NOT NULL,
|
||||
"BlocksetID" INTEGER NOT NULL,
|
||||
"MetadataID" INTEGER NULL
|
||||
);`
|
||||
: `CREATE TABLE "FileLookup" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"PrefixID" INTEGER NOT NULL,
|
||||
"Path" TEXT NOT NULL,
|
||||
"BlocksetID" INTEGER NOT NULL,
|
||||
"MetadataID" INTEGER NULL
|
||||
);
|
||||
CREATE TABLE "PathPrefix" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Prefix" TEXT NOT NULL
|
||||
);`}
|
||||
CREATE TABLE "FilesetEntry" (
|
||||
"FilesetID" INTEGER NOT NULL,
|
||||
"FileID" INTEGER NOT NULL,
|
||||
"Lastmodified" INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE "Blockset" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Length" INTEGER NOT NULL,
|
||||
"FullHash" TEXT NULL
|
||||
);
|
||||
CREATE TABLE "Block" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Hash" TEXT NOT NULL,
|
||||
"Size" INTEGER NOT NULL,
|
||||
"VolumeID" INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE "BlocksetEntry" (
|
||||
"BlocksetID" INTEGER NOT NULL,
|
||||
"Index" INTEGER NOT NULL,
|
||||
"BlockID" INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE "Remotevolume" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"OperationID" INTEGER NULL,
|
||||
"Name" TEXT NOT NULL,
|
||||
"Type" TEXT NULL,
|
||||
"Size" INTEGER NULL,
|
||||
"Hash" TEXT NULL,
|
||||
"State" TEXT NULL,
|
||||
"VerificationCount" INTEGER NULL,
|
||||
"DeleteGraceTime" INTEGER NULL,
|
||||
"ArchiveTime" INTEGER NULL,
|
||||
"LockExpirationTime" INTEGER NULL
|
||||
);
|
||||
${includeArchiveEntryIndex ? `CREATE TABLE "archive_entry_index" (
|
||||
"volume_name" TEXT NOT NULL,
|
||||
"entry_name" TEXT NOT NULL,
|
||||
"local_header_offset_plain" INTEGER NOT NULL,
|
||||
"data_offset_plain" INTEGER NOT NULL,
|
||||
"compressed_size" INTEGER NOT NULL,
|
||||
"uncompressed_size" INTEGER NOT NULL,
|
||||
"compression_method" TEXT NOT NULL,
|
||||
"crc32" TEXT NOT NULL,
|
||||
PRIMARY KEY ("volume_name", "entry_name")
|
||||
);` : ''}
|
||||
${includeVolumeCryptoCache ? `CREATE TABLE "volume_crypto_cache" (
|
||||
"volume_name" TEXT PRIMARY KEY,
|
||||
"stream_format" TEXT NULL,
|
||||
"header_probe_bytes" INTEGER NULL,
|
||||
"kdf_iterations" INTEGER NULL,
|
||||
"salt_hex" TEXT NULL,
|
||||
"iv_hex" TEXT NULL
|
||||
);` : ''}
|
||||
`);
|
||||
|
||||
const fileName = options.demoFileName ?? 'demo.mp4';
|
||||
const fileHash = options.demoFileHash ?? 'filehash-demo==';
|
||||
const fileEntriesSql =
|
||||
layout === 'file'
|
||||
? `
|
||||
INSERT INTO "File" ("ID", "Path", "BlocksetID", "MetadataID") VALUES
|
||||
(10, 'C:\\\\media\\\\movies', -1, NULL),
|
||||
(11, 'C:\\\\media\\\\movies\\\\${fileName}', 101, NULL),
|
||||
(12, 'C:\\\\media\\\\readme.txt', 102, NULL);
|
||||
`
|
||||
: `
|
||||
INSERT INTO "PathPrefix" ("ID", "Prefix") VALUES
|
||||
(1, '/source/media/'),
|
||||
(2, '/source/media/movies/');
|
||||
INSERT INTO "FileLookup" ("ID", "PrefixID", "Path", "BlocksetID", "MetadataID") VALUES
|
||||
(10, 1, 'movies/', -100, NULL),
|
||||
(11, 2, '${fileName}', 101, NULL),
|
||||
(12, 1, 'readme.txt', 102, NULL);
|
||||
`;
|
||||
|
||||
database.exec(`
|
||||
INSERT INTO "Fileset" ("ID", "Timestamp") VALUES (1, 1714896000);
|
||||
INSERT INTO "Configuration" ("Key", "Value") VALUES
|
||||
('blocksize', '102400'),
|
||||
('blockhash', 'SHA256'),
|
||||
('filehash', 'SHA256');
|
||||
${fileEntriesSql}
|
||||
INSERT INTO "FilesetEntry" ("FilesetID", "FileID", "Lastmodified") VALUES
|
||||
(1, 10, 638504448000000000),
|
||||
(1, 11, 638504448110000000),
|
||||
(1, 12, 638504448220000000);
|
||||
INSERT INTO "Blockset" ("ID", "Length", "FullHash") VALUES
|
||||
(101, 215040, '${fileHash}'),
|
||||
(102, 5, 'filehash-readme==');
|
||||
INSERT INTO "Block" ("ID", "Hash", "Size", "VolumeID") VALUES
|
||||
(201, '0td8NEaS7SMrQc5Gs0Sdxjb/1MXEEuwkyxRpguDiWsY=', 102400, 301),
|
||||
(202, 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=', 102400, 301),
|
||||
(203, 'uS/2KMSmm2IWlZ77JiHH1p/yp7Cvhr8CKmRHJNMRqwA=', 10240, 301),
|
||||
(204, 'ySjwbaRrk+rm6Vx0W0xP8A==', 5, 302);
|
||||
INSERT INTO "BlocksetEntry" ("BlocksetID", "Index", "BlockID") VALUES
|
||||
(101, 0, 201),
|
||||
(101, 1, 202),
|
||||
(101, 2, 203),
|
||||
(102, 0, 204);
|
||||
INSERT INTO "Remotevolume" ("ID", "OperationID", "Name", "Type", "Size", "Hash", "State", "VerificationCount", "DeleteGraceTime", "ArchiveTime", "LockExpirationTime") VALUES
|
||||
(301, 1, 'duplicati-demo.dblock.zip.aes', 'Blocks', NULL, NULL, 'Verified', 0, 0, 0, 0),
|
||||
(302, 1, 'duplicati-text.dblock.zip.aes', 'Blocks', NULL, NULL, 'Verified', 0, 0, 0, 0);
|
||||
${includeArchiveEntryIndex ? `INSERT INTO "archive_entry_index"
|
||||
("volume_name", "entry_name", "local_header_offset_plain", "data_offset_plain", "compressed_size", "uncompressed_size", "compression_method", "crc32")
|
||||
VALUES
|
||||
('duplicati-demo.dblock.zip.aes', '0td8NEaS7SMrQc5Gs0Sdxjb_1MXEEuwkyxRpguDiWsY=', 32768, 32852, 86124, 102400, '8', '6b4f0d2a'),
|
||||
('duplicati-demo.dblock.zip.aes', 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=', 119104, 119188, 85571, 102400, '8', '0fbd3c11'),
|
||||
('duplicati-demo.dblock.zip.aes', 'uS_2KMSmm2IWlZ77JiHH1p_yp7Cvhr8CKmRHJNMRqwA=', 205824, 205908, 9941, 10240, '8', '2a739d5b'),
|
||||
('duplicati-text.dblock.zip.aes', 'ySjwbaRrk-rm6Vx0W0xP8A==', 4096, 4176, 7, 5, '0', '00abc123');` : ''}
|
||||
${includeVolumeCryptoCache ? `INSERT INTO "volume_crypto_cache"
|
||||
("volume_name", "stream_format", "header_probe_bytes", "kdf_iterations", "salt_hex", "iv_hex")
|
||||
VALUES
|
||||
('duplicati-demo.dblock.zip.aes', 'v2', 103, 8192, NULL, '0123456789abcdef'),
|
||||
('duplicati-text.dblock.zip.aes', 'v2', 103, 8192, NULL, 'fedcba9876543210');` : ''}
|
||||
`);
|
||||
|
||||
database.close();
|
||||
|
||||
let remoteDir = null;
|
||||
let remoteFilesByName = {};
|
||||
if (withRemoteVolumes) {
|
||||
remoteDir = path.join(tempDirectory, 'remote');
|
||||
fs.mkdirSync(remoteDir, { recursive: true });
|
||||
|
||||
const plainZips = buildRemoteVolumeBuffers();
|
||||
remoteFilesByName = Object.fromEntries(
|
||||
Object.entries(plainZips).map(([volumeName, plainZipBuffer]) => {
|
||||
const encryptedBuffer = createAesCryptV2Buffer(plainZipBuffer, passphrase, {
|
||||
extensionRecords: includeAesExtensions
|
||||
? [
|
||||
{
|
||||
id: Buffer.from([0x10, 0x20]),
|
||||
data: Buffer.from('duplicati-test-extension', 'utf8')
|
||||
}
|
||||
]
|
||||
: []
|
||||
});
|
||||
const filename = path.join(remoteDir, volumeName);
|
||||
fs.writeFileSync(filename, encryptedBuffer);
|
||||
return [volumeName, filename];
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
databasePath,
|
||||
tempDirectory,
|
||||
remoteDir,
|
||||
remoteFilesByName,
|
||||
passphrase,
|
||||
cleanup() {
|
||||
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function createDuplicatiServerDbFixture(options = {}) {
|
||||
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-server-db-'));
|
||||
const databasePath = path.join(tempDirectory, options.filename ?? 'Duplicati-server.sqlite');
|
||||
const database = new DatabaseSync(databasePath);
|
||||
const backups = options.backups ?? [
|
||||
{
|
||||
id: 1,
|
||||
name: 'Example Backup',
|
||||
dbPath: '/data/Duplicati/EXAMPLE.sqlite'
|
||||
}
|
||||
];
|
||||
|
||||
database.exec(`
|
||||
CREATE TABLE "Backup" (
|
||||
"ID" INTEGER PRIMARY KEY,
|
||||
"Name" TEXT NOT NULL,
|
||||
"DBPath" TEXT NOT NULL,
|
||||
"TargetURL" TEXT NULL
|
||||
);
|
||||
`);
|
||||
|
||||
for (const backup of backups) {
|
||||
database
|
||||
.prepare('INSERT INTO "Backup" ("ID", "Name", "DBPath", "TargetURL") VALUES (?, ?, ?, ?)')
|
||||
.run(backup.id, backup.name, backup.dbPath, backup.targetUrl ?? null);
|
||||
}
|
||||
|
||||
database.close();
|
||||
|
||||
return {
|
||||
databasePath,
|
||||
tempDirectory,
|
||||
backups,
|
||||
cleanup() {
|
||||
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
analyzeFrameLuma,
|
||||
buildThumbnailCandidateTimes
|
||||
} from '../public/modules/thumbnail-utils.js';
|
||||
|
||||
test('buildThumbnailCandidateTimes prefers early seconds and appends a bounded fallback', () => {
|
||||
assert.deepEqual(buildThumbnailCandidateTimes(0), [0.15]);
|
||||
assert.deepEqual(buildThumbnailCandidateTimes(1.2), [0.15, 0.5, 1, 1.15]);
|
||||
assert.deepEqual(buildThumbnailCandidateTimes(6), [0.15, 0.5, 1, 2, 3.5, 5, 5.95]);
|
||||
});
|
||||
|
||||
test('analyzeFrameLuma marks near-black frames as black and bright frames as usable', () => {
|
||||
const blackFrame = new Uint8ClampedArray(64 * 4).fill(0);
|
||||
for (let index = 3; index < blackFrame.length; index += 4) {
|
||||
blackFrame[index] = 255;
|
||||
}
|
||||
|
||||
const brightFrame = new Uint8ClampedArray(64 * 4);
|
||||
for (let index = 0; index < brightFrame.length; index += 4) {
|
||||
brightFrame[index] = 200;
|
||||
brightFrame[index + 1] = 180;
|
||||
brightFrame[index + 2] = 120;
|
||||
brightFrame[index + 3] = 255;
|
||||
}
|
||||
|
||||
const blackAnalysis = analyzeFrameLuma(blackFrame, {
|
||||
stride: 1
|
||||
});
|
||||
const brightAnalysis = analyzeFrameLuma(brightFrame, {
|
||||
stride: 1
|
||||
});
|
||||
|
||||
assert.equal(blackAnalysis.isBlackFrame, true);
|
||||
assert.equal(brightAnalysis.isBlackFrame, false);
|
||||
assert.ok(blackAnalysis.darkRatio > brightAnalysis.darkRatio);
|
||||
assert.ok(brightAnalysis.averageLuma > blackAnalysis.averageLuma);
|
||||
});
|
||||
Reference in New Issue
Block a user