430 lines
14 KiB
JavaScript
430 lines
14 KiB
JavaScript
import { createCipheriv, createHash, createHmac, randomBytes } from 'node:crypto';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { deflateRawSync } from 'node:zlib';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
|
|
function createCrc32Table() {
|
|
const table = new Uint32Array(256);
|
|
for (let index = 0; index < 256; index += 1) {
|
|
let value = index;
|
|
for (let bit = 0; bit < 8; bit += 1) {
|
|
value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
|
}
|
|
table[index] = value >>> 0;
|
|
}
|
|
|
|
return table;
|
|
}
|
|
|
|
const CRC32_TABLE = createCrc32Table();
|
|
|
|
function crc32(buffer) {
|
|
let value = 0xffffffff;
|
|
for (const byte of buffer) {
|
|
value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
|
|
}
|
|
|
|
return (value ^ 0xffffffff) >>> 0;
|
|
}
|
|
|
|
function stretchPasswordV2(passphrase, externalIv) {
|
|
const passwordBytes = Buffer.from(String(passphrase), 'utf16le');
|
|
let digest = Buffer.concat([externalIv, Buffer.alloc(16, 0)]);
|
|
|
|
for (let index = 0; index < 8192; index += 1) {
|
|
const hash = createHash('sha256');
|
|
hash.update(digest);
|
|
hash.update(passwordBytes);
|
|
digest = hash.digest();
|
|
}
|
|
|
|
return digest;
|
|
}
|
|
|
|
function createAesCryptV2Buffer(plainBuffer, passphrase, options = {}) {
|
|
const externalIv = randomBytes(16);
|
|
const internalIv = randomBytes(16);
|
|
const internalKey = randomBytes(32);
|
|
const stretchedKey = stretchPasswordV2(passphrase, externalIv);
|
|
const sessionBytes = Buffer.concat([internalIv, internalKey]);
|
|
|
|
const headerCipher = createCipheriv('aes-256-cbc', stretchedKey, externalIv);
|
|
headerCipher.setAutoPadding(false);
|
|
const encryptedSession = Buffer.concat([
|
|
headerCipher.update(sessionBytes),
|
|
headerCipher.final()
|
|
]);
|
|
|
|
const headerHmac = createHmac('sha256', stretchedKey).update(encryptedSession).digest();
|
|
|
|
const fileSizeModulo = plainBuffer.length % 16;
|
|
const padLength = fileSizeModulo === 0 ? 0 : 16 - fileSizeModulo;
|
|
const paddedPlain = padLength > 0
|
|
? Buffer.concat([plainBuffer, Buffer.alloc(padLength, padLength)])
|
|
: plainBuffer;
|
|
|
|
const payloadCipher = createCipheriv('aes-256-cbc', internalKey, internalIv);
|
|
payloadCipher.setAutoPadding(false);
|
|
const ciphertext = Buffer.concat([
|
|
payloadCipher.update(paddedPlain),
|
|
payloadCipher.final()
|
|
]);
|
|
const payloadHmac = createHmac('sha256', internalKey).update(ciphertext).digest();
|
|
|
|
const extensionRecords = options.extensionRecords ?? [];
|
|
const extensionBytes = extensionRecords.flatMap((record) => {
|
|
const id = Buffer.from(record.id);
|
|
const data = Buffer.from(record.data ?? []);
|
|
const payload = Buffer.concat([id, data]);
|
|
const length = Buffer.alloc(2);
|
|
length.writeUInt16BE(payload.length, 0);
|
|
return [length, payload];
|
|
});
|
|
|
|
return Buffer.concat([
|
|
Buffer.from('AES', 'utf8'),
|
|
Buffer.from([2, 0]),
|
|
...extensionBytes,
|
|
Buffer.from([0, 0]),
|
|
externalIv,
|
|
encryptedSession,
|
|
headerHmac,
|
|
ciphertext,
|
|
Buffer.from([fileSizeModulo]),
|
|
payloadHmac
|
|
]);
|
|
}
|
|
|
|
function createZipBuffer(entries) {
|
|
const localParts = [];
|
|
const centralParts = [];
|
|
let localOffset = 0;
|
|
|
|
for (const entry of entries) {
|
|
const nameBuffer = Buffer.from(entry.name, 'utf8');
|
|
const input = Buffer.from(entry.data);
|
|
const compressedData =
|
|
entry.compression === 'store' ? input : deflateRawSync(input);
|
|
const compressionMethod = entry.compression === 'store' ? 0 : 8;
|
|
const crc = crc32(input);
|
|
|
|
const localHeader = Buffer.alloc(30);
|
|
localHeader.writeUInt32LE(0x04034b50, 0);
|
|
localHeader.writeUInt16LE(20, 4);
|
|
localHeader.writeUInt16LE(0, 6);
|
|
localHeader.writeUInt16LE(compressionMethod, 8);
|
|
localHeader.writeUInt16LE(0, 10);
|
|
localHeader.writeUInt16LE(0, 12);
|
|
localHeader.writeUInt32LE(crc, 14);
|
|
localHeader.writeUInt32LE(compressedData.length, 18);
|
|
localHeader.writeUInt32LE(input.length, 22);
|
|
localHeader.writeUInt16LE(nameBuffer.length, 26);
|
|
localHeader.writeUInt16LE(0, 28);
|
|
|
|
localParts.push(localHeader, nameBuffer, compressedData);
|
|
|
|
const centralHeader = Buffer.alloc(46);
|
|
centralHeader.writeUInt32LE(0x02014b50, 0);
|
|
centralHeader.writeUInt16LE(20, 4);
|
|
centralHeader.writeUInt16LE(20, 6);
|
|
centralHeader.writeUInt16LE(0, 8);
|
|
centralHeader.writeUInt16LE(compressionMethod, 10);
|
|
centralHeader.writeUInt16LE(0, 12);
|
|
centralHeader.writeUInt16LE(0, 14);
|
|
centralHeader.writeUInt32LE(crc, 16);
|
|
centralHeader.writeUInt32LE(compressedData.length, 20);
|
|
centralHeader.writeUInt32LE(input.length, 24);
|
|
centralHeader.writeUInt16LE(nameBuffer.length, 28);
|
|
centralHeader.writeUInt16LE(0, 30);
|
|
centralHeader.writeUInt16LE(0, 32);
|
|
centralHeader.writeUInt16LE(0, 34);
|
|
centralHeader.writeUInt16LE(0, 36);
|
|
centralHeader.writeUInt32LE(0, 38);
|
|
centralHeader.writeUInt32LE(localOffset, 42);
|
|
|
|
centralParts.push(centralHeader, nameBuffer);
|
|
localOffset += localHeader.length + nameBuffer.length + compressedData.length;
|
|
}
|
|
|
|
const centralDirectory = Buffer.concat(centralParts);
|
|
const localData = Buffer.concat(localParts);
|
|
const eocd = Buffer.alloc(22);
|
|
eocd.writeUInt32LE(0x06054b50, 0);
|
|
eocd.writeUInt16LE(0, 4);
|
|
eocd.writeUInt16LE(0, 6);
|
|
eocd.writeUInt16LE(entries.length, 8);
|
|
eocd.writeUInt16LE(entries.length, 10);
|
|
eocd.writeUInt32LE(centralDirectory.length, 12);
|
|
eocd.writeUInt32LE(localData.length, 16);
|
|
eocd.writeUInt16LE(0, 20);
|
|
|
|
return Buffer.concat([localData, centralDirectory, eocd]);
|
|
}
|
|
|
|
function buildRemoteVolumeBuffers() {
|
|
const demoEntries = [
|
|
{
|
|
name: '0td8NEaS7SMrQc5Gs0Sdxjb_1MXEEuwkyxRpguDiWsY=',
|
|
data: Buffer.alloc(102400, 0x11),
|
|
compression: 'deflate'
|
|
},
|
|
{
|
|
name: 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=',
|
|
data: Buffer.alloc(102400, 0x22),
|
|
compression: 'deflate'
|
|
},
|
|
{
|
|
name: 'uS_2KMSmm2IWlZ77JiHH1p_yp7Cvhr8CKmRHJNMRqwA=',
|
|
data: Buffer.alloc(10240, 0x33),
|
|
compression: 'deflate'
|
|
}
|
|
];
|
|
const textEntries = [
|
|
{
|
|
name: 'ySjwbaRrk-rm6Vx0W0xP8A==',
|
|
data: Buffer.from('hello', 'utf8'),
|
|
compression: 'store'
|
|
}
|
|
];
|
|
|
|
return {
|
|
'duplicati-demo.dblock.zip.aes': createZipBuffer(demoEntries),
|
|
'duplicati-text.dblock.zip.aes': createZipBuffer(textEntries)
|
|
};
|
|
}
|
|
|
|
export function createDuplicatiFixture(options = {}) {
|
|
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-api-'));
|
|
const databasePath = path.join(tempDirectory, options.filename ?? 'fixture.sqlite');
|
|
const database = new DatabaseSync(databasePath);
|
|
const includeArchiveEntryIndex = options.includeArchiveEntryIndex !== false;
|
|
const includeVolumeCryptoCache = options.includeVolumeCryptoCache !== false;
|
|
const layout = options.layout ?? 'file';
|
|
const passphrase = options.passphrase ?? 'test-passphrase';
|
|
const withRemoteVolumes = options.withRemoteVolumes === true;
|
|
const includeAesExtensions = options.includeAesExtensions === true;
|
|
|
|
database.exec(`
|
|
CREATE TABLE "Fileset" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Timestamp" INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE "Configuration" (
|
|
"Key" TEXT PRIMARY KEY,
|
|
"Value" TEXT NOT NULL
|
|
);
|
|
${layout === 'file'
|
|
? `CREATE TABLE "File" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Path" TEXT NOT NULL,
|
|
"BlocksetID" INTEGER NOT NULL,
|
|
"MetadataID" INTEGER NULL
|
|
);`
|
|
: `CREATE TABLE "FileLookup" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"PrefixID" INTEGER NOT NULL,
|
|
"Path" TEXT NOT NULL,
|
|
"BlocksetID" INTEGER NOT NULL,
|
|
"MetadataID" INTEGER NULL
|
|
);
|
|
CREATE TABLE "PathPrefix" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Prefix" TEXT NOT NULL
|
|
);`}
|
|
CREATE TABLE "FilesetEntry" (
|
|
"FilesetID" INTEGER NOT NULL,
|
|
"FileID" INTEGER NOT NULL,
|
|
"Lastmodified" INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE "Blockset" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Length" INTEGER NOT NULL,
|
|
"FullHash" TEXT NULL
|
|
);
|
|
CREATE TABLE "Block" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Hash" TEXT NOT NULL,
|
|
"Size" INTEGER NOT NULL,
|
|
"VolumeID" INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE "BlocksetEntry" (
|
|
"BlocksetID" INTEGER NOT NULL,
|
|
"Index" INTEGER NOT NULL,
|
|
"BlockID" INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE "Remotevolume" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"OperationID" INTEGER NULL,
|
|
"Name" TEXT NOT NULL,
|
|
"Type" TEXT NULL,
|
|
"Size" INTEGER NULL,
|
|
"Hash" TEXT NULL,
|
|
"State" TEXT NULL,
|
|
"VerificationCount" INTEGER NULL,
|
|
"DeleteGraceTime" INTEGER NULL,
|
|
"ArchiveTime" INTEGER NULL,
|
|
"LockExpirationTime" INTEGER NULL
|
|
);
|
|
${includeArchiveEntryIndex ? `CREATE TABLE "archive_entry_index" (
|
|
"volume_name" TEXT NOT NULL,
|
|
"entry_name" TEXT NOT NULL,
|
|
"local_header_offset_plain" INTEGER NOT NULL,
|
|
"data_offset_plain" INTEGER NOT NULL,
|
|
"compressed_size" INTEGER NOT NULL,
|
|
"uncompressed_size" INTEGER NOT NULL,
|
|
"compression_method" TEXT NOT NULL,
|
|
"crc32" TEXT NOT NULL,
|
|
PRIMARY KEY ("volume_name", "entry_name")
|
|
);` : ''}
|
|
${includeVolumeCryptoCache ? `CREATE TABLE "volume_crypto_cache" (
|
|
"volume_name" TEXT PRIMARY KEY,
|
|
"stream_format" TEXT NULL,
|
|
"header_probe_bytes" INTEGER NULL,
|
|
"kdf_iterations" INTEGER NULL,
|
|
"salt_hex" TEXT NULL,
|
|
"iv_hex" TEXT NULL
|
|
);` : ''}
|
|
`);
|
|
|
|
const fileName = options.demoFileName ?? 'demo.mp4';
|
|
const fileHash = options.demoFileHash ?? 'filehash-demo==';
|
|
const fileEntriesSql =
|
|
layout === 'file'
|
|
? `
|
|
INSERT INTO "File" ("ID", "Path", "BlocksetID", "MetadataID") VALUES
|
|
(10, 'C:\\\\media\\\\movies', -1, NULL),
|
|
(11, 'C:\\\\media\\\\movies\\\\${fileName}', 101, NULL),
|
|
(12, 'C:\\\\media\\\\readme.txt', 102, NULL);
|
|
`
|
|
: `
|
|
INSERT INTO "PathPrefix" ("ID", "Prefix") VALUES
|
|
(1, '/source/media/'),
|
|
(2, '/source/media/movies/');
|
|
INSERT INTO "FileLookup" ("ID", "PrefixID", "Path", "BlocksetID", "MetadataID") VALUES
|
|
(10, 1, 'movies/', -100, NULL),
|
|
(11, 2, '${fileName}', 101, NULL),
|
|
(12, 1, 'readme.txt', 102, NULL);
|
|
`;
|
|
|
|
database.exec(`
|
|
INSERT INTO "Fileset" ("ID", "Timestamp") VALUES (1, 1714896000);
|
|
INSERT INTO "Configuration" ("Key", "Value") VALUES
|
|
('blocksize', '102400'),
|
|
('blockhash', 'SHA256'),
|
|
('filehash', 'SHA256');
|
|
${fileEntriesSql}
|
|
INSERT INTO "FilesetEntry" ("FilesetID", "FileID", "Lastmodified") VALUES
|
|
(1, 10, 638504448000000000),
|
|
(1, 11, 638504448110000000),
|
|
(1, 12, 638504448220000000);
|
|
INSERT INTO "Blockset" ("ID", "Length", "FullHash") VALUES
|
|
(101, 215040, '${fileHash}'),
|
|
(102, 5, 'filehash-readme==');
|
|
INSERT INTO "Block" ("ID", "Hash", "Size", "VolumeID") VALUES
|
|
(201, '0td8NEaS7SMrQc5Gs0Sdxjb/1MXEEuwkyxRpguDiWsY=', 102400, 301),
|
|
(202, 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=', 102400, 301),
|
|
(203, 'uS/2KMSmm2IWlZ77JiHH1p/yp7Cvhr8CKmRHJNMRqwA=', 10240, 301),
|
|
(204, 'ySjwbaRrk+rm6Vx0W0xP8A==', 5, 302);
|
|
INSERT INTO "BlocksetEntry" ("BlocksetID", "Index", "BlockID") VALUES
|
|
(101, 0, 201),
|
|
(101, 1, 202),
|
|
(101, 2, 203),
|
|
(102, 0, 204);
|
|
INSERT INTO "Remotevolume" ("ID", "OperationID", "Name", "Type", "Size", "Hash", "State", "VerificationCount", "DeleteGraceTime", "ArchiveTime", "LockExpirationTime") VALUES
|
|
(301, 1, 'duplicati-demo.dblock.zip.aes', 'Blocks', NULL, NULL, 'Verified', 0, 0, 0, 0),
|
|
(302, 1, 'duplicati-text.dblock.zip.aes', 'Blocks', NULL, NULL, 'Verified', 0, 0, 0, 0);
|
|
${includeArchiveEntryIndex ? `INSERT INTO "archive_entry_index"
|
|
("volume_name", "entry_name", "local_header_offset_plain", "data_offset_plain", "compressed_size", "uncompressed_size", "compression_method", "crc32")
|
|
VALUES
|
|
('duplicati-demo.dblock.zip.aes', '0td8NEaS7SMrQc5Gs0Sdxjb_1MXEEuwkyxRpguDiWsY=', 32768, 32852, 86124, 102400, '8', '6b4f0d2a'),
|
|
('duplicati-demo.dblock.zip.aes', 'PN2oO6eQudCRSdx3zgk6SJvlI5BquP6djt5hG4ZfRCQ=', 119104, 119188, 85571, 102400, '8', '0fbd3c11'),
|
|
('duplicati-demo.dblock.zip.aes', 'uS_2KMSmm2IWlZ77JiHH1p_yp7Cvhr8CKmRHJNMRqwA=', 205824, 205908, 9941, 10240, '8', '2a739d5b'),
|
|
('duplicati-text.dblock.zip.aes', 'ySjwbaRrk-rm6Vx0W0xP8A==', 4096, 4176, 7, 5, '0', '00abc123');` : ''}
|
|
${includeVolumeCryptoCache ? `INSERT INTO "volume_crypto_cache"
|
|
("volume_name", "stream_format", "header_probe_bytes", "kdf_iterations", "salt_hex", "iv_hex")
|
|
VALUES
|
|
('duplicati-demo.dblock.zip.aes', 'v2', 103, 8192, NULL, '0123456789abcdef'),
|
|
('duplicati-text.dblock.zip.aes', 'v2', 103, 8192, NULL, 'fedcba9876543210');` : ''}
|
|
`);
|
|
|
|
database.close();
|
|
|
|
let remoteDir = null;
|
|
let remoteFilesByName = {};
|
|
if (withRemoteVolumes) {
|
|
remoteDir = path.join(tempDirectory, 'remote');
|
|
fs.mkdirSync(remoteDir, { recursive: true });
|
|
|
|
const plainZips = buildRemoteVolumeBuffers();
|
|
remoteFilesByName = Object.fromEntries(
|
|
Object.entries(plainZips).map(([volumeName, plainZipBuffer]) => {
|
|
const encryptedBuffer = createAesCryptV2Buffer(plainZipBuffer, passphrase, {
|
|
extensionRecords: includeAesExtensions
|
|
? [
|
|
{
|
|
id: Buffer.from([0x10, 0x20]),
|
|
data: Buffer.from('duplicati-test-extension', 'utf8')
|
|
}
|
|
]
|
|
: []
|
|
});
|
|
const filename = path.join(remoteDir, volumeName);
|
|
fs.writeFileSync(filename, encryptedBuffer);
|
|
return [volumeName, filename];
|
|
})
|
|
);
|
|
}
|
|
|
|
return {
|
|
databasePath,
|
|
tempDirectory,
|
|
remoteDir,
|
|
remoteFilesByName,
|
|
passphrase,
|
|
cleanup() {
|
|
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
|
}
|
|
};
|
|
}
|
|
|
|
export function createDuplicatiServerDbFixture(options = {}) {
|
|
const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicati-server-db-'));
|
|
const databasePath = path.join(tempDirectory, options.filename ?? 'Duplicati-server.sqlite');
|
|
const database = new DatabaseSync(databasePath);
|
|
const backups = options.backups ?? [
|
|
{
|
|
id: 1,
|
|
name: 'Example Backup',
|
|
dbPath: '/data/Duplicati/EXAMPLE.sqlite'
|
|
}
|
|
];
|
|
|
|
database.exec(`
|
|
CREATE TABLE "Backup" (
|
|
"ID" INTEGER PRIMARY KEY,
|
|
"Name" TEXT NOT NULL,
|
|
"DBPath" TEXT NOT NULL,
|
|
"TargetURL" TEXT NULL
|
|
);
|
|
`);
|
|
|
|
for (const backup of backups) {
|
|
database
|
|
.prepare('INSERT INTO "Backup" ("ID", "Name", "DBPath", "TargetURL") VALUES (?, ?, ?, ?)')
|
|
.run(backup.id, backup.name, backup.dbPath, backup.targetUrl ?? null);
|
|
}
|
|
|
|
database.close();
|
|
|
|
return {
|
|
databasePath,
|
|
tempDirectory,
|
|
backups,
|
|
cleanup() {
|
|
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
|
}
|
|
};
|
|
}
|