feat: add fnOS packaging, storage workflows and release pipeline
This commit is contained in:
@@ -0,0 +1,316 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using dy.net.model.dto;
|
||||
using dy.net.model.entity;
|
||||
using dy.net.service;
|
||||
using dy.net.storage;
|
||||
using dy.net.Tests.TestInfrastructure;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using SqlSugar;
|
||||
|
||||
namespace dy.net.Tests;
|
||||
|
||||
public sealed class OpenListTransferServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Transfer_UsesServerSideCopyAndReusesVerifiedTarget()
|
||||
{
|
||||
using var host = await OpenListTransferTestHost.CreateAsync();
|
||||
var bytes = Encoding.UTF8.GetBytes("OpenList 原生复制测试内容");
|
||||
|
||||
await using (var source = new MemoryStream(bytes, writable: false))
|
||||
Assert.Equal(bytes.Length, await host.Transfers.TransferAsync(
|
||||
"/关注/作者 名/视频 #1.mp4", source, bytes.Length, CancellationToken.None));
|
||||
|
||||
Assert.Equal(bytes, host.Handler.Files["/归档/关注/作者 名/视频 #1.mp4"]);
|
||||
Assert.Equal(1, host.Handler.CopyCount);
|
||||
var job = await host.Database.Queryable<OpenListTransferJob>().SingleAsync();
|
||||
Assert.Equal(OpenListTransferStatus.Succeeded, job.Status);
|
||||
Assert.False(File.Exists(job.LocalSourcePath));
|
||||
Assert.DoesNotContain(host.Handler.Files.Keys, x => x.Contains(".dysync-staging-", StringComparison.Ordinal));
|
||||
|
||||
await using (var duplicate = new MemoryStream(bytes, writable: false))
|
||||
Assert.Equal(bytes.Length, await host.Transfers.TransferAsync(
|
||||
"/关注/作者 名/视频 #1.mp4", duplicate, bytes.Length, CancellationToken.None));
|
||||
|
||||
Assert.Equal(1, host.Handler.CopyCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecoverOne_ResumesPersistedQueuedTransferAfterRestart()
|
||||
{
|
||||
using var host = await OpenListTransferTestHost.CreateAsync();
|
||||
var bytes = Encoding.UTF8.GetBytes("restart-safe-transfer");
|
||||
var id = "restartjob";
|
||||
var localDirectory = Path.Combine(host.Settings.LocalStagingPath, id);
|
||||
Directory.CreateDirectory(localDirectory);
|
||||
var localPath = Path.Combine(localDirectory, "恢复.mp4");
|
||||
await File.WriteAllBytesAsync(localPath, bytes);
|
||||
var now = DateTime.Now;
|
||||
await host.Database.Insertable(new OpenListTransferJob
|
||||
{
|
||||
Id = id,
|
||||
Status = OpenListTransferStatus.Queued,
|
||||
ConfigurationFingerprint = StorageConfigurationFingerprint.Create(host.Settings),
|
||||
LogicalTargetPath = "/恢复/恢复.mp4",
|
||||
ActualTargetPath = "/归档/恢复/恢复.mp4",
|
||||
LocalSourcePath = localPath,
|
||||
OpenListSourcePath = "/源挂载/restartjob/恢复.mp4",
|
||||
StagedTargetPath = "/归档/.dysync-staging-restartjob/恢复.mp4",
|
||||
ExpectedLength = bytes.Length,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
}).ExecuteCommandAsync();
|
||||
|
||||
Assert.True(await host.Transfers.RecoverOneAsync(CancellationToken.None));
|
||||
|
||||
var job = await host.Database.Queryable<OpenListTransferJob>().InSingleAsync(id);
|
||||
Assert.Equal(OpenListTransferStatus.Succeeded, job.Status);
|
||||
Assert.Equal(bytes, host.Handler.Files["/归档/恢复/恢复.mp4"]);
|
||||
Assert.False(File.Exists(localPath));
|
||||
}
|
||||
|
||||
private sealed class OpenListTransferTestHost : IDisposable
|
||||
{
|
||||
private readonly TemporaryDirectory _temporary;
|
||||
private readonly PerRequestFactory _factory;
|
||||
|
||||
private OpenListTransferTestHost(
|
||||
TemporaryDirectory temporary,
|
||||
SqlSugarClient database,
|
||||
PerRequestFactory factory,
|
||||
InMemoryOpenListHandler handler,
|
||||
OpenListSettings settings,
|
||||
OpenListTransferService transfers)
|
||||
{
|
||||
_temporary = temporary;
|
||||
Database = database;
|
||||
_factory = factory;
|
||||
Handler = handler;
|
||||
Settings = settings;
|
||||
Transfers = transfers;
|
||||
}
|
||||
|
||||
public SqlSugarClient Database { get; }
|
||||
public InMemoryOpenListHandler Handler { get; }
|
||||
public OpenListSettings Settings { get; }
|
||||
public OpenListTransferService Transfers { get; }
|
||||
|
||||
public static async Task<OpenListTransferTestHost> CreateAsync()
|
||||
{
|
||||
var temporary = new TemporaryDirectory();
|
||||
try
|
||||
{
|
||||
var database = new SqlSugarClient(new ConnectionConfig
|
||||
{
|
||||
ConnectionString = $"DataSource={Path.Combine(temporary.Path, "openlist.sqlite")}",
|
||||
DbType = DbType.Sqlite,
|
||||
InitKeyType = InitKeyType.Attribute,
|
||||
IsAutoCloseConnection = true
|
||||
});
|
||||
database.CodeFirst.InitTables(typeof(WebDavSettings), typeof(OpenListSettings), typeof(OpenListTransferJob));
|
||||
var localStaging = Path.Combine(temporary.Path, "staging");
|
||||
Directory.CreateDirectory(localStaging);
|
||||
var handler = new InMemoryOpenListHandler(localStaging, "/源挂载");
|
||||
var factory = new PerRequestFactory(handler);
|
||||
var protection = DataProtectionProvider.Create(
|
||||
new DirectoryInfo(Path.Combine(temporary.Path, "keys")),
|
||||
builder => builder.SetApplicationName("dysync.net"));
|
||||
var legacy = new WebDavSettingsService(database, protection);
|
||||
var settingsService = new OpenListSettingsService(database, protection, legacy);
|
||||
var settings = await settingsService.BuildCandidateAsync(new OpenListTestRequest
|
||||
{
|
||||
Endpoint = "https://openlist.example.test",
|
||||
BasePath = "/归档",
|
||||
LocalStagingPath = localStaging,
|
||||
SourcePath = "/源挂载",
|
||||
UserName = "account",
|
||||
Password = "password"
|
||||
});
|
||||
await settingsService.SaveAsync(settings, true, "tested");
|
||||
var client = new OpenListClient(factory);
|
||||
var transfers = new OpenListTransferService(database, settingsService, client);
|
||||
return new OpenListTransferTestHost(temporary, database, factory, handler, settings, transfers);
|
||||
}
|
||||
catch
|
||||
{
|
||||
temporary.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Database.Dispose();
|
||||
_factory.Dispose();
|
||||
_temporary.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PerRequestFactory : IHttpClientFactory, IDisposable
|
||||
{
|
||||
private readonly HttpMessageHandler _handler;
|
||||
|
||||
public PerRequestFactory(HttpMessageHandler handler) => _handler = handler;
|
||||
|
||||
public HttpClient CreateClient(string name) => new(_handler, disposeHandler: false);
|
||||
|
||||
public void Dispose() => _handler.Dispose();
|
||||
}
|
||||
|
||||
private sealed class InMemoryOpenListHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly string _localStaging;
|
||||
private readonly string _sourceRoot;
|
||||
private readonly HashSet<string> _directories = new(StringComparer.Ordinal) { "/" };
|
||||
|
||||
public InMemoryOpenListHandler(string localStaging, string sourceRoot)
|
||||
{
|
||||
_localStaging = Path.GetFullPath(localStaging);
|
||||
_sourceRoot = StoragePath.NormalizeRemote(sourceRoot);
|
||||
_directories.Add(_sourceRoot);
|
||||
}
|
||||
|
||||
public Dictionary<string, byte[]> Files { get; } = new(StringComparer.Ordinal);
|
||||
public int CopyCount { get; private set; }
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var api = request.RequestUri!.AbsolutePath;
|
||||
if (api.EndsWith("/api/auth/login", StringComparison.Ordinal))
|
||||
return Json("{\"code\":200,\"message\":\"success\",\"data\":{\"token\":\"test-token\"}}");
|
||||
|
||||
var body = request.Content == null
|
||||
? default
|
||||
: JsonDocument.Parse(await request.Content.ReadAsStringAsync(cancellationToken)).RootElement.Clone();
|
||||
if (api.EndsWith("/api/fs/get", StringComparison.Ordinal))
|
||||
return Get(body.GetProperty("path").GetString()!);
|
||||
if (api.EndsWith("/api/fs/list", StringComparison.Ordinal))
|
||||
return List(body.GetProperty("path").GetString()!);
|
||||
if (api.EndsWith("/api/fs/mkdir", StringComparison.Ordinal))
|
||||
{
|
||||
AddDirectory(body.GetProperty("path").GetString()!);
|
||||
return Success();
|
||||
}
|
||||
if (api.EndsWith("/api/fs/copy", StringComparison.Ordinal))
|
||||
{
|
||||
CopyCount++;
|
||||
var sourceDirectory = body.GetProperty("src_dir").GetString()!;
|
||||
var targetDirectory = body.GetProperty("dst_dir").GetString()!;
|
||||
var name = body.GetProperty("names")[0].GetString()!;
|
||||
var bytes = ReadFile(StoragePath.CombineRemote(sourceDirectory, name));
|
||||
Files[StoragePath.CombineRemote(targetDirectory, name)] = bytes;
|
||||
AddDirectory(targetDirectory);
|
||||
return Json("{\"code\":200,\"message\":\"success\",\"data\":{\"tasks\":[]}}");
|
||||
}
|
||||
if (api.EndsWith("/api/fs/move", StringComparison.Ordinal))
|
||||
{
|
||||
var sourceDirectory = body.GetProperty("src_dir").GetString()!;
|
||||
var targetDirectory = body.GetProperty("dst_dir").GetString()!;
|
||||
var name = body.GetProperty("names")[0].GetString()!;
|
||||
var source = StoragePath.CombineRemote(sourceDirectory, name);
|
||||
var target = StoragePath.CombineRemote(targetDirectory, name);
|
||||
Files[target] = Files[source];
|
||||
Files.Remove(source);
|
||||
AddDirectory(targetDirectory);
|
||||
return Success();
|
||||
}
|
||||
if (api.EndsWith("/api/fs/rename", StringComparison.Ordinal))
|
||||
{
|
||||
var source = StoragePath.NormalizeRemote(body.GetProperty("path").GetString());
|
||||
var target = StoragePath.CombineRemote(StoragePath.DirectoryName(source), body.GetProperty("name").GetString());
|
||||
Files[target] = Files[source];
|
||||
Files.Remove(source);
|
||||
return Success();
|
||||
}
|
||||
if (api.EndsWith("/api/fs/remove", StringComparison.Ordinal))
|
||||
{
|
||||
var path = StoragePath.CombineRemote(body.GetProperty("dir").GetString(), body.GetProperty("names")[0].GetString());
|
||||
Files.Remove(path);
|
||||
foreach (var child in Files.Keys.Where(x => x.StartsWith(path + "/", StringComparison.Ordinal)).ToList())
|
||||
Files.Remove(child);
|
||||
_directories.Remove(path);
|
||||
return Success();
|
||||
}
|
||||
throw new InvalidOperationException($"Unexpected OpenList API request: {api}");
|
||||
}
|
||||
|
||||
private HttpResponseMessage Get(string rawPath)
|
||||
{
|
||||
var path = StoragePath.NormalizeRemote(rawPath);
|
||||
if (Files.TryGetValue(path, out var bytes)) return Object(path, bytes.Length, false);
|
||||
if (TryMapSource(path, out var local) && File.Exists(local)) return Object(path, new FileInfo(local).Length, false);
|
||||
if (_directories.Contains(path)) return Object(path, 0, true);
|
||||
return Json("{\"code\":500,\"message\":\"object not found\",\"data\":null}");
|
||||
}
|
||||
|
||||
private HttpResponseMessage List(string rawPath)
|
||||
{
|
||||
var path = StoragePath.NormalizeRemote(rawPath);
|
||||
var content = _directories
|
||||
.Where(x => x != path && StoragePath.DirectoryName(x) == path)
|
||||
.Select(x => new { name = Path.GetFileName(x), size = 0L, is_dir = true })
|
||||
.Concat(Files.Where(x => StoragePath.DirectoryName(x.Key) == path)
|
||||
.Select(x => new { name = Path.GetFileName(x.Key), size = (long)x.Value.Length, is_dir = false }))
|
||||
.ToArray();
|
||||
return Json(JsonSerializer.Serialize(new
|
||||
{
|
||||
code = 200,
|
||||
message = "success",
|
||||
data = new { content, write = true }
|
||||
}));
|
||||
}
|
||||
|
||||
private byte[] ReadFile(string path)
|
||||
{
|
||||
if (Files.TryGetValue(path, out var bytes)) return bytes;
|
||||
if (TryMapSource(path, out var local) && File.Exists(local)) return File.ReadAllBytes(local);
|
||||
throw new FileNotFoundException("OpenList source missing", path);
|
||||
}
|
||||
|
||||
private bool TryMapSource(string path, out string local)
|
||||
{
|
||||
if (path.Equals(_sourceRoot, StringComparison.Ordinal))
|
||||
{
|
||||
local = _localStaging;
|
||||
return true;
|
||||
}
|
||||
if (!path.StartsWith(_sourceRoot + "/", StringComparison.Ordinal))
|
||||
{
|
||||
local = string.Empty;
|
||||
return false;
|
||||
}
|
||||
var segments = path[(_sourceRoot.Length + 1)..].Split('/');
|
||||
local = segments.Aggregate(_localStaging, Path.Combine);
|
||||
local = Path.GetFullPath(local);
|
||||
return local.StartsWith(_localStaging + Path.DirectorySeparatorChar, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private void AddDirectory(string rawPath)
|
||||
{
|
||||
var path = StoragePath.NormalizeRemote(rawPath);
|
||||
var current = string.Empty;
|
||||
foreach (var segment in path.Split('/', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
current = StoragePath.CombineRemote(current, segment);
|
||||
_directories.Add(current);
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpResponseMessage Object(string path, long size, bool directory) => Json(
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
code = 200,
|
||||
message = "success",
|
||||
data = new { name = Path.GetFileName(path), size, is_dir = directory, hash_info = new { } }
|
||||
}));
|
||||
|
||||
private static HttpResponseMessage Success() => Json("{\"code\":200,\"message\":\"success\",\"data\":null}");
|
||||
|
||||
private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(body, Encoding.UTF8, "application/json")
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user