Downloader rework

.github/workflows/clean_environment_tests.yaml

@@ -13,8 +13,8 @@ on:
     branches: [ main ]
     paths:
       - ".github/workflows/clean_environment_tests.yaml"
-      - "./src/**"
-      - "./tests/**"
+      - "src/**"
+      - "tests/**"
       - "**.props"
       - "**.targets"
     types: [ opened, synchronize, reopened, ready_for_review ]

src/Networking/NexusMods.Networking.HttpDownloader/DTOs/ChunkState.cs

@@ -0,0 +1,101 @@
+using System.Text.Json.Serialization;
+using JetBrains.Annotations;
+using NexusMods.Paths;
+
+namespace NexusMods.Networking.HttpDownloader.DTOs;
+
+
+/// <summary>
+/// Individual state of a chunk of a download
+/// </summary>
+public class ChunkState
+{
+    /// <summary>
+    /// Create a new chunk state with the given size and offset
+    /// </summary>
+    /// <param name="start"></param>
+    /// <param name="size"></param>
+    /// <param name="initChunk"></param>
+    /// <returns></returns>
+    public static ChunkState Create(Size start, Size size)
+    {
+        return new ChunkState
+        {
+            Completed = Size.Zero,
+            Read = Size.Zero,
+            Size = size,
+            Offset = start,
+        };
+    }
+
+    /// <summary>
+    /// The offset of the chunk within the output file
+    /// </summary>
+    public Size Offset { get; init; } = Size.Zero;
+
+    /// <summary>
+    /// The size of the chunk
+    /// </summary>
+    public Size Size { get; set; }
+
+    /// <summary>
+    /// The number of bytes read from the network into the chunk
+    /// </summary>
+    [JsonIgnore]
+    public Size Read { get; set; } = Size.Zero;
+
+    /// <summary>
+    /// The number of bytes written to the output file
+    /// </summary>
+    public Size Completed { get; set; } = Size.Zero;
+
+    /// <summary>
+    /// The source of the chunk, (the download information)
+    /// </summary>
+    [JsonIgnore]
+    public Source? Source { get; set; }
+
+    [PublicAPI]
+    public string SourceUrl => Source?.Request?.RequestUri?.AbsoluteUri ?? "No URL";
+
+    /// <summary>
+    /// Token used to cancel the download of the chunk
+    /// </summary>
+    [JsonIgnore]
+    public CancellationTokenSource? Cancel { get; set; }
+
+    /// <summary>
+    /// The time the download of the chunk started
+    /// </summary>
+    [JsonIgnore]
+    public DateTime Started { get; set; }
+
+    /// <summary>
+    /// The number of bytes per second being read from the network
+    /// </summary>
+    public Bandwidth BytesPerSecond => Read / (DateTime.Now - Started);
+
+    /// <summary>
+    /// True if the chunk has been read completely
+    /// </summary>
+    [JsonIgnore]
+    public bool IsReadComplete => Read == Size;
+
+    /// <summary>
+    /// True if the chunk has been written completely
+    /// </summary>
+    [JsonIgnore]
+    public bool IsWriteComplete => Completed == Size;
+
+    /// <summary>
+    /// True if the chunk has been read and written completely
+    /// </summary>
+    [JsonIgnore]
+    public bool IsComplete => IsReadComplete && IsWriteComplete;
+
+    /// <summary>
+    /// Returns the number of bytes remaining to be read
+    /// </summary>
+    [JsonIgnore]
+    public Size RemainingToRead => Size - Read;
+}

src/Networking/NexusMods.Networking.HttpDownloader/DTOs/DownloadState.cs

@@ -0,0 +1,84 @@
+using System.Text.Json.Serialization;
+using NexusMods.Paths;
+
+namespace NexusMods.Networking.HttpDownloader.DTOs;
+
+/// <summary>
+/// Overall state of a download
+/// </summary>
+class DownloadState
+{
+
+    /// <summary>
+    /// This is incremented by 1 every time the state is loaded and the download is resumed
+    /// </summary>
+    public int ResumeCount { get; set; }
+
+    /// <summary>
+    /// Total size of the download
+    /// </summary>
+    public Size TotalSize { get; set; }
+
+    /// <summary>
+    /// A list of the chunks that make up the download
+    /// </summary>
+    public List<ChunkState> Chunks { get; set; } = new();
+
+    /// <summary>
+    /// The destination of the download on the local file system
+    /// </summary>
+    [JsonIgnore]
+    public AbsolutePath Destination;
+
+    /// <summary>
+    /// The sources of the download (the download information, aka URL mirrors)
+    /// </summary>
+    [JsonIgnore]
+    public Source[]? Sources;
+
+    /// <summary>
+    /// Returns true if any chunk is incomplete
+    /// </summary>
+    [JsonIgnore]
+    public bool HasIncompleteChunk
+    {
+        get {
+            return Chunks.Any(chunk => !chunk.IsComplete);
+        }
+    }
+
+
+    /// <summary>
+    /// The file where the download state is stored
+    /// </summary>
+    [JsonIgnore]
+    public AbsolutePath StateFilePath => GetStateFilePath(Destination);
+
+    /// <summary>
+    /// The file where the download is stored while it is in progress
+    /// </summary>
+    [JsonIgnore]
+    public AbsolutePath TempFilePath => GetTempFilePath(Destination);
+
+    /// <summary>
+    /// Based on the destination, get the path to the state file
+    /// </summary>
+    /// <param name="destination"></param>
+    /// <returns></returns>
+    public static AbsolutePath GetStateFilePath(AbsolutePath destination) => destination.ReplaceExtension(new Extension(".progress"));
+
+    /// <summary>
+    /// Based on the destination, get the path to the temp file
+    /// </summary>
+    /// <param name="destination"></param>
+    /// <returns></returns>
+    public static AbsolutePath GetTempFilePath(AbsolutePath destination) => destination.ReplaceExtension(new Extension(".downloading"));
+
+
+    /// <summary>
+    /// Gets chunks that are not completely downloaded and written to disk
+    /// </summary>
+    /// <returns></returns>
+    [JsonIgnore]
+    public IEnumerable<ChunkState> UnfinishedChunks => Chunks.Where(chunk => !chunk.IsComplete);
+}

src/Networking/NexusMods.Networking.HttpDownloader/DTOs/ServerFeatures.cs

@@ -0,0 +1,35 @@
+using NexusMods.Paths;
+
+namespace NexusMods.Networking.HttpDownloader.DTOs;
+
+/// <summary>
+/// Defines the features of a download server. 
+/// </summary>
+class ServerFeatures
+{
+    /// <summary>
+    /// Not technically a feature, servers report the size of the download as part of this request
+    /// </summary>
+    public Size? DownloadSize { get; init; }
+
+    public bool SupportsResume { get; init; }
+
+    public static async Task<ServerFeatures> Request(HttpClient client, HttpRequestMessage message, CancellationToken token)
+    {
+        message.Method = HttpMethod.Head;
+        using var response = await client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token);
+        if (!response.IsSuccessStatusCode)
+        {
+            throw new HttpRequestException($"Server responded with {response.StatusCode}");
+        }
+
+        var features = new ServerFeatures
+        {
+            SupportsResume = response.Headers.AcceptRanges.Contains("bytes"),
+            DownloadSize = response.Content.Headers.ContentLength == null ? 
+                null : 
+                Size.FromLong(response.Content.Headers.ContentLength.Value)
+        };
+        return features;
+    }
+}

src/Networking/NexusMods.Networking.HttpDownloader/DTOs/Source.cs

@@ -0,0 +1,22 @@
+namespace NexusMods.Networking.HttpDownloader.DTOs;
+
+/// <summary>
+/// Information about a download source (URL mirror)
+/// </summary>
+public class Source
+{
+    /// <summary>
+    /// Request for the download
+    /// </summary>
+    public HttpRequestMessage? Request { get; init; }
+
+    /// <summary>
+    /// The priority of the source, slower sources should have a lower priority
+    /// </summary>
+    public int Priority { get; set; }
+
+    public override string ToString()
+    {
+        return Request?.RequestUri?.AbsoluteUri ?? "No URL";
+    }
+}

src/Networking/NexusMods.Networking.HttpDownloader/DTOs/WriteOrder.cs

@@ -0,0 +1,15 @@
+using System.Buffers;
+using NexusMods.Paths;
+
+namespace NexusMods.Networking.HttpDownloader.DTOs;
+
+/// <summary>
+/// Used to contain information that will be sent to the write queue.
+/// </summary>
+struct WriteOrder
+{
+    public Size Offset;
+    public Memory<byte> Data;
+    public IMemoryOwner<byte> Owner;
+    public ChunkState Chunk;
+}

src/Networking/NexusMods.Networking.HttpDownloader/HttpRequestMessageExtensions.cs

@@ -0,0 +1,25 @@
+namespace NexusMods.Networking.HttpDownloader;
+
+public static class HttpRequestMessageExtensions
+{
+    /// <summary>
+    /// Copies the request message so that it can be sent again.
+    /// </summary>
+    /// <param name="input"></param>
+    /// <returns></returns>
+    public static HttpRequestMessage Copy(this HttpRequestMessage input)
+    {
+        var newRequest = new HttpRequestMessage(input.Method, input.RequestUri);
+        foreach (var option in input.Options)
+        {
+            newRequest.Options.Set(new HttpRequestOptionsKey<object?>(option.Key), option.Value);
+        }
+
+        foreach (var header in input.Headers)
+        {
+            newRequest.Headers.TryAddWithoutValidation(header.Key, header.Value);
+        }
+
+        return newRequest;
+    }
+}

src/Networking/NexusMods.Networking.HttpDownloader/IHttpDownloaderSettings.cs

@@ -9,11 +9,6 @@ namespace NexusMods.Networking.HttpDownloader;
 [PublicAPI]
 public interface IHttpDownloaderSettings
 {
-    /// <summary>
-    /// Number of chunks to use while downloading. More chunks might mean larger server load.
-    /// </summary>
-    public int ChunkCount { get; }
-
     /// <summary>
     /// How many blocks can be queued to be written to disk. If this is too low, temporary disk slowdown could lead to
     /// worse download speed as the disk throttles the download threads. If this is high, consistently slow disk
@@ -38,9 +33,6 @@ public interface IHttpDownloaderSettings
 [PublicAPI]
 public class HttpDownloaderSettings : IHttpDownloaderSettings
 {
-    /// <inheritdoc />
-    public int ChunkCount { get; set; } = 4;
-
     /// <inheritdoc />
     public int WriteQueueLength { get; set; } = 16;
 
@@ -55,7 +47,6 @@ public class HttpDownloaderSettings : IHttpDownloaderSettings
     /// </summary>
     public void Sanitize()
     {
-        ChunkCount = ChunkCount <= 0 ? 4 : ChunkCount;
         WriteQueueLength = WriteQueueLength <= 0 ? 16 : WriteQueueLength;
         MinCancelAge = MinCancelAge <= 0 ? 500 : MinCancelAge;
         CancelSpeedFraction = CancelSpeedFraction <= 0 ? 0.66 : CancelSpeedFraction;

src/Networking/NexusMods.Networking.HttpDownloader/NexusMods.Networking.HttpDownloader.csproj

@@ -11,4 +11,9 @@
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
     </ItemGroup>
+
+
+    <ItemGroup>
+        <InternalsVisibleTo Include="NexusMods.Networking.HttpDownloader.Tests" />
+    </ItemGroup>
 </Project>

src/Networking/NexusMods.Networking.HttpDownloader/SimpleHttpDownloader.cs

@@ -39,11 +39,6 @@ public async Task<Hash> DownloadAsync(IReadOnlyList<HttpRequestMessage> sources,
             // Note: If download fails, job will be reported as 'failed', and will not participate in throughput calculations.
             state.Jobs.Add(job);
 
-            // Integrate local download.
-            var hash = await LocalFileDownloader.TryDownloadLocal(job, source, destination);
-            if (hash != null)
-                return hash.Value;
-
             var response = await _client.SendAsync(source, HttpCompletionOption.ResponseHeadersRead, token);
             if (!response.IsSuccessStatusCode)
             {

src/NexusMods.DataModel/Loadouts/LoadoutRegistry.cs

@@ -1,3 +1,4 @@
+using System.Collections.ObjectModel;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using DynamicData;
@@ -267,6 +268,8 @@ public IEnumerable<Loadout> AllLoadouts()
             .Select(id => Get(id)!);
     }
 
+
+
     /// <summary>
     /// An observable of all the revisions of a given LoadoutId
     /// </summary>

tests/Networking/NexusMods.Networking.Downloaders.Tests/HttpDownloadTaskTests.cs

@@ -3,19 +3,23 @@
 using Microsoft.Extensions.Logging;
 using NexusMods.Games.BethesdaGameStudios;
 using NexusMods.Games.TestFramework;
+using NexusMods.Networking.HttpDownloader.Tests;
 
 namespace NexusMods.Networking.Downloaders.Tests;
 
 public class HttpDownloadTaskTests : AGameTest<SkyrimSpecialEdition>
 {
     private readonly IServiceProvider _serviceProvider;
     private readonly DownloadService _downloadService;
-    public HttpDownloadTaskTests(IServiceProvider serviceProvider, DownloadService downloadService) : base(serviceProvider)
+    private readonly LocalHttpServer _server;
+
+    public HttpDownloadTaskTests(IServiceProvider serviceProvider, DownloadService downloadService, LocalHttpServer server) : base(serviceProvider)
     {
         _serviceProvider = serviceProvider;
         _downloadService = downloadService;
+        _server = server;
     }
-    
+
     [Theory]
     [InlineData("Resources/RootedAtGameFolder/-Skyrim 202X 9.0 - Architecture-2347-9-0-1664994366.zip")]
     [InlineData("Resources/RootedAtDataFolder/-Skyrim 202X 9.0 to 9.4 - Update Ravenrock.zip")]
@@ -29,7 +33,7 @@ public async Task DownloadModFromUrl(string url)
         var origNumMods = loadout.Value.Mods.Count;
         origNumMods.Should().Be(1); // game files
 
-        var makeUrl = $"file://{Path.GetFullPath(url)}";
+        var makeUrl = $"{_server.Uri}{url}";
         task.Init(makeUrl, loadout.Value);
         await task.StartAsync();
         loadout.Value.Mods.Count.Should().BeGreaterThan(origNumMods);