From 8ce691d8c64e16f34d9428304828e5774f3b6787 Mon Sep 17 00:00:00 2001 From: "Sewer." Date: Thu, 3 Oct 2024 17:06:23 +0100 Subject: [PATCH] [4/4] Query for Updates from Nexus, and Fix Miscellaneous Bugs in DataStore (#2113) * WIP: Documentation for Update Detection * Added: Additional Edge Cases to Update Logic Docs * Added: Extra case of `Archived in the Middle` * Fixed: Indentation for `file_updates` field. * Finalized 'Updating Mods' doc with simplified Implementation requested. * Fixed: Minor Notes from older Research Doc * Fixed: Added Missing 'Updating Mods' mkdocs sidebar item. * [Working WIP] Added: Initial Implementation of Generic Page Caching System used by Mod Pages * Tech Debt Reduction: Add additional V2 GraphQL Types and Correct Sizes of Existing Types * Added: Missing 'UInt32' types in attribute definitions * Improved: Accuracy of documentation for FileId struct. * Added: Method for constructing UidForMod and UidForFile from GraphQL API Results * Rename: ICanGetUid to ICanGetUidForMod * Added: Tests for UidForModTests and UidForFileTests * Removed: Unused Tests.cs file * Added: Mixin for V1 API Results to ModUpdates Library * Added: Mod Page Metadata now is ready for handling update info with V2 GameId and Friends * Added: Mixin for page metadata. * NexusModsModPageMetadata: Correctly Use uid as 'primary key' * Update: Use GameId from uid field of NexusModsModPageMetadata * Added: Fetch Mod Page Metadata from the DB * Added: Note about field in PageMetadataMixin * Use Uid in NexusModsFileMetadata, and Add Relevant Constructors for UidForFile and UidForMod * Added: Code for running actual update check, and relevant constructs. * V1: Fix field names on ModUpdate structure to align with Nexus V1 API * Added: Remaining Fixups to make the Update Check 'work' * Added: A note regarding adding more tests. * Added: Small note to ModUpdateMixin about choice of field. * Removed: FilesUpdatedAt field, as it is now currently unused. * Improve: Clarify last updated date is in UTC * Updated Note: It's no longer messy, but we still need to upgrade to V2. * Improved: Now also updates mod pages. * Merge interfaces into IModFeedItem * Fix some merge conflicts --------- Co-authored-by: halgari --- .../Json/Mod.cs | 4 + .../NexusModsFileMetadata.cs | 12 +- .../NexusModsModPageMetadata.cs | 15 ++- .../DTOs/ModUpdate.cs | 4 +- .../Types/V2/GameId.cs | 50 ++++++++ .../Types/V2/Uid/UidForFile.cs | 24 ++++ .../Types/V2/Uid/UidForMod.cs | 22 ++++ .../IModFeedItem.cs | 22 ++++ .../Mixins/ModFeedItemUpdateMixin.cs | 40 +++++++ .../Mixins/PageMetadataMixin.cs | 38 ++++++ .../MultiFeedCacheUpdater.cs | 13 +-- .../NexusMods.Networking.ModUpdates.csproj | 1 + .../PerFeedCacheUpdater.cs | 26 ++--- .../PerFeedCacheUpdaterResult.cs | 3 +- .../Extensions/FragmentExtensions.cs | 23 ++-- .../GraphQL/CommonFragments.graphql | 9 +- .../GraphQL/ModFiles.graphql | 9 ++ .../GraphQL/ModInfo.graphql | 6 +- .../NexusMods.Networking.NexusWebApi.csproj | 1 + .../NexusModsLibrary.cs | 57 +++++---- .../RunUpdateCheck.cs | 110 ++++++++++++++++++ .../InstallCollectionJob.cs | 7 +- .../Helpers/TestItem.cs | 7 +- ...xusMods.Networking.ModUpdates.Tests.csproj | 1 + .../RunUpdateCheckTests.cs | 77 ++++++++++++ .../Types/V2/UidForModTests.cs | 4 + 26 files changed, 511 insertions(+), 74 deletions(-) create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/IModFeedItem.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFeedItemUpdateMixin.cs create mode 100644 src/Networking/NexusMods.Networking.ModUpdates/Mixins/PageMetadataMixin.cs create mode 100644 src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModFiles.graphql create mode 100644 src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs create mode 100644 tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs diff --git a/src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs b/src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs index df73da513..2179db18a 100644 --- a/src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs +++ b/src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs @@ -1,5 +1,6 @@ using System.Text.Json.Serialization; using NexusMods.Abstractions.Games.DTO; +using NexusMods.Abstractions.NexusWebApi.Types.V2; namespace NexusMods.Abstractions.Collections.Json; @@ -17,6 +18,9 @@ public class Mod [JsonPropertyName("optional")] public bool Optional { get; init; } + /// + /// TODO: Deprecate this with + /// [JsonPropertyName("domainName")] public required GameDomain DomainName { get; init; } diff --git a/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsFileMetadata.cs b/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsFileMetadata.cs index 82c4c04bd..ac1bb93c1 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsFileMetadata.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsFileMetadata.cs @@ -2,6 +2,7 @@ using NexusMods.Abstractions.MnemonicDB.Attributes; using NexusMods.Abstractions.NexusWebApi.Types; using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; using NexusMods.MnemonicDB.Abstractions.Attributes; using NexusMods.MnemonicDB.Abstractions.Models; @@ -16,9 +17,9 @@ public partial class NexusModsFileMetadata : IModelDefinition private const string Namespace = "NexusMods.Library.NexusModsFileMetadata"; /// - /// The ID of the file. + /// Unique identifier for the file on Nexus Mods. /// - public static readonly FileIdAttribute FileId = new(Namespace, nameof(FileId)) { IsIndexed = true }; + public static readonly UidForFileAttribute Uid = new(Namespace, nameof(Uid)) { IsIndexed = true }; /// /// The name of the file. @@ -31,7 +32,12 @@ public partial class NexusModsFileMetadata : IModelDefinition public static readonly StringAttribute Version = new(Namespace, nameof(Version)); /// - /// The size of the file in bytes, this is optional in the NexusMods API for whatever reason. + /// The date the file was uploaded at. + /// + public static readonly DateTimeAttribute UploadedAt = new(Namespace, nameof(UploadedAt)); + + /// + /// The size in bytes of the file. /// public static readonly SizeAttribute Size = new(Namespace, nameof(Size)) { IsOptional = true }; diff --git a/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs b/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs index 79331e358..ff28578b6 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusModsLibrary/NexusModsModPageMetadata.cs @@ -1,8 +1,8 @@ using JetBrains.Annotations; using NexusMods.Abstractions.MnemonicDB.Attributes; -using NexusMods.Abstractions.NexusWebApi.Types; using NexusMods.Abstractions.Resources.DB; using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; using NexusMods.Abstractions.Telemetry; using NexusMods.MnemonicDB.Abstractions.Attributes; using NexusMods.MnemonicDB.Abstractions.Models; @@ -20,7 +20,7 @@ public partial class NexusModsModPageMetadata : IModelDefinition /// /// The ID of the mod page. /// - public static readonly ModIdAttribute ModId = new(Namespace, nameof(ModId)) { IsIndexed = true }; + public static readonly UidForModAttribute Uid = new(Namespace, nameof(Uid)) { IsIndexed = true }; /// /// The name of the mod page. @@ -30,8 +30,17 @@ public partial class NexusModsModPageMetadata : IModelDefinition /// /// The game of the mod page. /// + /// + /// This will be deprecated in the future, since V2 API only needs + /// which contains the The is a legacy field of the V1 API. + /// public static readonly GameDomainAttribute GameDomain = new(Namespace, nameof(GameDomain)) { IsIndexed = true }; + /// + /// The last time the mod page was updated (UTC). This is useful for cache invalidation. + /// + public static readonly DateTimeAttribute UpdatedAt = new(Namespace, nameof(UpdatedAt)); + /// /// Uri for the full sized picture of the mod. /// @@ -51,6 +60,6 @@ public partial class NexusModsModPageMetadata : IModelDefinition public partial struct ReadOnly { - public Uri GetUri() => NexusModsUrlBuilder.CreateGenericUri($"https://nexusmods.com/{GameDomain}/mods/{ModId}"); + public Uri GetUri() => NexusModsUrlBuilder.CreateGenericUri($"https://nexusmods.com/{GameDomain}/mods/{Uid.ModId}"); } } diff --git a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/DTOs/ModUpdate.cs b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/DTOs/ModUpdate.cs index 89c9b9ff1..c2b86c3f6 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/DTOs/ModUpdate.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/DTOs/ModUpdate.cs @@ -33,7 +33,7 @@ public class ModUpdate : IJsonArraySerializable /// /// Expressed as a Unix timestamp. /// - [JsonPropertyName("LatestFileUpdated")] + [JsonPropertyName("latest_file_update")] public long LatestFileUpdated { get; set; } /// @@ -47,7 +47,7 @@ public class ModUpdate : IJsonArraySerializable /// /// Expressed as a Unix timestamp. /// - [JsonPropertyName("LatestModActivity")] + [JsonPropertyName("latest_mod_activity")] public long LatestModActivity { get; set; } /// diff --git a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/GameId.cs b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/GameId.cs index a798ab4ad..971cfe9ef 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/GameId.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/GameId.cs @@ -1,3 +1,7 @@ +using NexusMods.Abstractions.Games.DTO; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; using TransparentValueObjects; namespace NexusMods.Abstractions.NexusWebApi.Types.V2; @@ -9,4 +13,50 @@ namespace NexusMods.Abstractions.NexusWebApi.Types.V2; { /// public static GameId DefaultValue => From(default(uint)); + + /// + /// Maps a given to a using known mappings. + /// This is a TEMPORARY API, until full migration to V2 is complete. + /// After that it should be REMOVED. + /// + public static GameId FromGameDomain(GameDomain domain) + { + return domain.Value switch + { + "stardewvalley" => (GameId)1704, + "cyberpunk2077" => (GameId)3333, + "baldursgate3" => (GameId)3474, + _ => throw new ArgumentOutOfRangeException(nameof(domain), domain, null), + }; + } + + /// + /// Maps a given to a using known mappings. + /// This is a TEMPORARY API, until full migration to V2 is complete. + /// After that it should be REMOVED. + /// + public GameDomain ToGameDomain() + { + var value = Value; + return value switch + { + 1704 => GameDomain.From("stardewvalley"), + 3333 => GameDomain.From("cyberpunk2077"), + 3474 => GameDomain.From("baldursgate3"), + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null), + }; + } +} + +/// +/// Game ID attribute, for game identifiers from the GraphQL (V2) API. +/// +public class GameIdAttribute(string ns, string name) + : ScalarAttribute(ValueTags.UInt32, ns, name) +{ + /// + protected override uint ToLowLevel(GameId value) => value.Value; + + /// + protected override GameId FromLowLevel(uint value, ValueTags tags, AttributeResolver resolver) => GameId.From(value); } diff --git a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForFile.cs b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForFile.cs index f644dab2b..9ad050f45 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForFile.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForFile.cs @@ -1,5 +1,8 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; namespace NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; /// @@ -25,6 +28,13 @@ public struct UidForFile /// public GameId GameId; + /// + public UidForFile(FileId fileId, GameId gameId) + { + FileId = fileId; + GameId = gameId; + } + /// /// Decodes a Nexus Mods API result which contains an 'uid' field into a . /// @@ -44,3 +54,17 @@ public struct UidForFile /// public static UidForFile FromUlong(ulong value) => Unsafe.As(ref value); } + +/// +/// Attribute that uniquely identifies a file on Nexus Mods. +/// See for more details. +/// +public class UidForFileAttribute(string ns, string name) + : ScalarAttribute(ValueTags.UInt64, ns, name) +{ + /// + protected override ulong ToLowLevel(UidForFile value) => value.AsUlong; + + /// + protected override UidForFile FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => UidForFile.FromUlong(value); +} diff --git a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForMod.cs b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForMod.cs index 1e5f8987d..c2fafe151 100644 --- a/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForMod.cs +++ b/src/Abstractions/NexusMods.Abstractions.NexusWebApi/Types/V2/Uid/UidForMod.cs @@ -1,5 +1,8 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; namespace NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; /// @@ -36,6 +39,11 @@ public struct UidForMod /// This throws if is not a valid number. /// public static UidForMod FromV2Api(string uid) => FromUlong(ulong.Parse(uid)); + + /// + /// Converts the UID to a string accepted by the V2 API. + /// + public string ToV2Api() => AsUlong.ToString(); /// /// Reinterprets the current as a single . @@ -47,3 +55,17 @@ public struct UidForMod /// public static UidForMod FromUlong(ulong value) => Unsafe.As(ref value); } + +/// +/// Attribute that uniquely identifies a mod on Nexus Mods. +/// See for more details. +/// +public class UidForModAttribute(string ns, string name) + : ScalarAttribute(ValueTags.UInt64, ns, name) +{ + /// + protected override ulong ToLowLevel(UidForMod value) => value.AsUlong; + + /// + protected override UidForMod FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver) => UidForMod.FromUlong(value); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/IModFeedItem.cs b/src/Networking/NexusMods.Networking.ModUpdates/IModFeedItem.cs new file mode 100644 index 000000000..d9d316698 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/IModFeedItem.cs @@ -0,0 +1,22 @@ +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; +namespace NexusMods.Networking.ModUpdates; + +/// +/// Represents an individual item from a 'mod feed'; with the 'mod feed' being +/// the result of an API call that returns one or more mods from the Nexus API. +/// (Either V1 or V2 API) +/// +public interface IModFeedItem +{ + /// + /// Returns a unique identifier for the given item, based on the ID format + /// used in the NexusMods V2 API. + /// + public UidForMod GetModPageId(); + + /// + /// Retrieves the time the item was last updated. + /// This date is in UTC. + /// + public DateTime GetLastUpdatedDateUtc(); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFeedItemUpdateMixin.cs b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFeedItemUpdateMixin.cs new file mode 100644 index 000000000..8ba3410d0 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFeedItemUpdateMixin.cs @@ -0,0 +1,40 @@ +using NexusMods.Abstractions.NexusWebApi.DTOs; +using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; +namespace NexusMods.Networking.ModUpdates.Mixins; + +/// +/// Implements the (V1) mod update API mixin. +/// +public readonly struct ModFeedItemUpdateMixin : IModFeedItem +{ + private readonly DateTime _lastUpdatedDate; + private readonly GameId _gameId; + private readonly ModId _modId; + + /// + private ModFeedItemUpdateMixin(ModUpdate update, GameId gameId) + { + // Note(sewer): V2 doesn't have 'last file updated' field, so we have to use 'last mod page update' time. + // Well, this whole struct is, will be making that ticket to backend, and replace + // this when V2 gets relevant API. + _lastUpdatedDate = update.LatestModActivityUtc; + _gameId = gameId; + _modId = update.ModId; + } + + /// + /// Transforms the result of a V1 API call for mod updates into the Mixin. + /// + public static IEnumerable FromUpdateResults(IEnumerable updates, GameId gameId) => updates.Select(update => new ModFeedItemUpdateMixin(update, gameId)); + + /// + public DateTime GetLastUpdatedDateUtc() => _lastUpdatedDate; + + /// + public UidForMod GetModPageId() => new() + { + GameId = _gameId, + ModId = _modId, + }; +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Mixins/PageMetadataMixin.cs b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/PageMetadataMixin.cs new file mode 100644 index 000000000..b091f528e --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/PageMetadataMixin.cs @@ -0,0 +1,38 @@ +using NexusMods.Abstractions.NexusModsLibrary; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; +using NexusMods.MnemonicDB.Abstractions; +namespace NexusMods.Networking.ModUpdates.Mixins; + +/// +/// Implements the MnemonicDB mod page mixin based on V2 API Results. +/// +public struct PageMetadataMixin : IModFeedItem +{ + private readonly NexusModsModPageMetadata.ReadOnly _metadata; + + private PageMetadataMixin(NexusModsModPageMetadata.ReadOnly metadata) => _metadata = metadata; + + /// + public UidForMod GetModPageId() => new() + { + GameId = _metadata.Uid.GameId, + ModId = _metadata.Uid.ModId, + }; + + /// + public EntityId GetModPageEntityId() => _metadata.Id; + + /// + public DateTime GetLastUpdatedDateUtc() => _metadata.UpdatedAt; // <= TODO: Change this with 'last file updated at' when V2 supports this field. + + /// + /// Returns the database entries containing page metadata(s) as a mixin. + /// + public static IEnumerable EnumerateDatabaseEntries(IDb db) => NexusModsModPageMetadata.All(db).Select(only => new PageMetadataMixin(only)); + + /// + public static implicit operator NexusModsModPageMetadata.ReadOnly(PageMetadataMixin mixin) => mixin._metadata; + + /// + public static implicit operator PageMetadataMixin(NexusModsModPageMetadata.ReadOnly metadata) => new(metadata); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs b/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs index 166d21efd..2171f8a7e 100644 --- a/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs +++ b/src/Networking/NexusMods.Networking.ModUpdates/MultiFeedCacheUpdater.cs @@ -1,5 +1,4 @@ using NexusMods.Abstractions.NexusWebApi.Types.V2; -using NexusMods.Networking.ModUpdates.Traits; namespace NexusMods.Networking.ModUpdates; /// @@ -12,7 +11,7 @@ namespace NexusMods.Networking.ModUpdates; /// you to use mods and API responses which are sourced from multiple feeds (games), /// as opposed to a single feed. /// -public class MultiFeedCacheUpdater where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUidForMod +public class MultiFeedCacheUpdater where TUpdateableItem : IModFeedItem { private readonly Dictionary> _updaters; @@ -41,7 +40,7 @@ public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) var groupedList = new List<(GameId, List)>(); foreach (var item in items) { - var gameId = item.GetUniqueId().GameId; + var gameId = item.GetModPageId().GameId; // Get or Update List for this GameId. var found = false; @@ -73,15 +72,15 @@ public MultiFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) /// include items corresponding to multiple feeds (games); the feed source /// is automatically detected. /// - /// Wrap elements in a struct that implements - /// and if necessary. + /// Wrap elements in a struct that implements + /// and if necessary. /// - public void Update(IEnumerable items) where T : ICanGetLastUpdatedTimestamp, ICanGetUidForMod + public void Update(IEnumerable items) where T : IModFeedItem { foreach (var item in items) { // Determine feed - var feed = item.GetUniqueId().GameId; + var feed = item.GetModPageId().GameId; // The result may contain items from feeds which we are not tracking. // For instance, results for other games. This is not an error, we diff --git a/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj b/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj index adbddb405..427c334d1 100644 --- a/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj +++ b/src/Networking/NexusMods.Networking.ModUpdates/NexusMods.Networking.ModUpdates.csproj @@ -15,6 +15,7 @@ + diff --git a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs index f60a07562..56b99dda2 100644 --- a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs +++ b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdater.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using NexusMods.Abstractions.NexusWebApi.Types.V2; using NexusMods.Networking.ModUpdates.Private; -using NexusMods.Networking.ModUpdates.Traits; namespace NexusMods.Networking.ModUpdates; /// @@ -11,8 +10,8 @@ namespace NexusMods.Networking.ModUpdates; /// /// This API consists of the following: /// -/// 1. Input [Constructor]: A set of items with a 'last update time' (see ) -/// and a 'unique id' (see ) that are relevant to the current 'feed' (game). +/// 1. Input [Constructor]: A set of items with a 'last update time' and a 'unique id' +/// (see ) that are relevant to the current 'feed' (game). /// /// 2. Update [Method]: Submit results from API endpoint returning 'most recently updated mods for game'. /// This updates the internal state of the . @@ -31,7 +30,7 @@ namespace NexusMods.Networking.ModUpdates; /// The 'Feed' in the context of the Nexus App is the individual game's 'updated.json' endpoint; /// i.e. a 'Game Mod Feed' /// -public class PerFeedCacheUpdater where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUidForMod +public class PerFeedCacheUpdater where TUpdateableItem : IModFeedItem { private readonly TUpdateableItem[] _items; private readonly Dictionary _itemToIndex; @@ -60,14 +59,14 @@ public PerFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) _actions = new CacheUpdaterAction[items.Length]; _itemToIndex = new Dictionary(items.Length); for (var x = 0; x < _items.Length; x++) - _itemToIndex[_items[x].GetUniqueId().ModId] = x; + _itemToIndex[_items[x].GetModPageId().ModId] = x; // Set the action to refresh cache for any mods which exceed max age. var utcNow = DateTime.UtcNow; var minCachedDate = utcNow - expiry; for (var x = 0; x < _items.Length; x++) { - if (_items[x].GetLastUpdatedDate() < minCachedDate) + if (_items[x].GetLastUpdatedDateUtc() < minCachedDate) _actions[x] = CacheUpdaterAction.NeedsUpdate; } } @@ -78,27 +77,26 @@ public PerFeedCacheUpdater(TUpdateableItem[] items, TimeSpan expiry) /// /// /// The items returned by the 'most recently updated mods for game' endpoint. - /// Wrap elements in a struct that implements - /// and if necessary. + /// Wrap elements in a struct that implements if necessary. /// - public void Update(IEnumerable items) where T : ICanGetLastUpdatedTimestamp, ICanGetUidForMod + public void Update(IEnumerable items) where T : IModFeedItem { foreach (var item in items) UpdateSingleItem(item); } - internal void UpdateSingleItem(T item) where T : ICanGetLastUpdatedTimestamp, ICanGetUidForMod + internal void UpdateSingleItem(T item) where T : IModFeedItem { // Try to get index of the item. // Not all the items from the update feed are locally stored, thus we need to // make sure we actually have this item. - if (!_itemToIndex.TryGetValue(item.GetUniqueId().ModId, out var index)) + if (!_itemToIndex.TryGetValue(item.GetModPageId().ModId, out var index)) return; var existingItem = _items[index]; // If the file timestamp is newer than our cached copy, the item needs updating. - if (item.GetLastUpdatedDate() > existingItem.GetLastUpdatedDate()) + if (item.GetLastUpdatedDateUtc() > existingItem.GetLastUpdatedDateUtc()) _actions[index] = CacheUpdaterAction.NeedsUpdate; else _actions[index] = CacheUpdaterAction.UpdateLastCheckedTimestamp; @@ -147,8 +145,8 @@ private void DebugVerifyAllItemsAreFromSameGame() { if (_items.Length == 0) return; - var firstGameId = _items[0].GetUniqueId().GameId; - var allSame = _items.All(x => x.GetUniqueId().GameId == firstGameId); + var firstGameId = _items[0].GetModPageId().GameId; + var allSame = _items.All(x => x.GetModPageId().GameId == firstGameId); if (!allSame) throw new ArgumentException("All items must have the same game id", nameof(_items)); } diff --git a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs index 78287340c..d1eee234a 100644 --- a/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs +++ b/src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs @@ -1,4 +1,3 @@ -using NexusMods.Networking.ModUpdates.Traits; namespace NexusMods.Networking.ModUpdates; /// @@ -6,7 +5,7 @@ namespace NexusMods.Networking.ModUpdates; /// of feeds. /// /// Wrapper for item supported by the cache updater. -public class PerFeedCacheUpdaterResult where TUpdateableItem : ICanGetLastUpdatedTimestamp, ICanGetUidForMod +public class PerFeedCacheUpdaterResult where TUpdateableItem : IModFeedItem { /// /// This is a list of items that is 'out of date'. diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/Extensions/FragmentExtensions.cs b/src/Networking/NexusMods.Networking.NexusWebApi/Extensions/FragmentExtensions.cs index f6fd50dc8..ec77a9f75 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/Extensions/FragmentExtensions.cs +++ b/src/Networking/NexusMods.Networking.NexusWebApi/Extensions/FragmentExtensions.cs @@ -1,8 +1,8 @@ using NexusMods.Abstractions.Games.DTO; using NexusMods.Abstractions.NexusModsLibrary; using NexusMods.Abstractions.NexusModsLibrary.Models; -using NexusMods.Abstractions.NexusWebApi.Types; using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; using NexusMods.MnemonicDB.Abstractions; using NexusMods.Paths; @@ -26,16 +26,21 @@ public static async Task Resolve(this IUserFragment userFragment, IDb userResolver.Add(User.AvatarImage,avatarImage); return userResolver.Id; } - + /// - /// Resolves the IModFragment to an entity in the database, inserting or updating as necessary. + /// Resolves the to an entity in the database, inserting or updating as necessary. /// - public static EntityId Resolve(this IModFileFragment modFileFragment, IDb db, ITransaction tx, EntityId modEId) + /// Fragment obtained from the GraphQL API call. + /// Provides DB access. + /// The current transaction for inserting items into database., + /// ID of the mod page entity. + public static EntityId Resolve(this IModFileFragment modFileFragment, IDb db, ITransaction tx, EntityId modPageEid) { - var nexusFileResolver = GraphQLResolver.Create(db, tx, (NexusModsFileMetadata.FileId, FileId.From((uint)modFileFragment.FileId)), (NexusModsFileMetadata.ModPageId, modEId)); - nexusFileResolver.Add(NexusModsFileMetadata.ModPageId, modEId); + var nexusFileResolver = GraphQLResolver.Create(db, tx, NexusModsFileMetadata.Uid, UidForFile.FromV2Api(modFileFragment.Uid)); + nexusFileResolver.Add(NexusModsFileMetadata.ModPageId, modPageEid); nexusFileResolver.Add(NexusModsFileMetadata.Name, modFileFragment.Name); nexusFileResolver.Add(NexusModsFileMetadata.Version, modFileFragment.Version); + nexusFileResolver.Add(NexusModsFileMetadata.UploadedAt, DateTimeOffset.FromUnixTimeSeconds(modFileFragment.Date).DateTime); if (ulong.TryParse(modFileFragment.SizeInBytes, out var size)) nexusFileResolver.Add(NexusModsFileMetadata.Size, Size.From(size)); return nexusFileResolver.Id; @@ -46,10 +51,10 @@ public static EntityId Resolve(this IModFileFragment modFileFragment, IDb db, IT /// public static EntityId Resolve(this IModFragment modFragment, IDb db, ITransaction tx) { - var nexusModResolver = GraphQLResolver.Create(db, tx, NexusModsModPageMetadata.ModId, ModId.From((uint)modFragment.ModId)); - + var nexusModResolver = GraphQLResolver.Create(db, tx, NexusModsModPageMetadata.Uid, UidForMod.FromV2Api(modFragment.Uid)); nexusModResolver.Add(NexusModsModPageMetadata.Name, modFragment.Name); nexusModResolver.Add(NexusModsModPageMetadata.GameDomain, GameDomain.From(modFragment.Game.DomainName)); + nexusModResolver.Add(NexusModsModPageMetadata.UpdatedAt, modFragment.UpdatedAt.UtcDateTime); if (Uri.TryCreate(modFragment.PictureUrl, UriKind.Absolute, out var fullSizedPictureUri)) nexusModResolver.Add(NexusModsModPageMetadata.FullSizedPictureUri, fullSizedPictureUri); @@ -63,7 +68,7 @@ private static async Task DownloadImage(HttpClient client, string? uri, { if (uri is null) return []; if (!Uri.TryCreate(uri, UriKind.Absolute, out var imageUri)) return []; - + return await client.GetByteArrayAsync(imageUri, token); } diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/CommonFragments.graphql b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/CommonFragments.graphql index f27b5fada..2cb14b476 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/CommonFragments.graphql +++ b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/CommonFragments.graphql @@ -8,9 +8,11 @@ fragment UserFragment on User { fragment ModFileFragment on ModFile { name, modId, - fileId, version, - sizeInBytes + sizeInBytes, + date, + # This contains fileId and gameId + uid } fragment ModFragment on Mod { @@ -18,7 +20,10 @@ fragment ModFragment on Mod { name game { domainName + id } thumbnailUrl pictureUrl + updatedAt, + uid } diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModFiles.graphql b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModFiles.graphql new file mode 100644 index 000000000..493d5167b --- /dev/null +++ b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModFiles.graphql @@ -0,0 +1,9 @@ +#include { ModFragment } from './CommonFragments.graphql' + +# Retrieves all files for a given mod. +query ModFiles($modId: ID!, $gameId: ID!) { + modFiles(modId: $modId, gameId: $gameId) { + ...ModFileFragment + } +} + diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModInfo.graphql b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModInfo.graphql index 894a14b68..6e1ce3ed6 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModInfo.graphql +++ b/src/Networking/NexusMods.Networking.NexusWebApi/GraphQL/ModInfo.graphql @@ -1,10 +1,10 @@ #include { ModFragment } from './CommonFragments.graphql' -query ModInfo($gameDomain: String!, $modId: Int!) +query ModInfo($gameId: Int!, $modId: Int!) { - legacyModsByDomain(ids: [{gameDomain: $gameDomain, modId: $modId}]) + legacyMods(ids: [{gameId: $gameId, modId: $modId}]) { - nodes + nodes { ...ModFragment } diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj b/src/Networking/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj index 4767aaf7d..7d6a4d484 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj +++ b/src/Networking/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj @@ -21,5 +21,6 @@ + diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/NexusModsLibrary.cs b/src/Networking/NexusMods.Networking.NexusWebApi/NexusModsLibrary.cs index 21076625f..eb018f569 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/NexusModsLibrary.cs +++ b/src/Networking/NexusMods.Networking.NexusWebApi/NexusModsLibrary.cs @@ -10,6 +10,7 @@ using NexusMods.Abstractions.NexusWebApi.DTOs; using NexusMods.Abstractions.NexusWebApi.Types; using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; using NexusMods.Extensions.BCL; using NexusMods.MnemonicDB.Abstractions; using NexusMods.Networking.HttpDownloader; @@ -45,14 +46,19 @@ public NexusModsLibrary(IServiceProvider serviceProvider) GameDomain gameDomain, CancellationToken cancellationToken = default) { - var modPageEntities = NexusModsModPageMetadata.FindByModId(_connection.Db, modId); + var uid = new UidForMod + { + GameId = GameId.FromGameDomain(gameDomain), + ModId = modId, + }; + var modPageEntities = NexusModsModPageMetadata.FindByUid(_connection.Db, uid); if (modPageEntities.TryGetFirst(x => x.GameDomain == gameDomain, out var modPage)) return modPage; using var tx = _connection.BeginTransaction(); - var modInfo = await _gqlClient.ModInfo.ExecuteAsync(gameDomain.ToString(), (int)modId.Value, cancellationToken); + var modInfo = await _gqlClient.ModInfo.ExecuteAsync((int)uid.GameId.Value, (int)modId.Value, cancellationToken); EntityId first = default; - foreach (var node in modInfo.Data!.LegacyModsByDomain.Nodes) + foreach (var node in modInfo.Data!.LegacyMods.Nodes) { first = node.Resolve(_connection.Db, tx); } @@ -82,19 +88,14 @@ public NexusModsLibrary(IServiceProvider serviceProvider) collectionResolver.Add(CollectionMetadata.Name, collectionInfo.Name); collectionResolver.Add(CollectionMetadata.Summary, collectionInfo.Summary); collectionResolver.Add(CollectionMetadata.Endorsements, (ulong)collectionInfo.Endorsements); - - var thumbnailUrl = collectionInfo.TileImage?.ThumbnailUrl; - if (thumbnailUrl is not null && Uri.TryCreate(thumbnailUrl, UriKind.Absolute, out var thumbnailUri)) - { - collectionResolver.Add(CollectionMetadata.TileImageUri, thumbnailUri); - } - - var headerImageUrl = collectionInfo.HeaderImage?.Url; - if (headerImageUrl is not null && Uri.TryCreate(headerImageUrl, UriKind.Absolute, out var headerImageUri)) - { - collectionResolver.Add(CollectionMetadata.BackgroundImageUri, headerImageUri); - } - + + if (Uri.TryCreate(collectionInfo.TileImage?.ThumbnailUrl, UriKind.Absolute, out var tileImageUri)) + collectionResolver.Add(CollectionMetadata.TileImageUri, tileImageUri); + + if (Uri.TryCreate(collectionInfo.HeaderImage?.Url, UriKind.Absolute, out var backgroundImageUri)) + collectionResolver.Add(CollectionMetadata.BackgroundImageUri, backgroundImageUri); + + var user = await collectionInfo.User.Resolve(db, tx, _httpClient, token); collectionResolver.Add(CollectionMetadata.Author, user); @@ -125,6 +126,14 @@ public NexusModsLibrary(IServiceProvider serviceProvider) var txResults = await tx.Commit(); return CollectionRevisionMetadata.Load(txResults.Db, txResults[revisionResolver.Id]); } + + private async Task DownloadImage(string? uri, CancellationToken token) + { + if (uri is null) return []; + if (!Uri.TryCreate(uri, UriKind.Absolute, out var imageUri)) return []; + + return await _httpClient.GetByteArrayAsync(imageUri, token); + } public async Task GetOrAddFile( FileId fileId, @@ -132,12 +141,13 @@ public NexusModsLibrary(IServiceProvider serviceProvider) GameDomain gameDomain, CancellationToken cancellationToken = default) { - var fileEntities = NexusModsFileMetadata.FindByFileId(_connection.Db, fileId); + var uid = new UidForFile(fileId, modPage.Uid.GameId); + var fileEntities = NexusModsFileMetadata.FindByUid(_connection.Db, uid); if (fileEntities.TryGetFirst(x => x.ModPageId == modPage, out var file)) return file; using var tx = _connection.BeginTransaction(); - var filesResponse = await _apiClient.ModFilesAsync(gameDomain.ToString(), modPage.ModId, cancellationToken); + var filesResponse = await _apiClient.ModFilesAsync(gameDomain.ToString(), modPage.Uid.ModId, cancellationToken); var files = filesResponse.Data.Files; if (!files.TryGetFirst(x => x.FileId == fileId, out var fileInfo)) @@ -151,8 +161,9 @@ public NexusModsLibrary(IServiceProvider serviceProvider) { Name = fileInfo.Name, Version = fileInfo.Version, - FileId = fileId, ModPageId = modPage, + Uid = UidForFile.FromUlong((ulong)fileInfo.Uid), + UploadedAt = fileInfo.UploadedTime, Size = size.HasValue ? size.Value : null, }; @@ -176,8 +187,8 @@ public async Task GetDownloadUri( var (key, expirationDate) = nxmData.Value; links = await _apiClient.DownloadLinksAsync( file.ModPage.GameDomain.ToString(), - file.ModPage.ModId, - file.FileId, + file.ModPage.Uid.ModId, + file.Uid.FileId, key: key, expireTime: expirationDate, token: cancellationToken @@ -188,8 +199,8 @@ public async Task GetDownloadUri( // NOTE(erri120): premium-only API links = await _apiClient.DownloadLinksAsync( file.ModPage.GameDomain.ToString(), - file.ModPage.ModId, - file.FileId, + file.ModPage.Uid.ModId, + file.Uid.FileId, token: cancellationToken ); } diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs b/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs new file mode 100644 index 000000000..24e888346 --- /dev/null +++ b/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs @@ -0,0 +1,110 @@ +using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.NexusModsLibrary; +using NexusMods.Abstractions.NexusWebApi; +using NexusMods.Abstractions.NexusWebApi.Types; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.Networking.ModUpdates; +using NexusMods.Networking.ModUpdates.Mixins; +using NexusMods.Networking.NexusWebApi.Extensions; +using StrawberryShake; +namespace NexusMods.Networking.NexusWebApi; + +/// +/// Utility class that encapsulates the logic for running the actual update check. +/// +public static class RunUpdateCheck +{ + /// + /// Identifies all mod pages whose information needs refreshed. + /// + public static async Task> CheckForModPagesWhichNeedUpdating(IDb db, INexusApiClient apiClient) + { + // Extract all GameDomain(s) + var modPages = PageMetadataMixin.EnumerateDatabaseEntries(db).ToArray(); + var gameIds = modPages.Select(x => (x.GetModPageId().GameId)).Distinct().ToArray(); + + // Note: The v1Timespan accounts for 1 month minus 5 minutes + // - We use 28 days because February is the shortest month at 28. + // - Serverside caches for 5 minutes, so we subtract that. + var v1Timespan = TimeSpan.FromDays(28).Subtract(TimeSpan.FromMinutes(5)); + var updater = new MultiFeedCacheUpdater(modPages, v1Timespan); + + foreach (var gameId in gameIds) + { + // Note (sewer): We need to update to V2 stat. + var modUpdates = await apiClient.ModUpdatesAsync(gameId.ToGameDomain().Value, PastTime.Month); + var updateResults = ModFeedItemUpdateMixin.FromUpdateResults(modUpdates.Data, gameId); + updater.Update(updateResults); + } + + return updater.BuildFlattened(); + } + + /// + /// Updates the metadata for mod pages returned from the API call. + /// + public static async Task UpdateModFilesForOutdatedPages(IDb db, ITransaction tx, ILogger logger, INexusGraphQLClient gqlClient, PerFeedCacheUpdaterResult result, CancellationToken cancellationToken) + { + // Note(sewer): Undetermined items may be removed items from the site; or + // caused by programmer error, so wr should log these whenever possible, + // but they should not cause a critical error; in case it's simply the result + // of mod removal such as DMCA takedown. + foreach (var mixin in result.UndeterminedItems) + { + try + { + await UpdateModPage(db, tx, gqlClient, cancellationToken, mixin); + } + catch (Exception e) + { + var id = mixin.GetModPageId(); + logger.LogError(e, "Failed to update metadata for Mod (GameID: {Page}, ModId: {ModId})", id.GameId, id.ModId); + } + } + + // Note(sewer): But I'm not sure where to put this yet, all the GraphQL stuff is source generated. + foreach (var mixin in result.OutOfDateItems) + { + // For the remaining items, failure to obtain result here should be truly exceptional. + await UpdateModPage(db, tx, gqlClient, cancellationToken, mixin); + } + } + + private static async Task UpdateModPage(IDb db, ITransaction tx, INexusGraphQLClient gqlClient, CancellationToken cancellationToken, PageMetadataMixin mixin) + { + var uid = mixin.GetModPageId(); + var modIdString = uid.ModId.Value.ToString(); + var gameIdString = uid.GameId.Value.ToString(); + + // Update Mod + var modInfo = await gqlClient.ModInfo.ExecuteAsync((int)uid.GameId.Value, (int)uid.ModId.Value, cancellationToken); + foreach (var node in modInfo.Data!.LegacyMods.Nodes) + node.Resolve(db, tx); + + // Update Mod Files + var filesByUid = await gqlClient.ModFiles.ExecuteAsync(modIdString, gameIdString, cancellationToken); + filesByUid.EnsureNoErrors(); + + var pageEntityId = mixin.GetModPageEntityId(); + foreach (var node in filesByUid.Data!.ModFiles) + node.Resolve(db, tx, pageEntityId); + } + + /// + /// Returns all files which have a 'newer' date than the current one. + /// + public static IEnumerable GetNewerFilesForExistingFile(IDb db, UidForFile uid) + { + var metadata = NexusModsFileMetadata.FindByUid(db, uid).First(); + return GetNewerFilesForExistingFile(metadata); + } + + /// + /// Returns all files which have a 'newer' date than the current one. + /// + public static IEnumerable GetNewerFilesForExistingFile(NexusModsFileMetadata.ReadOnly file) + { + return file.ModPage.Files.Where(x => x.UploadedAt > file.UploadedAt); + } +} diff --git a/src/NexusMods.Collections/InstallCollectionJob.cs b/src/NexusMods.Collections/InstallCollectionJob.cs index ca7b3ccc3..7c29f6d1e 100644 --- a/src/NexusMods.Collections/InstallCollectionJob.cs +++ b/src/NexusMods.Collections/InstallCollectionJob.cs @@ -13,6 +13,8 @@ using NexusMods.Abstractions.Loadouts; using NexusMods.Abstractions.NexusModsLibrary; using NexusMods.Abstractions.NexusWebApi.Types; +using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; using NexusMods.MnemonicDB.Abstractions; using NexusMods.Networking.NexusWebApi; using NexusMods.Paths; @@ -206,8 +208,9 @@ private async Task EnsureDownloaded(Mod mod) private async Task EnsureNexusModDownloaded(Mod mod) { var db = Connection.Db; - var file = NexusModsFileMetadata.FindByFileId(db, mod.Source.FileId) - .Where(f => f.ModPage.ModId == mod.Source.ModId) + var uid = new UidForFile(mod.Source.FileId, GameId.FromGameDomain(mod.DomainName)); + var file = NexusModsFileMetadata.FindByUid(db, uid) + .Where(f => f.ModPage.Uid.ModId == mod.Source.ModId) .FirstOrOptional(f => f.LibraryFiles.Any()); if (file.HasValue) diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs index cee5ced92..3317d01ef 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs @@ -1,16 +1,15 @@ using NexusMods.Abstractions.NexusWebApi.Types.V2; using NexusMods.Abstractions.NexusWebApi.Types.V2.Uid; -using NexusMods.Networking.ModUpdates.Traits; namespace NexusMods.Networking.ModUpdates.Tests.Helpers; // Helper class to simulate updateable items -public class TestItem : ICanGetLastUpdatedTimestamp, ICanGetUidForMod +public class TestItem : IModFeedItem { public DateTime LastUpdated { get; set; } public UidForMod Uid { get; set; } - public DateTime GetLastUpdatedDate() => LastUpdated; - public UidForMod GetUniqueId() => Uid; + public DateTime GetLastUpdatedDateUtc() => LastUpdated; + public UidForMod GetModPageId() => Uid; // Helper method to create a test item public static TestItem Create(uint gameId, uint modId, DateTime lastUpdated) diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj index ca2fea394..55009efdb 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/NexusMods.Networking.ModUpdates.Tests.csproj @@ -4,6 +4,7 @@ + diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs new file mode 100644 index 000000000..35a9a3b92 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs @@ -0,0 +1,77 @@ +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.GC; +using NexusMods.Abstractions.Loadouts.Files; +using NexusMods.Abstractions.MnemonicDB.Attributes.Extensions; +using NexusMods.Abstractions.NexusModsLibrary; +using NexusMods.Games.RedEngine.Cyberpunk2077; +using NexusMods.Games.TestFramework; +using NexusMods.Networking.NexusWebApi; +using Xunit.Abstractions; +using FileId = NexusMods.Abstractions.NexusWebApi.Types.V2.FileId; +using ModId = NexusMods.Abstractions.NexusWebApi.Types.V2.ModId; + +namespace NexusMods.Networking.ModUpdates.Tests; + +public class RunUpdateCheckTests : ACyberpunkIsolatedGameTest +{ + private readonly NexusModsLibrary _nexusModsLibrary; + + public RunUpdateCheckTests(ITestOutputHelper outputHelper) : base(outputHelper) + { + _nexusModsLibrary = ServiceProvider.GetRequiredService(); + } + + // TODO: Add more tests in here. - Sewer + // We shouldn't be relying on live site data for testing, however, we are still in + // the process of getting the remaining V2 APIs in. In order to avoid wasted effort, + // we won't be mocking the V1 APIs; so more complex/stable tests involving mocks will + // move after full V2 move. + + [Fact] + [Trait("RequiresNetworking", "True")] + public async Task UpdatingModPageMetadata_ViaWebApi_ShouldWork() + { + // Create loadout + var loadout = await CreateLoadout(); + + // Install a version of CET into the loadout. + var modId = ModId.From(107u); // CET + var fileId = FileId.From(18963u); // 1.18.1 + + await using var tempFile = TemporaryFileManager.CreateFile(); + var downloadJob = await _nexusModsLibrary.CreateDownloadJob( + destination: tempFile, + gameDomain: Cyberpunk2077Game.StaticDomain, + modId: modId, + fileId: fileId + ); + + // install to loadout + var libraryFile = await LibraryService.AddDownload(downloadJob); + await LibraryService.InstallItem(libraryFile.AsLibraryItem(), loadout); + + // Ensure we're actually doing work + var updates = await RunUpdateCheck.CheckForModPagesWhichNeedUpdating(Connection.Db, NexusNexusApiClient); + + // We're relying on real data (CET), not a placeholder page. + // Creating a placeholder is against TOS/Guidelines, so for now we + // can only assert some 'general' knowledge. + // A single mod page got updated here. + updates.OutOfDateItems.Should().HaveCount(1); + var outOfDateMod = NexusModsModPageMetadata.FindByUid(Connection.Db, updates.OutOfDateItems.First().GetModPageId()).First(); + var outOfDateFileUid = outOfDateMod.Files.First().Uid; + + // Fetch updated content for mod pages. + using var tx = Connection.BeginTransaction(); + var gqlClient = ServiceProvider.GetRequiredService(); + await RunUpdateCheck.UpdateModFilesForOutdatedPages(Connection.Db, tx, Logger, gqlClient, updates, CancellationToken.None); + await tx.Commit(); + + // Get the collection of newer mods, there should at least be 43 at time of + // collection. We're not filtering out archived, so this number can never change + var newerMods = RunUpdateCheck.GetNewerFilesForExistingFile(Connection.Db, outOfDateFileUid); + newerMods.Should().HaveCountGreaterThan(42); + } +} diff --git a/tests/Networking/NexusMods.Networking.NexusWebApi.Tests/Types/V2/UidForModTests.cs b/tests/Networking/NexusMods.Networking.NexusWebApi.Tests/Types/V2/UidForModTests.cs index f93af9155..d117c430c 100644 --- a/tests/Networking/NexusMods.Networking.NexusWebApi.Tests/Types/V2/UidForModTests.cs +++ b/tests/Networking/NexusMods.Networking.NexusWebApi.Tests/Types/V2/UidForModTests.cs @@ -69,6 +69,10 @@ public void FromV2Api_ValidInput_ReturnsCorrectUidForMod(uint expectedGameId, ui // Assert result.GameId.Should().Be((GameId)expectedGameId); result.ModId.Should().Be((ModId)expectedModId); + + // Assert round trip + var newString = result.ToV2Api(); + newString.Should().Be(uidString); } [Fact]