From 6464122b297640f5bc77fad739ec20bb54fb43c3 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Fri, 6 Sep 2024 14:38:58 +0100 Subject: [PATCH] Update to send new bits --- README.md | 3 + docs/Reference/Generated/MigrationTools.xml | 12 +- .../GetShieldIoWorkItemMetrics.cs | 181 ++++++++++++++++-- .../profile.arm.json | 113 +++++++++++ .../Services/ActivitySourceProvider.cs | 1 - 5 files changed, 283 insertions(+), 27 deletions(-) create mode 100644 src/MigrationTools.Telemetery/Properties/ServiceDependencies/MigrationToolsTelemetery - Web Deploy/profile.arm.json diff --git a/README.md b/README.md index 5dbc262e0..5dccb3762 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,9 @@ The documentation for the preview is on [Preview](https://nkdagility.com/docs/az | Migration Run Total | **19bn Seconds** | Thats **316m hours** or **13m days** of run time in the last 30 days. | | Average Work item Migration Time | **22s** | Work Item (includes all revisions, links, and attachments for the work item) | +![All Work Items Migrated with new version](https://img.shields.io/endpoint?url=https%3A%2F%2Fmigrationtoolstelemetery.azurewebsites.net%2Fapi%2FGetShieldIoWorkItemMetrics_WorkItemTotals%3Fcode%3DN0Lfmkrz_Rzh1lrd5eol1IuDIUfOVnX0oFaT8k6m_uX8AzFu27y_og%3D%3D) + + This tool uses OpenTelemetery to collect metrics and logs, and Application Insights to store and analyse them. Exceptions are also sent to [Elmah.io](https://elmah.io) for analysis and improvement. ## Advanced tools diff --git a/docs/Reference/Generated/MigrationTools.xml b/docs/Reference/Generated/MigrationTools.xml index 8b2ccd50b..0929d08d0 100644 --- a/docs/Reference/Generated/MigrationTools.xml +++ b/docs/Reference/Generated/MigrationTools.xml @@ -263,27 +263,27 @@ - => @"91ab41d2" + => @"a0ce5517" - => @"91ab41d255a21e0a3ae40850da2712fb2645bf73" + => @"a0ce5517df8106a9be489e4a09e6afb26176e098" - => @"2024-09-06T09:30:18+01:00" + => @"2024-09-06T13:56:19+01:00" - => @"2" + => @"3" - => @"v16.0.0-2-g91ab41d2" + => @"v16.0.0-3-ga0ce5517" @@ -318,7 +318,7 @@ - => @"2" + => @"3" diff --git a/src/MigrationTools.Telemetery/GetShieldIoWorkItemMetrics.cs b/src/MigrationTools.Telemetery/GetShieldIoWorkItemMetrics.cs index 5056f5932..430bca04e 100644 --- a/src/MigrationTools.Telemetery/GetShieldIoWorkItemMetrics.cs +++ b/src/MigrationTools.Telemetery/GetShieldIoWorkItemMetrics.cs @@ -1,55 +1,196 @@ -using System.Text; +using System; +using System.Text; +using System.Threading.Tasks; +using System.Net.Http; +using System.Collections.Generic; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Newtonsoft.Json; namespace MigrationTools.Telemetery { - public class GetShieldIoWorkItemMetrics { - private static readonly HttpClient client = new HttpClient(); - private readonly ILogger _logger; + private readonly IMemoryCache _cache; + private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(10); // Cache duration - public GetShieldIoWorkItemMetrics(ILogger logger) + public GetShieldIoWorkItemMetrics(ILogger logger, IMemoryCache cache) { _logger = logger; + _cache = cache; } [Function("GetShieldIoWorkItemMetrics_WorkItemTotals")] - public async Task GetShieldIoWorkItemMetrics_WorkItemTotals([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequest req) + public async Task GetShieldIoWorkItemMetrics_WorkItemTotals( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequest req) { _logger.LogInformation("Processing request for Application Insights data."); string appId = Environment.GetEnvironmentVariable("APP_INSIGHTS_APP_ID"); string apiKey = Environment.GetEnvironmentVariable("APP_INSIGHTS_API_KEY"); - string query = @" - customMetrics - | where name == 'work_items_processed_total' - | summarize Total = sum(value) by application_Version"; - var payload = new + if (string.IsNullOrEmpty(appId) || string.IsNullOrEmpty(apiKey)) { - query = query - }; + return new BadRequestObjectResult("Application Insights environment variables not set"); + } - client.DefaultRequestHeaders.Add("x-api-key", apiKey); - var content = new StringContent(JsonConvert.SerializeObject(payload), Encoding.UTF8, "application/json"); - HttpResponseMessage response = await client.PostAsync($"https://api.applicationinsights.io/v1/apps/{appId}/query", content); + string versionPrefix = req.Query["version"]; // Get the 'version' query parameter if provided - if (response.IsSuccessStatusCode) + // Check if cached result exists + if (!_cache.TryGetValue("ai_work_items", out AppInsightsResponse cachedData)) { - string result = await response.Content.ReadAsStringAsync(); - return new OkObjectResult(result); + _logger.LogInformation("Cache miss. Fetching data from Application Insights."); + + string query = @" + customMetrics + | where name == 'work_items_processed_total' + | summarize Total = sum(value) by application_Version"; + + var payload = new + { + query = query + }; + + client.DefaultRequestHeaders.Add("x-api-key", apiKey); + var content = new StringContent(JsonConvert.SerializeObject(payload), Encoding.UTF8, "application/json"); + HttpResponseMessage response = await client.PostAsync($"https://api.applicationinsights.io/v1/apps/{appId}/query", content); + + if (response.IsSuccessStatusCode) + { + string result = await response.Content.ReadAsStringAsync(); + cachedData = JsonConvert.DeserializeObject(result); + + // Cache the result with a defined expiration + _cache.Set("ai_work_items", cachedData, CacheDuration); + } + else + { + return new BadRequestObjectResult($"Error fetching data from Application Insights: {response.StatusCode}"); + } } else { - return new BadRequestObjectResult("Error fetching data from Application Insights"); + _logger.LogInformation("Cache hit. Returning cached data."); } + + // Check if the user requested a specific version prefix or if we should return the sum + double totalWorkItems = 0; + string formattedVersionLabel = "All Versions"; // Default label if no version is provided + if (!string.IsNullOrEmpty(versionPrefix)) + { + // Sum totals for all versions that start with the provided version prefix + totalWorkItems = GetWorkItemTotalForVersionPrefix(cachedData, versionPrefix); + if (totalWorkItems == -1) + { + // No matching versions found, return "No Data" message + return new JsonResult(new + { + schemaVersion = 1, + label = $"Total Work Items ({FormatVersionLabel(versionPrefix)})", + message = "No Data", + color = "orange" + }); + } + + // Format the version label as X.Y.Z (3 digits) + formattedVersionLabel = FormatVersionLabel(versionPrefix); + } + else + { + // Sum all work items for all versions + totalWorkItems = GetTotalWorkItems(cachedData); + if (totalWorkItems == 0) + { + // No data found at all, return "No Data" + return new JsonResult(new + { + schemaVersion = 1, + label = "Total Work Items (All Versions)", + message = "No Data", + color = "orange" + }); + } + } + + // Create the response for Shields.io with total work items + var shieldsIoResponse = new + { + schemaVersion = 1, + label = $"Total Work Items ({formattedVersionLabel})", + message = totalWorkItems.ToString(), + color = "orange" + }; + + return new JsonResult(shieldsIoResponse); } + + // Method to calculate the total work items for all versions + private double GetTotalWorkItems(AppInsightsResponse appInsightsData) + { + double total = 0; + foreach (var row in appInsightsData.Tables[0].Rows) + { + total += Convert.ToDouble(row[1]); // Sum the "Total" column + } + return total; + } + + // Method to get the work item total for all versions that start with the provided version prefix + private double GetWorkItemTotalForVersionPrefix(AppInsightsResponse appInsightsData, string versionPrefix) + { + double total = 0; + bool foundAnyMatchingVersion = false; + + foreach (var row in appInsightsData.Tables[0].Rows) + { + string version = row[0].ToString(); // "application_Version" column + if (version.StartsWith(versionPrefix)) + { + total += Convert.ToDouble(row[1]); // Sum the "Total" column for matching versions + foundAnyMatchingVersion = true; + } + } + + return foundAnyMatchingVersion ? total : -1; // Return -1 if no matching versions were found + } + + // Helper method to format the version label to always have three parts (e.g., X.Y.Z) + private string FormatVersionLabel(string version) + { + var versionParts = version.Split('.'); + if (versionParts.Length == 1) + { + return $"{versionParts[0]}.0.0"; // Pad with .0.0 + } + else if (versionParts.Length == 2) + { + return $"{versionParts[0]}.{versionParts[1]}.0"; // Pad with .0 + } + return version; // Already in X.Y.Z format + } + } + + // Classes to represent the structure of the Application Insights response + public class AppInsightsResponse + { + public List Tables { get; set; } + } + + public class Table + { + public string Name { get; set; } + public List Columns { get; set; } + public List> Rows { get; set; } + } + + public class Column + { + public string Name { get; set; } + public string Type { get; set; } } } diff --git a/src/MigrationTools.Telemetery/Properties/ServiceDependencies/MigrationToolsTelemetery - Web Deploy/profile.arm.json b/src/MigrationTools.Telemetery/Properties/ServiceDependencies/MigrationToolsTelemetery - Web Deploy/profile.arm.json new file mode 100644 index 000000000..fcacd5fc3 --- /dev/null +++ b/src/MigrationTools.Telemetery/Properties/ServiceDependencies/MigrationToolsTelemetery - Web Deploy/profile.arm.json @@ -0,0 +1,113 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_dependencyType": "compute.appService.windows" + }, + "parameters": { + "resourceGroupName": { + "type": "string", + "defaultValue": "MigrationTools", + "metadata": { + "description": "Name of the resource group for the resource. It is recommended to put resources under same resource group for better tracking." + } + }, + "resourceGroupLocation": { + "type": "string", + "defaultValue": "northeurope", + "metadata": { + "description": "Location of the resource group. Resource groups could have different location than resources, however by default we use API versions from latest hybrid profile which support all locations for resource types we support." + } + }, + "resourceName": { + "type": "string", + "defaultValue": "MigrationToolsTelemetery", + "metadata": { + "description": "Name of the main resource to be created by this template." + } + }, + "resourceLocation": { + "type": "string", + "defaultValue": "[parameters('resourceGroupLocation')]", + "metadata": { + "description": "Location of the resource. By default use resource group's location, unless the resource provider is not supported there." + } + } + }, + "variables": { + "appServicePlan_name": "[concat('Plan', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]", + "appServicePlan_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Web/serverFarms/', variables('appServicePlan_name'))]" + }, + "resources": [ + { + "type": "Microsoft.Resources/resourceGroups", + "name": "[parameters('resourceGroupName')]", + "location": "[parameters('resourceGroupLocation')]", + "apiVersion": "2019-10-01" + }, + { + "type": "Microsoft.Resources/deployments", + "name": "[concat(parameters('resourceGroupName'), 'Deployment', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]", + "resourceGroup": "[parameters('resourceGroupName')]", + "apiVersion": "2019-10-01", + "dependsOn": [ + "[parameters('resourceGroupName')]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "location": "[parameters('resourceLocation')]", + "name": "[parameters('resourceName')]", + "type": "Microsoft.Web/sites", + "apiVersion": "2015-08-01", + "tags": { + "[concat('hidden-related:', variables('appServicePlan_ResourceId'))]": "empty" + }, + "dependsOn": [ + "[variables('appServicePlan_ResourceId')]" + ], + "kind": "app", + "properties": { + "name": "[parameters('resourceName')]", + "kind": "app", + "httpsOnly": true, + "reserved": false, + "serverFarmId": "[variables('appServicePlan_ResourceId')]", + "siteConfig": { + "metadata": [ + { + "name": "CURRENT_STACK", + "value": "dotnetcore" + } + ] + } + }, + "identity": { + "type": "SystemAssigned" + } + }, + { + "location": "[parameters('resourceLocation')]", + "name": "[variables('appServicePlan_name')]", + "type": "Microsoft.Web/serverFarms", + "apiVersion": "2015-08-01", + "sku": { + "name": "S1", + "tier": "Standard", + "family": "S", + "size": "S1" + }, + "properties": { + "name": "[variables('appServicePlan_name')]" + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/src/MigrationTools/Services/ActivitySourceProvider.cs b/src/MigrationTools/Services/ActivitySourceProvider.cs index b823e9283..0f2f7af19 100644 --- a/src/MigrationTools/Services/ActivitySourceProvider.cs +++ b/src/MigrationTools/Services/ActivitySourceProvider.cs @@ -23,7 +23,6 @@ public class ActivitySourceProvider { public static readonly string ActivitySourceName = "MigrationTools"; private static string OpenTelemetryConnectionString = "InstrumentationKey=823d0de3-69c9-42ee-b902-de7675f681bc;IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com/;LiveEndpoint=https://westeurope.livediagnostics.monitor.azure.com/;ApplicationId=4dd8f684-2f91-48ac-974f-dc898b686786"; - //private static string MigrationToolAIConnectionString = "InstrumentationKey=2d666f84-b3fb-4dcf-9aad-65de038d2772;IngestionEndpoint=https://northeurope-0.in.applicationinsights.azure.com/;LiveEndpoint=https://northeurope.livediagnostics.monitor.azure.com/;ApplicationId=9146fe72-5c18-48d7-a0f2-8fb891ef1277"; public static ActivitySource ActivitySource { get; private set; }