Skip to content

Commit

Permalink
Categorize crash as TCP problem when connection never happens (#999)
Browse files Browse the repository at this point in the history
  • Loading branch information
@premun
premun authored Feb 27, 2023
1 parent 02fe71c commit 5943c94
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 29 deletions.
18 changes: 18 additions & 0 deletions src/Microsoft.DotNet.XHarness.Apple/AppOperations/AppTester.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ namespace Microsoft.DotNet.XHarness.Apple;

public interface IAppTester
{
bool ListenerConnected { get; }

Task<(TestExecutingResult Result, string ResultMessage)> TestApp(
AppBundleInformation appInformation,
TestTargetOs target,
Expand Down Expand Up @@ -66,6 +68,12 @@ public class AppTester : AppRunnerBase, IAppTester
private readonly ILogs _logs;
private readonly IHelpers _helpers;

/// <summary>
/// Denotes whether we had a successful connection over TCP during the run.
/// This is used later to determine if a cause for a failed run is a failing TCP connection.
/// </summary>
public bool ListenerConnected { get; private set; }

public AppTester(
IMlaunchProcessManager processManager,
ISimpleListenerFactory simpleListenerFactory,
Expand Down Expand Up @@ -106,6 +114,7 @@ public AppTester(
var testLog = _logs.Create($"test-{TestTarget.MacCatalyst.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: false);
var appOutputLog = _logs.Create(appInformation.BundleIdentifier + ".log", LogType.ApplicationLog.ToString(), timestamp: true);

ListenerConnected = false;
var (deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
RunMode.MacOS,
log: _mainLog,
Expand Down Expand Up @@ -163,6 +172,7 @@ public AppTester(

var testLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: false);

ListenerConnected = false;
var (deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
runMode,
log: _mainLog,
Expand Down Expand Up @@ -208,6 +218,10 @@ public AppTester(
{
await deviceListener.StopAsync();
}
else if (task.IsCompleted && task.Result)
{
ListenerConnected = true;
}
}, cancellationToken)
.DoNotAwait();

Expand Down Expand Up @@ -431,6 +445,10 @@ private async Task RunDeviceTests(
{
await deviceListener.StopAsync();
}
else if (task.IsCompleted && task.Result)
{
ListenerConnected = true;
}
}, cancellationToken)
.DoNotAwait();

Expand Down
55 changes: 27 additions & 28 deletions src/Microsoft.DotNet.XHarness.Apple/Orchestration/TestOrchestrator.cs
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.Common;
using Microsoft.DotNet.XHarness.Common.CLI;
using Microsoft.DotNet.XHarness.Common.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;

using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.Common;
using Microsoft.DotNet.XHarness.Common.CLI;
using Microsoft.DotNet.XHarness.Common.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;

namespace Microsoft.DotNet.XHarness.Apple;

public interface ITestOrchestrator
Expand All @@ -39,9 +39,9 @@ Task<ExitCode> OrchestrateTest(
CancellationToken cancellationToken);
}

/// <summary>
/// Common ancestor for `test` and `just-test` orchestrators.
/// </summary>
/// <summary>
/// Common ancestor for `test` and `just-test` orchestrators.
/// </summary>
public class TestOrchestrator : BaseOrchestrator, ITestOrchestrator
{
private readonly IAppTesterFactory _appTesterFactory;
Expand Down Expand Up @@ -250,7 +250,7 @@ private async Task<ExitCode> ExecuteApp(
skippedTestClasses: classMethodFilters?.ToArray(),
cancellationToken: cancellationToken);

return ParseResult(testResult, resultMessage);
return ParseResult(testResult, resultMessage, appTester.ListenerConnected);
}

private async Task<ExitCode> ExecuteMacCatalystApp(
Expand Down Expand Up @@ -280,7 +280,7 @@ private async Task<ExitCode> ExecuteMacCatalystApp(
skippedTestClasses: classMethodFilters?.ToArray(),
cancellationToken: cancellationToken);

return ParseResult(testResult, resultMessage);
return ParseResult(testResult, resultMessage, appTester.ListenerConnected);
}

private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool isSimulator)
Expand All @@ -291,7 +291,7 @@ private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool
return _appTesterFactory.Create(communicationChannel, isSimulator, _mainLog, _logs, logCallback);
}

private ExitCode ParseResult(TestExecutingResult testResult, string resultMessage)
private ExitCode ParseResult(TestExecutingResult testResult, string resultMessage, bool listenerConnected)
{
string newLine = Environment.NewLine;
const string checkLogsMessage = "Check logs for more information";
Expand All @@ -303,11 +303,10 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
{
if (_errorKnowledgeBase.IsKnownTestIssue(log, out var issue))
{
if (issue.SuggestedExitCode.HasValue && (ExitCode)issue.SuggestedExitCode.Value == ExitCode.TCP_CONNECTION_FAILED)
if (!listenerConnected && issue.SuggestedExitCode.HasValue && (ExitCode)issue.SuggestedExitCode.Value == ExitCode.TCP_CONNECTION_FAILED)
{
tcpErrorFound = true;
}

else
{
_logger.LogError(message + newLine + issue.HumanMessage);
Expand All @@ -325,8 +324,8 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
_logger.LogError(message + newLine + checkLogsMessage);
}

//TCP errors are encounter all the time but they are not always the cause of the failure
//If the app crashed, TCP_CONNECTION_FAILED and there was not other exit code we will return TCP_CONNECTION_FAILED
// TCP errors are encounter all the time but they are not always the cause of the failure
// If the app crashed, TCP_CONNECTION_FAILED and there was not other exit code we will return TCP_CONNECTION_FAILED
if (defaultExitCode == ExitCode.APP_CRASH && tcpErrorFound)
{
return ExitCode.TCP_CONNECTION_FAILED;
Expand Down Expand Up @@ -367,4 +366,4 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
return ExitCode.GENERAL_FAILURE;
}
}
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.XHarness.iOS.Shared/TestReporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public class TestReporter : ITestReporter

public bool ResultsUseXml => _xmlJargon != XmlResultJargon.Missing;

private bool TestExecutionStarted => _listener.ConnectedTask.Status == TaskStatus.RanToCompletion && _listener.ConnectedTask.Result;
private bool TestExecutionStarted => _listener.ConnectedTask.IsCompleted && _listener.ConnectedTask.Result;

public TestReporter(
IMlaunchProcessManager processManager,
Expand Down
58 changes: 58 additions & 0 deletions tests/Microsoft.DotNet.XHarness.Apple.Tests/Orchestration/TestOrchestratorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,4 +472,62 @@ public async Task OrchestrateMacCatalystTestTest()
_appInstaller.VerifyNoOtherCalls();
_appUninstaller.VerifyNoOtherCalls();
}

[Fact]
public async Task OrchestrateDeviceTestWithFailingTcpTest()
{
// Setup
var testTarget = new TestTargetOs(TestTarget.Device_iOS, "14.2");

var extraArguments = new[] { "--some arg1", "--some arg2" };

_appTester
.Setup(x => x.TestApp(
_appBundleInformation,
testTarget,
_device.Object,
null,
TimeSpan.FromMinutes(30),
It.IsAny<TimeSpan>(),
false,
extraArguments,
It.IsAny<IEnumerable<(string, string)>>(),
It.IsAny<XmlResultJargon>(),
It.IsAny<string[]?>(),
It.IsAny<string[]?>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync((TestExecutingResult.Crashed, "Test execution timed out"))
.Verifiable();

_appTester
.SetupGet(x => x.ListenerConnected)
.Returns(false);

KnownIssue? issue = new KnownIssue("App crashed", null, (int)ExitCode.TCP_CONNECTION_FAILED);
_errorKnowledgeBase
.Setup(x => x.IsKnownTestIssue(It.IsAny<IFileBackedLog>(), out issue))
.Returns(true);

// Act
var result = await _testOrchestrator.OrchestrateTest(
AppPath,
testTarget,
DeviceName,
TimeSpan.FromMinutes(30),
TimeSpan.FromMinutes(3),
CommunicationChannel.UsbTunnel,
XmlResultJargon.xUnit,
Array.Empty<string>(),
Array.Empty<string>(),
includeWirelessDevices: true,
resetSimulator: false,
enableLldb: false,
signalAppEnd: false,
Array.Empty<(string, string)>(),
extraArguments,
new CancellationToken());

// Verify
Assert.Equal(ExitCode.TCP_CONNECTION_FAILED, result);
}
}

0 comments on commit 5943c94

Please sign in to comment.