Categorize crash as TCP problem when connection never happens

src/Microsoft.DotNet.XHarness.Apple/AppOperations/AppTester.cs

@@ -22,6 +22,8 @@ namespace Microsoft.DotNet.XHarness.Apple;
 
 public interface IAppTester
 {
+    bool ListenerConnected { get; }
+
     Task<(TestExecutingResult Result, string ResultMessage)> TestApp(
         AppBundleInformation appInformation,
         TestTargetOs target,
@@ -66,6 +68,12 @@ public class AppTester : AppRunnerBase, IAppTester
     private readonly ILogs _logs;
     private readonly IHelpers _helpers;
 
+    /// <summary>
+    /// Denotes whether we had a successful connection over TCP during the run.
+    /// This is used later to determine if a cause for a failed run is a failing TCP connection.
+    /// </summary>
+    public bool ListenerConnected { get; private set; }
+
     public AppTester(
         IMlaunchProcessManager processManager,
         ISimpleListenerFactory simpleListenerFactory,
@@ -106,6 +114,7 @@ public AppTester(
         var testLog = _logs.Create($"test-{TestTarget.MacCatalyst.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: false);
         var appOutputLog = _logs.Create(appInformation.BundleIdentifier + ".log", LogType.ApplicationLog.ToString(), timestamp: true);
 
+        ListenerConnected = false;
         var (deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
             RunMode.MacOS,
             log: _mainLog,
@@ -163,6 +172,7 @@ public AppTester(
 
         var testLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: false);
 
+        ListenerConnected = false;
         var (deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
             runMode,
             log: _mainLog,
@@ -208,6 +218,10 @@ public AppTester(
                     {
                         await deviceListener.StopAsync();
                     }
+                    else if (task.IsCompleted && task.Result)
+                    {
+                        ListenerConnected = true;
+                    }
                 }, cancellationToken)
                 .DoNotAwait();
 
@@ -431,6 +445,10 @@ private async Task RunDeviceTests(
                 {
                     await deviceListener.StopAsync();
                 }
+                else if (task.IsCompleted && task.Result)
+                {
+                    ListenerConnected = true;
+                }
             }, cancellationToken)
             .DoNotAwait();
 

src/Microsoft.DotNet.XHarness.Apple/Orchestration/TestOrchestrator.cs

@@ -1,21 +1,21 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using System;
-using System.Collections.Generic;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.DotNet.XHarness.Common;
-using Microsoft.DotNet.XHarness.Common.CLI;
-using Microsoft.DotNet.XHarness.Common.Logging;
-using Microsoft.DotNet.XHarness.iOS.Shared;
-using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
-using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
-using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
-
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.XHarness.Common;
+using Microsoft.DotNet.XHarness.Common.CLI;
+using Microsoft.DotNet.XHarness.Common.Logging;
+using Microsoft.DotNet.XHarness.iOS.Shared;
+using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
+using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
+using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
+
 namespace Microsoft.DotNet.XHarness.Apple;
 
 public interface ITestOrchestrator
@@ -39,9 +39,9 @@ Task<ExitCode> OrchestrateTest(
         CancellationToken cancellationToken);
 }
 
-/// <summary>
-/// Common ancestor for `test` and `just-test` orchestrators.
-/// </summary>
+/// <summary>
+/// Common ancestor for `test` and `just-test` orchestrators.
+/// </summary>
 public class TestOrchestrator : BaseOrchestrator, ITestOrchestrator
 {
     private readonly IAppTesterFactory _appTesterFactory;
@@ -250,7 +250,7 @@ private async Task<ExitCode> ExecuteApp(
             skippedTestClasses: classMethodFilters?.ToArray(),
             cancellationToken: cancellationToken);
 
-        return ParseResult(testResult, resultMessage);
+        return ParseResult(testResult, resultMessage, appTester.ListenerConnected);
     }
 
     private async Task<ExitCode> ExecuteMacCatalystApp(
@@ -280,7 +280,7 @@ private async Task<ExitCode> ExecuteMacCatalystApp(
             skippedTestClasses: classMethodFilters?.ToArray(),
             cancellationToken: cancellationToken);
 
-        return ParseResult(testResult, resultMessage);
+        return ParseResult(testResult, resultMessage, appTester.ListenerConnected);
     }
 
     private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool isSimulator)
@@ -291,7 +291,7 @@ private IAppTester GetAppTester(CommunicationChannel communicationChannel, bool
         return _appTesterFactory.Create(communicationChannel, isSimulator, _mainLog, _logs, logCallback);
     }
 
-    private ExitCode ParseResult(TestExecutingResult testResult, string resultMessage)
+    private ExitCode ParseResult(TestExecutingResult testResult, string resultMessage, bool listenerConnected)
     {
         string newLine = Environment.NewLine;
         const string checkLogsMessage = "Check logs for more information";
@@ -303,11 +303,10 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
             {
                 if (_errorKnowledgeBase.IsKnownTestIssue(log, out var issue))
                 {
-                    if (issue.SuggestedExitCode.HasValue && (ExitCode)issue.SuggestedExitCode.Value == ExitCode.TCP_CONNECTION_FAILED)
+                    if (!listenerConnected && issue.SuggestedExitCode.HasValue && (ExitCode)issue.SuggestedExitCode.Value == ExitCode.TCP_CONNECTION_FAILED)
                     {
                         tcpErrorFound = true;
                     }
-
                     else
                     {
                         _logger.LogError(message + newLine + issue.HumanMessage);
@@ -325,8 +324,8 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
                 _logger.LogError(message + newLine + checkLogsMessage);
             }
 
-            //TCP errors are encounter all the time but they are not always the cause of the failure
-            //If the app crashed, TCP_CONNECTION_FAILED and there was not other exit code we will return TCP_CONNECTION_FAILED
+            // TCP errors are encounter all the time but they are not always the cause of the failure
+            // If the app crashed, TCP_CONNECTION_FAILED and there was not other exit code we will return TCP_CONNECTION_FAILED
             if (defaultExitCode == ExitCode.APP_CRASH && tcpErrorFound)
             {
                 return ExitCode.TCP_CONNECTION_FAILED;
@@ -367,4 +366,4 @@ ExitCode LogProblem(string message, ExitCode defaultExitCode)
                 return ExitCode.GENERAL_FAILURE;
         }
     }
-}
+}

src/Microsoft.DotNet.XHarness.iOS.Shared/TestReporter.cs

@@ -76,7 +76,7 @@ public class TestReporter : ITestReporter
 
     public bool ResultsUseXml => _xmlJargon != XmlResultJargon.Missing;
 
-    private bool TestExecutionStarted => _listener.ConnectedTask.Status == TaskStatus.RanToCompletion && _listener.ConnectedTask.Result;
+    private bool TestExecutionStarted => _listener.ConnectedTask.IsCompleted && _listener.ConnectedTask.Result;
 
     public TestReporter(
         IMlaunchProcessManager processManager,

tests/Microsoft.DotNet.XHarness.Apple.Tests/Orchestration/TestOrchestratorTests.cs

@@ -472,4 +472,62 @@ public async Task OrchestrateMacCatalystTestTest()
         _appInstaller.VerifyNoOtherCalls();
         _appUninstaller.VerifyNoOtherCalls();
     }
+
+    [Fact]
+    public async Task OrchestrateDeviceTestWithFailingTcpTest()
+    {
+        // Setup
+        var testTarget = new TestTargetOs(TestTarget.Device_iOS, "14.2");
+
+        var extraArguments = new[] { "--some arg1", "--some arg2" };
+
+        _appTester
+            .Setup(x => x.TestApp(
+                _appBundleInformation,
+                testTarget,
+                _device.Object,
+                null,
+                TimeSpan.FromMinutes(30),
+                It.IsAny<TimeSpan>(),
+                false,
+                extraArguments,
+                It.IsAny<IEnumerable<(string, string)>>(),
+                It.IsAny<XmlResultJargon>(),
+                It.IsAny<string[]?>(),
+                It.IsAny<string[]?>(),
+                It.IsAny<CancellationToken>()))
+            .ReturnsAsync((TestExecutingResult.Crashed, "Test execution timed out"))
+            .Verifiable();
+
+        _appTester
+            .SetupGet(x => x.ListenerConnected)
+            .Returns(false);
+
+        KnownIssue? issue = new KnownIssue("App crashed", null, (int)ExitCode.TCP_CONNECTION_FAILED);
+        _errorKnowledgeBase
+            .Setup(x => x.IsKnownTestIssue(It.IsAny<IFileBackedLog>(), out issue))
+            .Returns(true);
+
+        // Act
+        var result = await _testOrchestrator.OrchestrateTest(
+            AppPath,
+            testTarget,
+            DeviceName,
+            TimeSpan.FromMinutes(30),
+            TimeSpan.FromMinutes(3),
+            CommunicationChannel.UsbTunnel,
+            XmlResultJargon.xUnit,
+            Array.Empty<string>(),
+            Array.Empty<string>(),
+            includeWirelessDevices: true,
+            resetSimulator: false,
+            enableLldb: false,
+            signalAppEnd: false,
+            Array.Empty<(string, string)>(),
+            extraArguments,
+            new CancellationToken());
+
+        // Verify
+        Assert.Equal(ExitCode.TCP_CONNECTION_FAILED, result);
+    }
 }