Update settings migration workflow

.github/workflows/ios-end-to-end-tests-settings-migration.yml

@@ -3,8 +3,12 @@ name: iOS settings migration tests
 concurrency:
   group: ios-end-to-end-tests
   cancel-in-progress: false
+
 permissions:
   contents: read
+  issues: write
+  pull-requests: write
+
 on:
   workflow_dispatch:
   schedule:
@@ -14,113 +18,105 @@ on:
     # Github Actions enabled, so these don't go unnoticed.
     # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
     - cron: '0 0 * * *'
+
 env:
   TEST_DEVICE_UDID: 00008130-0019181022F3803A
+  OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280
+
 jobs:
-  test:
-    name: Settings migration end to end tests
-    runs-on: [self-hosted, macOS, ios-test]
-    env:
-      OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280
+  configure_rust:
+    name: Configure Rust
+    runs-on:
+      - self-hosted
+      - macOS
+      - ios-test
+    continue-on-error: true
     steps:
-      - name: Configure Rust
+      - name: Configure Rust Toolchain
         uses: actions-rs/toolchain@v1.0.6
         with:
           toolchain: stable
           override: true
           target: aarch64-apple-ios
 
-      - name: Uninstall app
+  uninstall_app:
+    name: Uninstall Old App
+    runs-on:
+      - self-hosted
+      - macOS
+      - ios-test
+    needs:
+      - configure_rust
+    steps:
+      - name: Uninstall Old App from Device
         timeout-minutes: 5
         run: ios-deploy --id ${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
 
-      - name: Checkout old repository version
+  checkout_old_version:
+    name: Checkout Old Repository Version
+    runs-on:
+      - self-hosted
+      - macOS
+      - ios-test
+    needs:
+      - uninstall_app
+    steps:
+      - name: Checkout Old Version
         uses: actions/checkout@v4
         with:
           ref: ${{ env.OLD_APP_COMMIT_HASH }}
+      - name: Verify Old Commit Hash
+        run: git rev-parse HEAD
 
-      - name: Change DNS settings on old app version
-        uses: ./.github/actions/ios-end-to-end-tests
-        with:
-          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
-          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
-          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
-          xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
-          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
-
-      - name: Store test report for changing DNS settings
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: test-report-change-dns-settings
-          path: ios/test-report/junit.xml
-
-      - name: Checkout repository to get the current app version
-        uses: actions/checkout@v4
-
-      - name: Verify DNS settings still changed on current app version
-        uses: ./.github/actions/ios-end-to-end-tests
-        if: always()
-        with:
-          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
-          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
-          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
-          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
-          xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
-
-      - name: Store test report for verifying DNS settings
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: test-report-verify-dns-settings
-          path: ios/test-report/junit.xml
-
-      - name: Checkout old repository version
+  checkout_current_version:
+    name: Checkout Current Repository Version
+    runs-on:
+      - self-hosted
+      - macOS
+      - ios-test
+    needs:
+      - configure_rust
+    steps:
+      - name: Checkout Current Version
         uses: actions/checkout@v4
-        with:
-          ref: ${{ env.OLD_APP_COMMIT_HASH }}
+      - name: Verify Current Commit Hash
+        run: git rev-parse HEAD
 
-      - name: Change all other settings on old app version
-        uses: ./.github/actions/ios-end-to-end-tests
-        if: always()
-        with:
-          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
-          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
-          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
-          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
-          xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
-
-      - name: Store test report for changing all settings
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: test-report-change-all-other-settings
-          path: ios/test-report/junit.xml
+  change_dns_settings:
+    name: Change DNS Settings on Old App Version
+    needs:
+      - checkout_old_version
+    uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main
+    with:
+      arg_tests_json_key: "MullvadVPNUITestsChangeDNSSettings"
+    secrets: inherit
 
-      - name: Checkout repository to get the current app version
-        uses: actions/checkout@v4
+  verify_dns_settings:
+    name: Verify DNS Settings on Current App Version
+    needs:
+      - change_dns_settings
+      - checkout_current_version
+    uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main
+    with:
+      arg_tests_json_key: "MullvadVPNUITestsVerifyDNSSettingsChanged"
+    secrets: inherit
 
-      - name: Verify all other settings still changed on current app version
-        uses: ./.github/actions/ios-end-to-end-tests
-        if: always()
-        with:
-          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
-          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
-          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
-          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
-          xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
+  change_other_settings:
+    name: Change All Other Settings on Old App Version
+    needs:
+      - verify_dns_settings
+      - checkout_old_version
+    uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main
+    with:
+      arg_tests_json_key: "MullvadVPNUITestsChangeSettings"
+    secrets: inherit
 
-      - name: Store test report for verifying all other settings
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: test-report-verify-all-other-settings
-          path: ios/test-report/junit.xml
+  verify_other_settings:
+    name: Verify All Other Settings on Current App Version
+    needs:
+      - change_other_settings
+      - checkout_current_version
+    uses: mullvad/mullvadvpn-app/.github/workflows/ios-end-to-end-tests.yml@main
+    with:
+      arg_tests_json_key: "MullvadVPNUITestsVerifySettingsChanged"
+    secrets: inherit

ios/MullvadVPN.xcodeproj/project.pbxproj

@@ -631,10 +631,6 @@
 		8529693A2B4F0238007EAD4C /* TermsOfServicePage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852969392B4F0238007EAD4C /* TermsOfServicePage.swift */; };
 		8529693C2B4F0257007EAD4C /* Alert.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8529693B2B4F0257007EAD4C /* Alert.swift */; };
 		852A26462BA9C9CB006EB9C8 /* DNSSettingsPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */; };
-		852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */; };
-		852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */; };
-		852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */; };
-		852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */; };
 		852D054D2BC3DE3A008578D2 /* APIAccessPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */; };
 		852D054F2BC43DF7008578D2 /* AddAccessMethodPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */; };
 		8532E6872B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */; };
@@ -1951,10 +1947,6 @@
 		852969392B4F0238007EAD4C /* TermsOfServicePage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TermsOfServicePage.swift; sourceTree = "<group>"; };
 		8529693B2B4F0257007EAD4C /* Alert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Alert.swift; sourceTree = "<group>"; };
 		852A26452BA9C9CB006EB9C8 /* DNSSettingsPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DNSSettingsPage.swift; sourceTree = "<group>"; };
-		852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifySettingsChanged.xctestplan; sourceTree = "<group>"; };
-		852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeSettings.xctestplan; sourceTree = "<group>"; };
-		852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan; sourceTree = "<group>"; };
-		852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = MullvadVPNUITestsChangeDNSSettings.xctestplan; sourceTree = "<group>"; };
 		852D054C2BC3DE3A008578D2 /* APIAccessPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = APIAccessPage.swift; sourceTree = "<group>"; };
 		852D054E2BC43DF7008578D2 /* AddAccessMethodPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddAccessMethodPage.swift; sourceTree = "<group>"; };
 		8532E6862B8CCED600ACECD1 /* ProblemReportSubmittedPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProblemReportSubmittedPage.swift; sourceTree = "<group>"; };
@@ -3851,11 +3843,7 @@
 				7A83C4002A55B81A00DFB83A /* MullvadVPNCI.xctestplan */,
 				7A02D4EA2A9CEC7A00C19E31 /* MullvadVPNScreenshots.xctestplan */,
 				852969302B4D9E70007EAD4C /* MullvadVPNUITestsAll.xctestplan */,
-				852BC6722BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan */,
-				852BC66D2BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan */,
 				85006A8E2B73EF67004AD8FB /* MullvadVPNUITestsSmoke.xctestplan */,
-				852BC66E2BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan */,
-				852BC66C2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan */,
 			);
 			path = TestPlans;
 			sourceTree = "<group>";
@@ -5046,10 +5034,6 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				852BC66F2BAB44F500A47558 /* MullvadVPNUITestsVerifySettingsChanged.xctestplan in Resources */,
-				852BC6712BAB44F500A47558 /* MullvadVPNUITestsVerifyDNSSettingsChanged.xctestplan in Resources */,
-				852BC6702BAB44F500A47558 /* MullvadVPNUITestsChangeSettings.xctestplan in Resources */,
-				852BC6732BAB450B00A47558 /* MullvadVPNUITestsChangeDNSSettings.xctestplan in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

ios/MullvadVPNUITests/tests.json

@@ -1,19 +1,30 @@
 {
-    "tests": {
-        "nightly": [
-            "AccountTests",
-            "ConnectivityTests",
-            "CustomListsTests",
-            "RelayTests",
-            "SettingsTests"
-        ],
-        "pr-merge-to-main": [
-            "AccountTests/testLogin",
-            "AccountTests/testCreateAccount"
-        ],
-        "api-tests": [
-            "AccountTests"
-        ]
-    }
+   "tests":{
+      "nightly":[
+         "AccountTests",
+         "ConnectivityTests",
+         "CustomListsTests",
+         "RelayTests",
+         "SettingsTests"
+      ],
+      "pr-merge-to-main":[
+         "AccountTests/testLogin",
+         "AccountTests/testCreateAccount"
+      ],
+      "api-tests":[
+         "AccountTests"
+      ],
+      "MullvadVPNUITestsChangeDNSSettings":[
+         "SettingsMigrationTests/testChangeCustomDNSSettings"
+      ],
+      "MullvadVPNUITestsVerifyDNSSettingsChanged":[
+         "SettingsMigrationTests/testVerifyCustomDNSSettingsStillChanged"
+      ],
+      "MullvadVPNUITestsChangeSettings":[
+         "SettingsMigrationTests/testChangeVPNSettings"
+      ],
+      "MullvadVPNUITestsVerifySettingsChanged":[
+         "SettingsMigrationTests/testVerifySettingsStillChanged"
+      ]
+   }
 }
-

ios/TestPlans/MullvadVPNUITestsAll.xctestplan

@@ -22,8 +22,7 @@
         "LoggedInWithTimeUITestCase",
         "LoggedInWithoutTimeUITestCase",
         "LoggedOutUITestCase",
-        "ScreenshotTests",
-        "SettingsMigrationTests"
+        "ScreenshotTests"
       ],
       "target" : {
         "containerPath" : "container:MullvadVPN.xcodeproj",