Use an environment variable to record failures.

element-hq · Sep 19, 2024 · c6c82a4 · c6c82a4
1 parent 3ae050b
commit c6c82a4
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 27 deletions.
diff --git a/PreviewTests/Sources/PreviewTests.swift b/PreviewTests/Sources/PreviewTests.swift
@@ -11,18 +11,19 @@ import XCTest
 @testable import ElementX
 @testable import SnapshotTesting
 
-#if canImport(AccessibilitySnapshot)
-import AccessibilitySnapshot
-#endif
-
 class PreviewTests: XCTestCase {
     private let deviceConfig: ViewImageConfig = .iPhoneX
-    private var simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
-    private var requiredOSVersion = (major: 18, minor: 0)
+    private let simulatorDevice: String? = "iPhone14,6" // iPhone SE 3rd Generation
+    private let requiredOSVersion = (major: 18, minor: 0)
     private let snapshotDevices = ["iPhone 16", "iPad"]
+    private var recordMode: SnapshotTestingConfiguration.Record = .missing
 
     override func setUp() {
         super.setUp()
+
+        if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
+            recordMode = .failed
+        }
 
         checkEnvironments()
         UIView.setAnimationsEnabled(false)
@@ -102,7 +103,7 @@ class PreviewTests: XCTestCase {
             .fixedSize(horizontal: false, vertical: true)
         )
 
-        let failure = withSnapshotTesting(record: .missing) {
+        return withSnapshotTesting(record: recordMode) {
             verifySnapshot(of: matchingView,
                            as: .prefireImage(precision: { precision },
                                              perceptualPrecision: { perceptualPrecision },
@@ -112,16 +113,6 @@ class PreviewTests: XCTestCase {
                            named: name,
                            testName: testName)
         }
-
-        #if canImport(AccessibilitySnapshot)
-        let vc = UIHostingController(rootView: matchingView)
-        vc.view.frame = UIScreen.main.bounds
-        assertSnapshot(matching: vc,
-                       as: .wait(for: delay, on: .accessibilityImage(showActivationPoints: .always)),
-                       named: name.flatMap { $0 + ".accessibility" },
-                       testName: testName)
-        #endif
-        return failure
     }
 }
 

diff --git a/PreviewTests/SupportingFiles/PreviewTests.xctestplan b/PreviewTests/SupportingFiles/PreviewTests.xctestplan
@@ -31,6 +31,11 @@
       {
         "key" : "IS_RUNNING_UNIT_TESTS",
         "value" : "1"
+      },
+      {
+        "enabled" : false,
+        "key" : "RECORD_FAILURES",
+        "value" : "true"
       }
     ],
     "testExecutionOrdering" : "random",

diff --git a/UITests/Sources/Application.swift b/UITests/Sources/Application.swift
@@ -10,6 +10,10 @@ import XCTest
 
 enum Application {
     static func launch(_ identifier: UITestsScreenIdentifier, disableTimelineAccessibility: Bool = true) -> XCUIApplication {
+        if ProcessInfo().environment["RECORD_FAILURES"].map(Bool.init) == true {
+            XCUIApplication.recordMode = .failed
+        }
+
         checkEnvironments()
 
         let app = XCUIApplication()
@@ -47,6 +51,8 @@ enum Application {
 }
 
 extension XCUIApplication {
+    static var recordMode: SnapshotTestingConfiguration.Record = .missing
+
     @MainActor
     /// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
     /// - Parameter identifier: Identifier of the UI test screen
@@ -69,17 +75,17 @@ extension XCUIApplication {
             snapshot = snapshot.inset(by: insets)
         }
 
-        let failure = verifySnapshot(of: snapshot,
-                                     as: .image(precision: precision,
-                                                perceptualPrecision: 0.98,
-                                                scale: nil),
-                                     // use any kind of suffix here to snapshot the same file multiple times and avoid countering on the library side
-                                     named: "UI",
-                                     testName: snapshotName)
+        let failure = withSnapshotTesting(record: Self.recordMode) {
+            verifySnapshot(of: snapshot,
+                           as: .image(precision: precision,
+                                      perceptualPrecision: 0.98,
+                                      scale: nil),
+                           // use any kind of suffix here to snapshot the same file multiple times and avoid countering on the library side
+                           named: "UI",
+                           testName: snapshotName)
+        }
 
-        if let failure,
-           !failure.contains("No reference was found on disk."),
-           !failure.contains("to test against the newly-recorded snapshot") {
+        if let failure {
             XCTFail(failure)
         }
     }

diff --git a/UITests/SupportingFiles/UITests.xctestplan b/UITests/SupportingFiles/UITests.xctestplan
@@ -27,6 +27,11 @@
       {
         "key" : "SNAPSHOT_ARTIFACTS",
         "value" : "\/tmp\/__FailedScreenshots__"
+      },
+      {
+        "enabled" : false,
+        "key" : "RECORD_FAILURES",
+        "value" : "true"
       }
     ],
     "locationScenario" : {