update confluence fixtures

test/integration/connectors/expected_results/confluence/file_data/1540126.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:55:50.911Z",
     "date_modified": "2023-07-09T12:56:10.564Z",
-    "date_processed": "1734443241.275071",
+    "date_processed": "1734444194.981228",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1540126"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/MFS/1540126.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/MFS/1540126.html",
   "display_name": "Template - Project plan"
 }

test/integration/connectors/expected_results/confluence/file_data/1605859.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:54:40.304Z",
     "date_modified": "2023-07-13T14:13:27.275Z",
-    "date_processed": "1734443239.170847",
+    "date_processed": "1734444192.9658291",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1605859"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/1605859.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/1605859.html",
   "display_name": "test-teamspace"
 }

test/integration/connectors/expected_results/confluence/file_data/1605928.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:54:45.162Z",
     "date_modified": "2023-07-09T12:54:45.162Z",
-    "date_processed": "1734443241.465524",
+    "date_processed": "1734444195.189387",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1605928"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/MFS/1605928.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/MFS/1605928.html",
   "display_name": "Template - Meeting notes"
 }

test/integration/connectors/expected_results/confluence/file_data/1605942.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:54:45.226Z",
     "date_modified": "2023-07-09T12:54:45.226Z",
-    "date_processed": "1734443241.68679",
+    "date_processed": "1734444195.374103",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1605942"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/MFS/1605942.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/MFS/1605942.html",
   "display_name": "Template - Weekly status report"
 }

test/integration/connectors/expected_results/confluence/file_data/1605956.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:54:45.288Z",
     "date_modified": "2023-07-09T12:54:45.288Z",
-    "date_processed": "1734443241.8711371",
+    "date_processed": "1734444195.620225",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1605956"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/MFS/1605956.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/MFS/1605956.html",
   "display_name": "Get the most out of your team space"
 }

test/integration/connectors/expected_results/confluence/file_data/1605989.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-09T12:56:40.842Z",
     "date_modified": "2023-07-09T12:57:59.173Z",
-    "date_processed": "1734443239.539961",
+    "date_processed": "1734444193.309644",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1605989"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/1605989.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/1605989.html",
   "display_name": "Testdoc3"
 }

test/integration/connectors/expected_results/confluence/file_data/1802252.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-11T17:01:39.240Z",
     "date_modified": "2023-07-11T17:01:47.340Z",
-    "date_processed": "1734443239.771659",
+    "date_processed": "1734444193.52543",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1802252"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/1802252.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/1802252.html",
   "display_name": "Testdoc2"
 }

test/integration/connectors/expected_results/confluence/file_data/1867777.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-11T17:01:19.072Z",
     "date_modified": "2023-07-11T17:02:21.081Z",
-    "date_processed": "1734443239.973421",
+    "date_processed": "1734444193.871711",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "1867777"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/1867777.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/1867777.html",
   "display_name": "Testdoc1"
 }

test/integration/connectors/expected_results/confluence/file_data/229477.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-06-30T17:25:25.504Z",
     "date_modified": "2023-06-30T17:25:30.898Z",
-    "date_processed": "1734443240.947705",
+    "date_processed": "1734444194.642272",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "229477"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/MFS/229477.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/MFS/229477.html",
   "display_name": "My first space"
 }

test/integration/connectors/expected_results/confluence/file_data/2589690.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-13T14:27:12.821Z",
     "date_modified": "2023-07-13T14:28:03.779Z",
-    "date_processed": "1734443240.192368",
+    "date_processed": "1734444194.0566368",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "2589690"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/2589690.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/2589690.html",
   "display_name": "Testdoc4"
 }

test/integration/connectors/expected_results/confluence/file_data/2589704.json

@@ -15,7 +15,7 @@
     },
     "date_created": "2023-07-13T14:28:06.310Z",
     "date_modified": "2023-07-14T22:16:58.907Z",
-    "date_processed": "1734443240.414808",
+    "date_processed": "1734444194.2324412",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,6 +24,6 @@
     "document_id": "2589704"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmprfkrc6wg/testteamsp/2589704.html",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmp1twncsjk/testteamsp/2589704.html",
   "display_name": "Testdoc5"
 }

test/integration/connectors/utils/validation/source.py

@@ -214,7 +214,10 @@ def run_all_validations(
     if configs.validate_file_data:
         run_expected_results_validation(
             expected_output_dir=test_output_dir / "file_data",
-            all_file_data=predownload_file_data + postdownload_file_data,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=predownload_file_data,
+                all_postdownload_file_data=postdownload_file_data,
+            ),
             configs=configs,
         )
     download_files = get_files(dir_path=download_dir)
@@ -230,6 +233,19 @@ def run_all_validations(
         )
 
 
+def get_all_file_data(
+    all_postdownload_file_data: list[FileData], all_predownload_file_data: list[FileData]
+) -> list[FileData]:
+    all_file_data = all_postdownload_file_data
+    indexed_file_data = [
+        fd
+        for fd in all_predownload_file_data
+        if fd.identifier not in [f.identifier for f in all_file_data]
+    ]
+    all_file_data += indexed_file_data
+    return all_file_data
+
+
 async def source_connector_validation(
     indexer: Indexer,
     downloader: Downloader,
@@ -271,17 +287,13 @@ async def source_connector_validation(
         )
     else:
         print("Running fixtures update")
-        all_file_data = all_postdownload_file_data
-        indexed_file_data = [
-            fd
-            for fd in all_predownload_file_data
-            if fd.identifier not in [f.identifier for f in all_file_data]
-        ]
-        all_file_data += indexed_file_data
         update_fixtures(
             output_dir=test_output_dir,
             download_dir=download_dir,
-            all_file_data=all_file_data,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
             save_downloads=configs.validate_downloaded_files,
             save_filedata=configs.validate_file_data,
         )