-
Notifications
You must be signed in to change notification settings - Fork 1
/
run_raster_highest.py
34 lines (25 loc) · 1.31 KB
/
run_raster_highest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
import subprocess
import shlex
import pdgstaging # to get filepaths
import PRODUCTION_IWP_CONFIG
IWP_CONFIG = PRODUCTION_IWP_CONFIG.IWP_CONFIG
def run_raster_higest():
# trying 20k with 5 CPU nodes was too much!
BATCH_SIZE = 8_000
print(f"Collecting all STAGED files from `{IWP_CONFIG['dir_staged']}`...")
stager = pdgstaging.TileStager(IWP_CONFIG, check_footprints=False)
stager.tiles.add_base_dir('output_of_staging', IWP_CONFIG['dir_staged'], '.gpkg')
staged_paths = stager.tiles.get_filenames_from_dir( base_dir = 'output_of_staging' )
print("Total files:", len(staged_paths))
print("Total batches (runs required):", len(staged_paths) // BATCH_SIZE)
for i in range(0, len(staged_paths), BATCH_SIZE):
start_idx = i
end_idx = i+BATCH_SIZE
print(f"Starting files index {i} to {i+BATCH_SIZE}")
subprocess.run(f"python IN_PROGRESS_VIZ_WORKFLOW.py -s {start_idx} -e {end_idx}", check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # capture_output=True,
# rsync results to server, after each itr :)
# Warning: deletes source files after rsync.
subprocess.run(shlex.split("/u/kastanday/viz/viz-workflow/utilities/rsync_merge_raster_to_scratch.py"), shell=True)
print("Completed all batches. Exiting.")
if __name__ == '__main__':
run_raster_higest()