Skip to content

Commit

Permalink
fix rate
Browse files Browse the repository at this point in the history
  • Loading branch information
JordanLaserGit committed Jan 5, 2024
1 parent 431adc2 commit 95fd2c6
Showing 1 changed file with 4 additions and 8 deletions.
12 changes: 4 additions & 8 deletions forcingprocessor/src/forcingprocessor/forcingprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,14 +242,12 @@ def multiprocess_write(data,t_ax,catchments,nprocs,output_bucket,out_path,ii_app
ntasked = len(np.nonzero(catchments_per_proc)[0])
if nprocs > ntasked:
if ii_verbose: print(f'Not enough work for {nprocs} requested processes, downsizing to {ntasked}')
nprocs = ntasked

ncatchments = len(catchments)
out_path_list = []
append_list = []
print_list = []
bucket_list = []
nprocess_list = []
worker_time_list = []
worker_data_list = []
worker_catchment_list = []
Expand All @@ -276,7 +274,6 @@ def multiprocess_write(data,t_ax,catchments,nprocs,output_bucket,out_path,ii_app
append_list.append(ii_append)
print_list.append(ii_print)
worker_time_list.append(t_ax)
nprocess_list.append(nprocs)
bucket_list.append(output_bucket)

worker_catchments = {}
Expand All @@ -297,7 +294,6 @@ def multiprocess_write(data,t_ax,catchments,nprocs,output_bucket,out_path,ii_app
out_path_list,
append_list,
print_list,
nprocess_list
):
ids.append(results[0])
dfs.append(results[1])
Expand All @@ -322,8 +318,8 @@ def write_data(
bucket,
out_path,
ii_append,
ii_print,
nprocesss
ii_print

):
s3_client = boto3.session.Session().client("s3")

Expand Down Expand Up @@ -395,9 +391,9 @@ def write_data(
if ii_print and ii_verbose:
if (j + 1) % write_int == 0 or j == nfiles - 1:
t_accum = time.perf_counter() - t00
rate = ((j+1)/t_accum)
rate = ((j+1)*ntasked/t_accum)
bytes2bits = 8
bandwidth_Mbps = rate * file_size_MB *nprocesss * bytes2bits
bandwidth_Mbps = rate * file_size_MB * ntasked * bytes2bits
estimate_total_time = nfiles / rate
report_usage()
msg = f"\n{(j+1)*ntasked} files written out of {nfiles*ntasked}\n"
Expand Down

0 comments on commit 95fd2c6

Please sign in to comment.