Seemed to run without error but no website created.

[rljacob]

Seemed to run without error. But website wasn't created.

Logs are in /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/post/scripts

[default]
input = /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186
output = /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186
case = WCYCLref88ee186
www = /lcrc/group/e3sm/public_html/diagnostic_output/jacob/E3SM/
partition = compute
environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"

[climo]
active = True
walltime = "1:00:00"
years = "0011:0050:40",

  [[ atm_monthly_180x360_aave ]]
  input_subdir = "atm/hist"
  input_files = "eam.h0"
  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
  frequency = "monthly"

  [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
  input_subdir = "atm/hist"
  input_files = "eam.h4"
  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
  vars = "PRECT"
  frequency = "diurnal_8xdaily"

[ts]
active = True
walltime = "0:30:00"
years = "0011:0050:40",

  [[ atm_monthly_180x360_aave ]]
  input_subdir = "atm/hist"
  input_files = "eam.h0"
  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
  frequency = "monthly"
  #vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,IC
EFRAC,LANDFRAC,OCNFRAC,Mass_bc,Mass_dst,Mass_mom,Mass_ncl,Mass_pom,Mass_so4,Mass_soa,AODALL,AODBC,AODDUST,AODPOM,AODSO4,AODSOA,AODSS
,AODVIS,PS"
  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICE
FRAC,LANDFRAC,OCNFRAC,AODALL,AODBC,AODDUST,AODPOM,AODSO4,AODSOA,AODSS,AODVIS,PS"

  [[ atm_daily_180x360_aave ]]
  input_subdir = "atm/hist"
  input_files = "eam.h1"
  mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
  frequency = "daily"
  vars = "PRECT"

  [[ atm_monthly_glb ]]
  input_subdir = "atm/hist"
  input_files = "eam.h0"
  mapping_file = "glb"
  frequency = "monthly"

  [[ land_monthly ]]
  input_subdir = "lnd/hist"
  input_files = "elm.h0"
  mapping_file = /home/ac.ambradl/compy-goodies/cplcaas/current/map_r05_to_cmip6_180x360_aave.20230414.nc
  frequency = "monthly"
  vars = "FSH,RH2M"
  extra_vars = "landfrac"

  [[ rof_monthly ]]
  input_subdir = "rof/hist"
  input_files = "mosart.h0"
  mapping_file = ""
  frequency = "monthly"
  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
  extra_vars = 'areatotal2'

[e3sm_diags]
active = True
years = "0011:0050:40",
ts_num_years = 40
ref_start_yr = 1979
ref_final_yr = 2016

  [[ atm_monthly_180x360_aave ]]
  short_name = 'WCYCLref88ee186'
  grid = '180x360_aave'
  sets = 'lat_lon','zonal_mean_xy','zonal_mean_2d','polar','cosp_histogram','meridional_mean_2d','enso_diags','qbo','diurnal_cycle',
'annual_cycle_zonal_mean','streamflow','zonal_mean_2d_stratosphere'
  reference_data_path = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/climatology'
  obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
  climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
  climo_diurnal_frequency = "diurnal_8xdaily"
  output_format_subplot = "pdf",

  [[ atm_monthly_180x360_aave_mvm ]]
  grid = '180x360_aave'
  climo_subsection = "atm_monthly_180x360_aave"
  diff_title = "Difference"
  ref_name = "v2_1.LR.piControl"
  ref_start_yr = 101
  ref_final_yr = 200
  ref_years = "101-200",
  reference_data_path = "/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.piControl/post/atm/180x360_aave/clim"
  run_type = "model_vs_model"
  short_ref_name = "v2_1.LR.piControl"
  swap_test_ref = False
  tag = "model_vs_model"
  ts_num_years_ref = 40
  ts_subsection = "atm_monthly_180x360_aave"
  years = "0011-0050",

[mpas_analysis]
active = True
walltime = "4:00:00"
parallelTaskCount = 6
anomalyRefYear = 0011
ts_years = "0011-0050",
enso_years = "0011-0050",
climo_years = "0011-0050",
mesh = "EC30to60E2r2"
shortTermArchive = False

[global_time_series]
atmosphere_only = False
input_subdir = 'run'
active = True
debug = False
years = "0011-0050",
ts_num_years = 40
figstr = "WCYCLref88ee186"
moc_file = "mocTimeSeries_0011-0050.nc"
experiment_name = "WCYCLref88ee186"
ts_years = "0011-0050",
climo_years = "0011-0050"

[forsyth2]

Debugging steps:

$ cd /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/post/scripts

$  grep "OK" *status
climo_atm_monthly_180x360_aave_0011-0050.status:OK
climo_atm_monthly_diurnal_8xdaily_180x360_aave_0011-0050.status:OK
ts_atm_daily_180x360_aave_0011-0050-0040.status:OK
ts_atm_monthly_180x360_aave_0011-0050-0040.status:OK
ts_atm_monthly_glb_0011-0050-0040.status:OK
ts_rof_monthly_0011-0050-0040.status:OK

$ grep -v "OK" *status
e3sm_diags_atm_monthly_180x360_aave_model_vs_obs_0011-0050.status:WAITING 366493
e3sm_diags_atm_monthly_180x360_aave_mvm_model_vs_model_0011-0050_vs_0101-0200.status:WAITING 366494
global_time_series_0011-0050.status:WAITING 366496
mpas_analysis_ts_0011-0050_climo_0011-0050.status:ERROR (2)
ts_land_monthly_0011-0050-0040.status:ERROR (1)

$ cat ts_land_monthly_0011-0050-0040.o385112
ts_only
ncclimo: ERROR Unable to find specified regrid map /home/ac.ambradl/compy-goodies/cplcaas/current/map_r05_to_cmip6_180x360_aave.20230414.nc
ncclimo: HINT Supply the full path-name for the regridding map

$ ls /home/ac.ambradl/compy-goodies/cplcaas/current/map_r05_to_cmip6_180x360_aave.20230414.nc
ls: cannot access '/home/ac.ambradl/compy-goodies/cplcaas/current/map_r05_to_cmip6_180x360_aave.20230414.nc': Permission denied
# I can't access this file; does it exist?

$ tail -n 27 mpas_analysis_ts_0011-0050_climo_0011-0050.o385114
Traceback (most recent call last):
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/mpas_analysis/__main__.py", line 440, in add_task_and_subtasks
    analysisTask.setup_and_check()
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/mpas_analysis/shared/time_series/mpas_time_series_task.py", line 154, in setup_and_check
    super(MpasTimeSeriesTask, self).setup_and_check()
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/mpas_analysis/shared/analysis_task.py", line 208, in setup_and_check
    self.namelist = NameList(namelistFileName)
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/mpas_analysis/shared/io/namelist_streams_interface.py", line 104, in __init__
    self.nml = convert_namelist_to_dict(fname)
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/mpas_analysis/shared/io/namelist_streams_interface.py", line 48, in convert_namelist_to_dict
    nml = f90nml.read(fname).todict()
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/f90nml/__init__.py", line 31, in read
    return parser.read(nml_path)
  File "/lcrc/soft/climate/e3sm-unified/base/envs/e3sm_unified_1.8.0_chrysalis/lib/python3.10/site-packages/f90nml/parser.py", line 279, in read
    nml_file = open(nml_fname, 'r') if nml_is_path else nml_fname
FileNotFoundError: [Errno 2] No such file or directory: '/lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/run/mpassi_in'
Warning: mpasTimeSeriesSeaIce failed during check and will not be run
Warning: prerequisite of timeSeriesSeaIceAreaVol failed during check, so this task will not be run

66 tasks and subtasks failed during setup.


Log files for executed tasks can be found in /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/post/scripts/../analysis/mpas_analysis/ts_0011-0050_climo_0011-0050/logs
Total setup time: 0:00:05.93
Total run time: 0:00:11.66
Generating webpage for viewing results...
Done.

$ ls /lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/run/mpassi_in
ls: cannot access '/lcrc/group/e3sm/jacob/scratch/chrys/archive/WCYCLref88ee186/run/mpassi_in': No such file or directory

It looks like a ts task and the mpas_analysis task failed. E3SM Diags depends on ts; Global time series depends on ts and mpas_analysis. So, nothing that produces plots succeeded. It looks like there are some missing files?

[rljacob]

Ok I got almost everything to work except for MPAS global time series. The error is:

Finding common directory ...
Common directory:
Scanning files ...
mpaso.glb.001101-005012.nc
Setting reference time units to days since 0011-01-01 00:00:00
Setting reference time units to days since 0011-01-01 00:00:00
glb.xml written
Copy moc file
cp: cannot stat 'mocTimeSeries_0011-0050.nc': No such file or directory

That file does exist but its in post/analysis/mpas_analysis/ts_0011-0050_climo_0011-0050/timeseries/moc . Should it be somewhere else?

[rljacob]

I edited the cfg to give the full path to mocTimeSeries and that worked! Its in https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/jacob/E3SM/WCYCLref88ee186.

One thing I noticed is the mpas-analysis web site wasn't copied to the above dir. I had to tar it up and move it there. Is that expected?

[forsyth2]

Ok great. Yeah, the the [global_time_series] moc_fileparameter is useful for that (e.g., https://github.com/E3SM-Project/zppy/blob/main/tests/integration/generated/test_complete_run_chrysalis.cfg#L144).

The mpas-analysis web site should show up in that same directory, no tar required. That said, if you handled the input differently than the standard workflow, that could explain why you needed to copy it yourself.

The HTML copy line is https://github.com/E3SM-Project/zppy/blob/main/zppy/templates/mpas_analysis.bash#L347: rsync -a --delete ${identifier}/html/ ${www}/${case}/mpas_analysis/${identifier}/, where identifier=ts_${tsY1}-${tsY2}_climo_${climY1}-${climY2}. So, I suppose if the website was generated in a different origin directory, this line would just copy over nothing.

[rljacob]

I did another zppy run on a second case and set all the files up before hand. This time mpas-analysis copied the html correctly.

I'm going to close this since the initial issue is solved.

[xylar]

Thanks @milenaveneziani and @forsyth2 for addressing this. Sorry I wasn't able to jump in in time to be helpful.

@rljacob, if there are remaining issues with how zppy handles short-term archiving with MPAS-Analysis, could you make a GitHub issue with those? If there are issues with the required files that MPAS-Analysis needs not being documented well within zppy, that would also be something to raise as a GitHub issue. I have my hands pretty full and haven't spent much time working on zppy myself but I'm aware that there could be some shortcomings related to MPAS-Analysis. My hope is to address these in a more comprehensive way as we transition from MPAS-Analysis to Polaris, but we should make sure the existing functionality and documentation are adequate until that transition happens.