From 235e0bf346677ed46f21eefcc1d3bf47ef9aa506 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Wed, 18 Dec 2024 09:41:32 -0700
Subject: [PATCH 01/10] Bugfix #2830 develop fix missing log output (#2841)

* Per #2830, skip closing of log handlers for METplusConfig objects that are created for copying values for process list instances so they are not closed before the end of the run.

* remove some output directories after tests are run
---
 internal/tests/pytests/util/run_util/test_run_util.py | 10 ++++++++++
 metplus/util/config_metplus.py                        |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py
index 9b95e891e..62d9e8149 100644
--- a/internal/tests/pytests/util/run_util/test_run_util.py
+++ b/internal/tests/pytests/util/run_util/test_run_util.py
@@ -51,6 +51,10 @@
     'INPUT_THRESH',
 ]
 
+def remove_output_base(config):
+    config_output_base = config.getdir("OUTPUT_BASE")
+    if config_output_base and os.path.exists(config_output_base):
+        ru.shutil.rmtree(config_output_base)
 
 def get_run_util_configs(conf_name):
     script_dir = os.path.dirname(__file__)
@@ -130,6 +134,7 @@ def test_pre_run_setup():
     expected_stage = os.path.join(actual.get('config', 'OUTPUT_BASE'), 'stage')
     assert actual.get('config', 'STAGING_DIR') == expected_stage
     assert actual.get('user_env_vars', 'GODS_OF_WEATHER') == 'Indra_Thor_Zeus'
+    remove_output_base(actual)
 
 
 @pytest.mark.util
@@ -139,6 +144,7 @@ def test_pre_run_setup_env_vars():
         actual = ru.pre_run_setup(conf_inputs)
     assert actual.env['MY_ENV_VAR'] == '42'
     assert actual.get('config', 'OMP_NUM_THREADS') == '4'
+    remove_output_base(actual)
 
 
 @pytest.mark.util
@@ -262,6 +268,7 @@ def test_run_metplus(capfd, config_dict, expected, check_err):
     else:
         assert err == ''
 
+    remove_output_base(config)
 
 @pytest.mark.parametrize(
     "side_effect,return_value,check_err",
@@ -285,6 +292,8 @@ def test_run_metplus_errors(capfd, side_effect, return_value, check_err):
         else:
             assert err == check_err 
 
+        remove_output_base(config)
+
 
 @pytest.mark.util
 def test_get_wrapper_instance(metplus_config):
@@ -308,6 +317,7 @@ def test_get_wrapper_instance_raises(capfd, side_effect, check_err):
     assert actual == None
     out, err = capfd.readouterr()
     assert check_err in err
+    remove_output_base(config)
 
 
 @pytest.mark.util
diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py
index 238b2c10b..bbfd010a6 100644
--- a/metplus/util/config_metplus.py
+++ b/metplus/util/config_metplus.py
@@ -462,6 +462,8 @@ def __init__(self, conf=None, run_id=None):
         super().__init__(conf)
         self._cycle = None
         self.run_id = run_id if run_id else str(uuid.uuid4())[0:8]
+        # if run ID is specified, this is a copy of a config
+        self.is_copy = run_id is not None
         self._logger = logging.getLogger(f'metplus.{self.run_id}')
         # config.logger is called in wrappers, so set this name
         # so the code doesn't break
@@ -475,6 +477,9 @@ def __init__(self, conf=None, run_id=None):
 
     def __del__(self):
         """!When object is deleted, close and remove all log handlers"""
+        # do not close log handlers if this is a copied config object
+        if self.is_copy:
+            return
         handlers = self.logger.handlers[:]
         for handler in handlers:
             self.logger.removeHandler(handler)

From 4ff29503994b589c1cb6c80d158416a104243391 Mon Sep 17 00:00:00 2001
From: Molly Smith <molly.b.smith@noaa.gov>
Date: Wed, 18 Dec 2024 12:49:24 -0700
Subject: [PATCH 02/10] Updated METexpress versions in component_versions.py
 (#2842)

---
 metplus/component_versions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metplus/component_versions.py b/metplus/component_versions.py
index 9e363431c..e0c7cacc7 100755
--- a/metplus/component_versions.py
+++ b/metplus/component_versions.py
@@ -21,7 +21,7 @@
         'metcalcpy': '3.0.0',
         'metdataio': '3.0.0',
         'metviewer': '6.0.0',
-        'metexpress': None,
+        'metexpress': '6.0.0',
     },
     '5.1': {
         'metplus': '5.1.0',
@@ -30,7 +30,7 @@
         'metcalcpy': '2.1.0',
         'metdataio': '2.1.0',
         'metviewer': '5.1.0',
-        'metexpress': '5.3.3',
+        'metexpress': '5.3.5',
     },
 }
 

From 78987fbcd9c23e36470c5b1309ebbd6d6d877f56 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:57:31 -0700
Subject: [PATCH 03/10] added script to regroup release notes for official
 release by parsing dev release notes. improve naming of drop downs for
 generating dev release notes to be consistent

---
 .../compile_official_release_notes.py         | 54 +++++++++++++++++++
 .../dev_tools/generate_release_notes.py       |  5 +-
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 internal/scripts/dev_tools/compile_official_release_notes.py

diff --git a/internal/scripts/dev_tools/compile_official_release_notes.py b/internal/scripts/dev_tools/compile_official_release_notes.py
new file mode 100644
index 000000000..29beb6cc5
--- /dev/null
+++ b/internal/scripts/dev_tools/compile_official_release_notes.py
@@ -0,0 +1,54 @@
+#! /usr/bin/env python3
+
+# Written by George McCabe <mccabe@ucar.edu>
+# Helper script that parses release notes from development releases,
+# gathers all issues by category, sorts them by issue number, then
+# outputs the formatted content
+# Note: Careful review and massaging of output is likely needed
+# Run this script from the top of the repository to parse
+# Assumes location and name of release notes RST file
+
+import re
+
+infile = './docs/Users_Guide/release-notes.rst'
+
+with open(infile, 'r') as file_handle:
+    content = file_handle.read().splitlines()
+
+category = None
+items = {}
+# gather issues and organize them by category
+for line in content:
+    if match := re.match(r'  .. dropdown:: (.*)', line):
+        category = match.group(1)
+        if not items.get(category):
+            items[category] = []
+        continue
+    if category is None: continue
+    if not line: continue
+    if line.strip().startswith('.. _'):
+        break
+    if line.lstrip().startswith('*'):
+        items[category].append(line)
+    elif line.strip() == 'NONE' or line.startswith('MET') or line.startswith('---') or line.startswith('==='):
+        continue
+    else:
+        items[category][-1] += f'\n{line}'
+
+# get issues in each category to sort
+issues = {}
+for cat, item_list in items.items():
+    if not issues.get(cat):
+        issues[cat] = {}
+    for issue in item_list:
+        match = re.match(r'.*\#(\d+).*', issue.replace('\n', ''))
+        if match:
+            issues[cat][match.group(1)] = issue
+
+# sort issues within each category and print formatted result
+for cat in issues:
+    nums = sorted([int(item) for item in issues[cat].keys()])
+    print(f"  .. dropdown:: {cat}\n")
+    for num in nums:
+        print(issues[cat][str(num)])
+    print()
diff --git a/internal/scripts/dev_tools/generate_release_notes.py b/internal/scripts/dev_tools/generate_release_notes.py
index 9d029c965..5a6591052 100755
--- a/internal/scripts/dev_tools/generate_release_notes.py
+++ b/internal/scripts/dev_tools/generate_release_notes.py
@@ -65,7 +65,10 @@ def print_issues_by_category(repo_name, issues_by_category):
     for category, issues in issues_by_category.items():
         print()
         if category != 'none':
-            print(f"  .. dropdown:: {category}\n")
+            header = category
+            if header in ('Enhancement', 'New Wrapper', 'New Use Case'):
+                header = f'{header}s'
+            print(f"  .. dropdown:: {header}\n")
         elif issues:
             print('COULD NOT PARSE CATEGORY FROM THESE:\n')
 

From e53d7da448222f747560197a2d147e8d4cc383ac Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:06:07 -0700
Subject: [PATCH 04/10] improve script to handle different formatting for
 categories with no issues

---
 internal/scripts/dev_tools/compile_official_release_notes.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/scripts/dev_tools/compile_official_release_notes.py b/internal/scripts/dev_tools/compile_official_release_notes.py
index 29beb6cc5..4aba87605 100644
--- a/internal/scripts/dev_tools/compile_official_release_notes.py
+++ b/internal/scripts/dev_tools/compile_official_release_notes.py
@@ -30,7 +30,9 @@
         break
     if line.lstrip().startswith('*'):
         items[category].append(line)
-    elif line.strip() == 'NONE' or line.startswith('MET') or line.startswith('---') or line.startswith('==='):
+    elif line.strip().lower() == 'none' or line.startswith('MET') or line.startswith('---') or line.startswith('==='):
+        continue
+    elif not items.get(category):
         continue
     else:
         items[category][-1] += f'\n{line}'

From ec38af9bb01035c63e0b5a104f7c08782a2ca68b Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:29:47 -0700
Subject: [PATCH 05/10] per suggestion from @bikegeek, output 'None' if there
 were no items under a category

---
 internal/scripts/dev_tools/compile_official_release_notes.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/internal/scripts/dev_tools/compile_official_release_notes.py b/internal/scripts/dev_tools/compile_official_release_notes.py
index 4aba87605..1b797c3f7 100644
--- a/internal/scripts/dev_tools/compile_official_release_notes.py
+++ b/internal/scripts/dev_tools/compile_official_release_notes.py
@@ -51,6 +51,9 @@
 for cat in issues:
     nums = sorted([int(item) for item in issues[cat].keys()])
     print(f"  .. dropdown:: {cat}\n")
+    if not nums:
+        print('     None\n')
+        continue
     for num in nums:
         print(issues[cat][str(num)])
     print()

From db692d3716d15a11a77854870c7ca9c24fd8d811 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 19 Dec 2024 09:37:03 -0700
Subject: [PATCH 06/10] reorder instructions to follow more natural progression

---
 .../finalize_release_on_github_official.rst          | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst
index e3d4018ff..c2ec51b6e 100644
--- a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst
+++ b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst
@@ -5,6 +5,12 @@ Return to GitHub to finalize the details of this release.
 
 .. dropdown:: Instructions
 
+  * Update issues:
+
+    * Close the GitHub issue for creating this official release.
+
+    * If necessary, reassign any remaining issues for the current milestone to other milestones.
+
   * Update milestones:
 
     * Edit the milestone for the current release by updating the *Due date* with the actual release date.
@@ -15,12 +21,6 @@ Return to GitHub to finalize the details of this release.
 
     * If necessary, create a new milestone for the next official release (e.g. next vX.Y.Z release).
 
-  * Update issues:
-
-    * Close the GitHub issue for creating this official release.
-
-    * If necessary, reassign any remaining issues for the current milestone to other milestones.
-
   * Update projects:
 
     * Close the existing development project for the current milestone.

From c58f59fc16554501cf1b1619ad6435ce9655417c Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 19 Dec 2024 09:57:17 -0700
Subject: [PATCH 07/10] update instructions for updating ReadTheDocs based on
 the changes to the RTD web interface

---
 .../release_steps/update_docs_official.rst             | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/Release_Guide/release_steps/update_docs_official.rst b/docs/Release_Guide/release_steps/update_docs_official.rst
index 9250763b1..d29d9cb3a 100644
--- a/docs/Release_Guide/release_steps/update_docs_official.rst
+++ b/docs/Release_Guide/release_steps/update_docs_official.rst
@@ -13,15 +13,15 @@ repository will need to do the following to update the default branch:
     
   * Click on the appropriate METplus component project
     
-  * Click on Admin in the top menu
-    
-  * Click on Advanced Settings in the left menu
+  * Click on Settings in the top right menu
      
   * Select the new default branch in the dropdown menu for "Default branch"
-    (e.g. main_v4.0.0)
+    (e.g. main_v4.0.0) and click the Save button at the bottom of the page
     
   * Ensure that "latest" points to the new default branch by clicking on
-    "View Docs"in the upper right corner and confirm that the version number
+    "latest" build that just started and click on "Version latest" to view the
+    build run. When it finishes running, click on "View Docs" on the right of
+    the page and confirm that the version number
     displayed in the header is the desired version for "latest".
     
  

From 7920b252780349f74d1bc44b2920f0e106dd0759 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 19 Dec 2024 09:57:34 -0700
Subject: [PATCH 08/10] add next coordinated release to version lookup table

---
 metplus/component_versions.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/metplus/component_versions.py b/metplus/component_versions.py
index e0c7cacc7..10e4810e7 100755
--- a/metplus/component_versions.py
+++ b/metplus/component_versions.py
@@ -14,6 +14,15 @@
 import sys
 
 VERSION_LOOKUP = {
+    '6.1': {
+        'metplus': '6.1.0',
+        'met': '12.1.0',
+        'metplotpy': '3.1.0',
+        'metcalcpy': '3.1.0',
+        'metdataio': '3.1.0',
+        'metviewer': '6.1.0',
+        'metexpress': None,
+    },
     '6.0': {
         'metplus': '6.0.0',
         'met': '12.0.0',

From 9d4081073ccd67570985f5a9922ff30d3f2e57b6 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 19 Dec 2024 10:11:58 -0700
Subject: [PATCH 09/10] move instructions to create directory for data for new
 dev cycle on DTC web server to be done for the rc1 release instead of after
 the official release

---
 docs/Release_Guide/metplus_development.rst    |  1 +
 docs/Release_Guide/metplus_official.rst       |  1 -
 .../metplus/update_web_server_data.rst        | 56 ++++++++++---------
 3 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/docs/Release_Guide/metplus_development.rst b/docs/Release_Guide/metplus_development.rst
index ecd25965c..fd8cc4aa9 100644
--- a/docs/Release_Guide/metplus_development.rst
+++ b/docs/Release_Guide/metplus_development.rst
@@ -21,4 +21,5 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra
 .. include:: release_steps/metplus/update_version_on_develop.rst
 .. include:: release_steps/metplus/update_dtc_website.rst
 .. include:: release_steps/finalize_release_on_github_development.rst
+.. include:: release_steps/metplus/update_web_server_data.rst
 .. include:: release_steps/release_acceptance_testing.rst
diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst
index ad83b2f6b..c09ce003c 100644
--- a/docs/Release_Guide/metplus_official.rst
+++ b/docs/Release_Guide/metplus_official.rst
@@ -24,5 +24,4 @@ Create a new vX.Y.Z official release from the develop branch.
 .. include:: release_steps/update_docs_official.rst
 .. include:: release_steps/metplus/add_next_version_to_lookup.rst
 .. include:: release_steps/metplus/coordinated_release_announcement.rst
-.. include:: release_steps/metplus/update_web_server_data.rst
 .. include:: release_steps/set_beta_deletion_reminder_official.rst
diff --git a/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst b/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst
index 8f7d664bf..aaf4d59ba 100644
--- a/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst
+++ b/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst
@@ -4,37 +4,43 @@ Update DTC Web Server Data
 Create Directory for Next Release
 """""""""""""""""""""""""""""""""
 
-On the DTC web server where the sample input data for use cases is hosted,
-run the setup_next_release_data.py script for the next upcoming release
-to set up the data directory for the next major/minor version development.
-The script can be found in the METplus repository in internal/tests/use_cases.
-The file should be found in the home directory of the met_test user on
-the DTC web server host. It is linked to the file in the METplus repository.
-Pull the latest changes from the develop branch before running the script::
+.. dropdown:: If creating a rc1 release
 
-    runas met_test
-    cd /home/met_test/METplus
-    git checkout develop
-    git pull
+  On the DTC web server where the sample input data for use cases is hosted,
+  run the setup_next_release_data.py script for the next upcoming release
+  to set up the data directory for the next major/minor version development.
+  The script can be found in the METplus repository in internal/tests/use_cases.
+  The file should be found in the home directory of the met_test user on
+  the DTC web server host. It is linked to the file in the METplus repository.
+  Pull the latest changes from the develop branch before running the script::
 
-Now run the script passing in the version of the next release, i.e.
-if creating the v4.1.0 release, pass in v5.0 as the argument::
+      runas met_test
+      cd /home/met_test/METplus
+      git checkout develop
+      git pull
 
-    new_version=v5.0
-    /home/met_test/setup_next_release_data.py ${new_version}
+  Now run the script passing in the version of the next release, i.e.
+  if creating the v6.0.0-rc1 release, pass in v6.1 as the argument::
 
-See the comments in the script for more details.
-Ensure that the script runs without error and that the newly created
-directory contains links to all of the sample data tar files::
+      new_version=v6.1
+      /home/met_test/setup_next_release_data.py ${new_version}
 
-    ls -lh /home/met_test/METplus_Data/${new_version}
+  See the comments in the script for more details.
+  Ensure that the script runs without error and that the newly created
+  directory contains links to all of the sample data tar files::
 
-Untar each of the sample data tarfiles so the model_applications and
-met_test directories exist::
+      ls -lh /home/met_test/METplus_Data/${new_version}
 
-    cd /home/met_test/METplus_Data/${new_version}
-    for f in sample_data*; do echo tar xzf $f;tar xzf $f; done
+  Untar each of the sample data tarfiles so the model_applications and
+  met_test directories exist::
 
-Check if the met_test and model_applications directories now exist::
+      cd /home/met_test/METplus_Data/${new_version}
+      for f in sample_data*; do echo tar xzf $f;tar xzf $f; done
 
-    ls -lh /home/met_test/METplus_Data/${new_version}
+  Check if the met_test and model_applications directories now exist::
+
+      ls -lh /home/met_test/METplus_Data/${new_version}
+
+.. dropdown:: If creating a betaN or rc2+ release
+
+  Continue to the next instruction.

From bf720408bc038f087b268dbc653c5b56a215d845 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 19 Dec 2024 10:16:01 -0700
Subject: [PATCH 10/10] update versions of packages that have vulnerabilities

---
 internal/tests/pytests/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/tests/pytests/requirements.txt b/internal/tests/pytests/requirements.txt
index 0a4449537..804ca34ea 100644
--- a/internal/tests/pytests/requirements.txt
+++ b/internal/tests/pytests/requirements.txt
@@ -1,4 +1,4 @@
-certifi==2023.7.22
+certifi==2024.7.4
 cftime==1.6.2
 coverage==7.2.7
 exceptiongroup==1.1.2
@@ -8,7 +8,7 @@ numpy==1.25.2
 packaging==23.1
 pandas==2.0.3
 pdf2image==1.16.3
-Pillow==10.0.0
+Pillow==10.3.0
 pluggy==1.2.0
 pytest==7.4.0
 pytest-cov==4.1.0