From a21f3f05165d8f6265db61c899fd77e4cef8daef Mon Sep 17 00:00:00 2001 From: Josie Lyon <82548520+lyonthefrog@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:47:19 -0500 Subject: [PATCH] Das 1867 (#69) * Updated Harmony service test notebooks to reference the correct environment in the documentation. * DAS-1867: Updated reference files. Previous one had a typo in a string within a global parameter. * DAS-1867: Updated utilties.py function to use DataTrees instead of Datasets. * DAS-1867: Updated version number. * Remove wild whitespace. --------- Co-authored-by: Josie Lyon --- test/harmony/Harmony.ipynb | 15 +++++++ test/hga/HGA_Regression.ipynb | 16 +++++++ test/hoss/HOSS_Regression.ipynb | 2 +- test/hybig/HyBIG_Regression.ipynb | 16 +++++++ test/n2z/N2Z_Regression.ipynb | 16 +++++++ test/regridder/Regridder_Regression.ipynb | 16 +++++++ .../SwathProjector_Regression.ipynb | 2 +- .../TrajectorySubsetter_Regression.ipynb | 42 +++++++++++------- test/trajectory-subsetter/environment.yaml | 2 +- .../trajectory_subsetter_bbox_reference.h5 | Bin 1364654 -> 1364654 bytes .../trajectory_subsetter_polygon_reference.h5 | Bin 1096565 -> 1096565 bytes ...trajectory_subsetter_temporal_reference.h5 | Bin 7890871 -> 7890871 bytes ...trajectory_subsetter_variable_reference.h5 | Bin 10197207 -> 10197207 bytes test/trajectory-subsetter/utilities.py | 24 +++++----- test/trajectory-subsetter/version.txt | 2 +- .../VariableSubsetter_Regression.ipynb | 2 +- 16 files changed, 122 insertions(+), 33 deletions(-) diff --git a/test/harmony/Harmony.ipynb b/test/harmony/Harmony.ipynb index 8e15a29..85aa172 100644 --- a/test/harmony/Harmony.ipynb +++ b/test/harmony/Harmony.ipynb @@ -22,6 +22,21 @@ "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-harmony`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/test/hga/HGA_Regression.ipynb b/test/hga/HGA_Regression.ipynb index 9bb714b..21f2004 100644 --- a/test/hga/HGA_Regression.ipynb +++ b/test/hga/HGA_Regression.ipynb @@ -43,6 +43,22 @@ "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8240032", + "metadata": {}, + "outputs": [], + "source": [ + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-hga`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, { "cell_type": "markdown", "id": "7f0766b7", diff --git a/test/hoss/HOSS_Regression.ipynb b/test/hoss/HOSS_Regression.ipynb index dbd510a..7e2c459 100644 --- a/test/hoss/HOSS_Regression.ipynb +++ b/test/hoss/HOSS_Regression.ipynb @@ -21,7 +21,7 @@ "\n", "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", "\n", - "`conda env create -f ./environment.yaml && conda activate papermill`\n", + "`conda env create -f ./environment.yaml && conda activate papermill-hoss`\n", "\n", "A `.netrc` file must also be located in the `test` directory of this repository." ] diff --git a/test/hybig/HyBIG_Regression.ipynb b/test/hybig/HyBIG_Regression.ipynb index 22f6c5f..ceb265b 100644 --- a/test/hybig/HyBIG_Regression.ipynb +++ b/test/hybig/HyBIG_Regression.ipynb @@ -43,6 +43,22 @@ "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6e210ef", + "metadata": {}, + "outputs": [], + "source": [ + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-hybig`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, { "cell_type": "markdown", "id": "802241b5", diff --git a/test/n2z/N2Z_Regression.ipynb b/test/n2z/N2Z_Regression.ipynb index 3e71959..d0b6542 100644 --- a/test/n2z/N2Z_Regression.ipynb +++ b/test/n2z/N2Z_Regression.ipynb @@ -43,6 +43,22 @@ "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bdda441", + "metadata": {}, + "outputs": [], + "source": [ + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-n2z`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, { "cell_type": "markdown", "id": "4979305a", diff --git a/test/regridder/Regridder_Regression.ipynb b/test/regridder/Regridder_Regression.ipynb index 9efceb8..0105157 100644 --- a/test/regridder/Regridder_Regression.ipynb +++ b/test/regridder/Regridder_Regression.ipynb @@ -41,6 +41,22 @@ "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ca1e5bb", + "metadata": {}, + "outputs": [], + "source": [ + "## Prerequisites\n", + "\n", + "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", + "\n", + "`conda env create -f ./environment.yaml && conda activate papermill-regridder`\n", + "\n", + "A `.netrc` file must also be located in the `test` directory of this repository." + ] + }, { "cell_type": "markdown", "id": "4979305a", diff --git a/test/swath-projector/SwathProjector_Regression.ipynb b/test/swath-projector/SwathProjector_Regression.ipynb index aeb9901..548a7df 100644 --- a/test/swath-projector/SwathProjector_Regression.ipynb +++ b/test/swath-projector/SwathProjector_Regression.ipynb @@ -23,7 +23,7 @@ "\n", "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", "\n", - "`conda env create -f ./environment.yaml && conda activate papermill`\n", + "`conda env create -f ./environment.yaml && conda activate papermill-swath-projector`\n", "\n", "A `.netrc` file must also be located in the `test` directory of this repository." ] diff --git a/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb b/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb index 6f326c0..0d55d4b 100644 --- a/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb +++ b/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb @@ -21,7 +21,7 @@ "\n", "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", "\n", - "`conda env create -f ./environment.yaml && conda activate papermill`\n", + "`conda env create -f ./environment.yaml && conda activate papermill-trajectory-subsetter`\n", "\n", "A `.netrc` file must also be located in the `test` directory of this repository." ] @@ -172,7 +172,11 @@ "cell_type": "code", "execution_count": null, "id": "74bf1567", - "metadata": {}, + "metadata": { + "tags": [ + "raises-exception" + ] + }, "outputs": [], "source": [ "if trajectory_subsetter_info is not None:\n", @@ -185,8 +189,7 @@ " assert exists(ts_variable_file_name), 'Unsuccessful Trajectory Subsetter variable subset request.'\n", "\n", " compare_results_to_reference_file(ts_variable_file_name,\n", - " 'reference_files/trajectory_subsetter_variable_reference.h5',\n", - " '/BEAM0000')\n", + " 'reference_files/trajectory_subsetter_variable_reference.h5')\n", "\n", " print_success('Trajectory Subsetter variable subset request.')\n", "else:\n", @@ -207,7 +210,11 @@ "cell_type": "code", "execution_count": null, "id": "c51eb82c", - "metadata": {}, + "metadata": { + "tags": [ + "raises-exception" + ] + }, "outputs": [], "source": [ "if trajectory_subsetter_info is not None:\n", @@ -222,8 +229,7 @@ " assert exists(ts_temporal_file_name), 'Unsuccessful Trajectory Subsetter temporal subset request.'\n", "\n", " compare_results_to_reference_file(ts_temporal_file_name,\n", - " 'reference_files/trajectory_subsetter_temporal_reference.h5',\n", - " '/BEAM0000')\n", + " 'reference_files/trajectory_subsetter_temporal_reference.h5')\n", "\n", " print_success('Trajectory Subsetter temporal subset request.')\n", "else:\n", @@ -247,7 +253,11 @@ "cell_type": "code", "execution_count": null, "id": "ffb9742c", - "metadata": {}, + "metadata": { + "tags": [ + "raises-exception" + ] + }, "outputs": [], "source": [ "if trajectory_subsetter_info is not None:\n", @@ -262,8 +272,7 @@ " assert exists(ts_bbox_file_name), 'Unsuccessful Trajectory Subsetter bounding box subset request.'\n", "\n", " compare_results_to_reference_file(ts_bbox_file_name,\n", - " 'reference_files/trajectory_subsetter_bbox_reference.h5',\n", - " '/BEAM0000')\n", + " 'reference_files/trajectory_subsetter_bbox_reference.h5')\n", "\n", " print_success('Trajectory Subsetter bounding box spatial subset request.')\n", "else:\n", @@ -284,7 +293,11 @@ "cell_type": "code", "execution_count": null, "id": "a10d6af4", - "metadata": {}, + "metadata": { + "tags": [ + "raises-exception" + ] + }, "outputs": [], "source": [ "if trajectory_subsetter_info is not None:\n", @@ -298,8 +311,7 @@ " assert exists(ts_polygon_file_name), 'Unsuccessful Trajectory Subsetter polygon spatial subset request.'\n", "\n", " compare_results_to_reference_file(ts_polygon_file_name,\n", - " 'reference_files/trajectory_subsetter_polygon_reference.h5',\n", - " '/BEAM0000')\n", + " 'reference_files/trajectory_subsetter_polygon_reference.h5')\n", "\n", " print_success('Trajectory Subsetter polygon spatial subset request.')\n", "else:\n", @@ -338,7 +350,7 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "papermill-trajectory-subsetter", "language": "python", "name": "python3" }, @@ -352,7 +364,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.15" + "version": "3.10.12" }, "name": "SDS_Regression.ipynb" }, diff --git a/test/trajectory-subsetter/environment.yaml b/test/trajectory-subsetter/environment.yaml index d59f48a..4e1ffaa 100644 --- a/test/trajectory-subsetter/environment.yaml +++ b/test/trajectory-subsetter/environment.yaml @@ -9,6 +9,6 @@ dependencies: - python-dateutil=2.7.5 - numpy=1.24.3 - netCDF4=1.6.3 - - xarray=2022.12.0 + - xarray-datatree=0.0.12 - pip: - harmony-py diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_bbox_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_bbox_reference.h5 index 45b5c62e29b7e52edd0de11e51a7c2423c9d2b63..0ff4fb23ec7dc68badbb4994eb3aec0259cbf9f7 100644 GIT binary patch delta 433 zcmZ3tG^pv>umLfQp6FI7wWP{*kH3dY5R(E-6h=?HS;^uY$Fg>E7^AfS zNE@0z@v={kn=a{@6J5OZ&DV&kzE7DRXS>dF1ge%ocFc;XmYq8a8W zOuu+qP@;XBG!GE-0x=&D^8>NK_G!|B8SAh)OM1HIDZ%F)YmznZ`a64gPhWUUM66x! zv>*@*ZPz<3ob($**YpL)L@HS{*4rGLE_+ diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_polygon_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_polygon_reference.h5 index 3679bf51e9cfea204cc209f5bd7a7c47a0afde2c..96cf8cfbc401e10cfe41684782ab2e81245ec3e4 100644 GIT binary patch delta 408 zcmezR&FSkmrwL{PDLa0o_yaMFp6FI7bvD{4#oxmvh)Dq^3Zp09tYrBZ-gR(t7^5{< zD@X!{H!oylVr2RA;hDhHK+dMyUd$eM1Hg-PoBbZ1CO%PteaVO z&s%EM+{D)2#Ks83OhC*G#4JF}3dA6Bb|B{1-o(cF_6der+v{I*u4QDAG`{QCZt#{9 zh`E568;E(f8@%PM%*D_Fbfon3D^YyUSsLeXY@7aYtAJGd&1gO#=HGrZTHtiBAk4Qg zx?+0W1_7Dr7uO5Ku=F-{znX5iK>#ysw%2SBuwi0plCYN8zGss_9rJYSZ35biI@{B> S32-yAv>ZCEwY+q@01E()g~1H~ delta 408 zcmezR&FSkmrwL{PtY!O?Ka}oIhS3wc;54Q?Pwcm{917iN|H=_kk2MfY{ z3!^Kh*KH7xnSOD-KnzQz%E#F0h8qMh!)ANU1_2u;mJ{6ileX{KBv8l9STx;gn}GKA Uv~2?1j4bDKrk+?{x?O+;07HVMc>n+a diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_temporal_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_temporal_reference.h5 index dcd1aa1a9ebb391d0d88dbd3e5c7946a7766ed08..24821a6bbce4cd865e2a4cd8e8eb4801e6d015c3 100644 GIT binary patch delta 823 zcmZ|NPi)I!7{KxTe!qA1Z>FQDmg)Q*8~ewMj%$=gC=|8&F_(M zKl{x-&1=s+D>R$cbWL>4|9W9g>mQ2BW>)i< z&y8;)=_pH>kHp9Sb<~>gN@m4F>3DlDZOf!>8gv*CC_yg22#c`eP>D_0j4jxTZP<<~?7&V`qXxUM8?~rIJsQx6J=lvT z>_ap5;{d#9!9g5CD|{+#^9P#Ca|(sK!=*SQEj~!7;PW>CDXd{5Xs@wBram za1@<5hAte(33Q_ey*PP|z)eixmYP$y7p97S zw6X&-CFbknE~_FWy+w5u56Ko?#2}C&H`E1G&)UjrJozCmKE9ra7+p$L+UQ(LQ5Eqj?^nv0R^O+Ujg;asX0@;4 zowvzmJQmCU>nJkbWXy@5cT4Q|6P9GcqCkZPfeftX*C7+@u>l*g307>z7GxnCIoJvt zwqZN$a9{_VaA7BQ!HwPUAQyX(hkO)ZFTB`?{WyR^9K<0UMiGi}1SKfNQIw$^6{y59 z97h$ZaRMh%1D{M-{DI2szYp=;c&N3>(i(qA5f9v-bB$t$BRS*qqZX%7htsIX88n~~ zO=w07TG57foJ9u$=)^gkM;9*OA}--Fx)H<`gm4u-=tUS2T$3}t=t}HgS9X~%=cC_* z^^^6txzwvWlZ!Hn>$rhF^kV>n7{V|{Fp4qU#5iu@He#5Ni}KFW)3l9ds#}kXu209WnKSE9w4*Ib>+2L> zTBTQk^1Sbx=Xsy^O+EQ8rVy*uM_&Bw z3`33a24B&$&u`3br=wY8QnPA|zxRnfI)_DS5p4SOvHuPA3mQQz9czCp4QNss%Tm8! zk{)Y6iEXdHjY$ddT}JJHJ1V8&tRXS4Z{$}npbG_b0yNMAKdamdDaRG8B;J7(@=my7%&}0C`Jipz=)YJVHRd%4(6g1WtfNgSO7E1QGtc9pb}MBgvF@F z64am;RxHIbEQbwttiVdFLLF9P4eH^*S~Q>$POQUvG{J=p*a$Z~*o0PJtk&RYVa5&lSjEXkckoMg zq+`(Fu6H&iKSt$(kjoQ9FRtM_LI@**8@P#Eh~hTxAci>ZLP7$4xQ8U};{hHjE|2`w zp0CbXjosUWu9jLCj&@{uH$3l+*sNzN6- N-si*SfmA{k{{XPThr<8> delta 972 zcmZY7NlX)A6u|LJ848L^Q7mN?0PIbY&ZT$e~$mIef~ z^eAma%xiL2NKx_S-`dHWHUD;$Nzc;<#bVj_E8y1${dxg9q(MYFlKBi|ViZOr3u7=A z<1iiuWMcv*A_tRTL@rFoLp};nh{-6z6qqp;(@>1*n1Pv?h1r;cxhTOr%ttB8P>uzt zKqVHU3e~V+5f-BcR@krvOR)^gu>vbmi#n`AJ?wB`HP)a3POQZ`xZuWmG@?oM>piba zQ@&koukt(;#Cu<3HmTfX3R)~4Y(O(MViPuF3$~&K+prxwuoJtm8?D%bz1WBSIDmsV zgu`gV5wzndj^Q{u(1|W|;{;Bk2VR^)FHYkOd^n49I1j&Su>?Nkro52BDv<52z{kv* zAHVOrY8?&oyO3fGI^6+Wz(rg_5SMWUSJ8)S2;n+zAdCoZLP8WV+=7hT=*Jz^=~kZD zvbFKnVo$Pop`QIloEAFQnCc` MMO$nro`@>qAAHnwJ^%m! diff --git a/test/trajectory-subsetter/utilities.py b/test/trajectory-subsetter/utilities.py index 1f9a2c6..4e4828e 100644 --- a/test/trajectory-subsetter/utilities.py +++ b/test/trajectory-subsetter/utilities.py @@ -7,28 +7,26 @@ from harmony import Client, Request from harmony.harmony import ProcessingFailedException -import xarray as xr +import datatree def compare_results_to_reference_file(results_file_name: str, - reference_file_name: str, - group_path: str) -> None: - """ Use native `xarray` functionality to compare data values and metadata - attributes of downloaded results to a reference file. + reference_file_name: str) -> None: + """ Use `DataTree` functionality to compare data values, variables, + coordinates, metadata, and all their corresponding attributes of + downloaded results to a reference file. """ - reference_data = xr.open_dataset(reference_file_name, group=group_path) - results_data = xr.open_dataset(results_file_name, group=group_path) + reference_data = datatree.open_datatree(reference_file_name) + results_data = datatree.open_datatree(results_file_name) - assert results_data.equals(reference_data), ('Output and reference files ' - 'do not match.') + assert results_data.identical(reference_data), ('Output and reference files ' + 'do not match.') - reference_data.close() - results_data.close() reference_data = None results_data = None - - + + def submit_and_download(harmony_client: Client, request: Request, output_file_name: str): """ Submit a Harmony request via a `harmony-py` client. Wait for the diff --git a/test/trajectory-subsetter/version.txt b/test/trajectory-subsetter/version.txt index b1e80bb..845639e 100644 --- a/test/trajectory-subsetter/version.txt +++ b/test/trajectory-subsetter/version.txt @@ -1 +1 @@ -0.1.3 +0.1.4 diff --git a/test/variable-subsetter/VariableSubsetter_Regression.ipynb b/test/variable-subsetter/VariableSubsetter_Regression.ipynb index 2b0411e..6c968cb 100644 --- a/test/variable-subsetter/VariableSubsetter_Regression.ipynb +++ b/test/variable-subsetter/VariableSubsetter_Regression.ipynb @@ -17,7 +17,7 @@ "\n", "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n", "\n", - "`conda env create -f ./environment.yaml && conda activate papermill`\n", + "`conda env create -f ./environment.yaml && conda activate papermill-variable-subsetter`\n", "\n", "A `.netrc` file must also be located in the `test` directory of this repository." ]