diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..a506297 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# For more information, see: +# https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view + +# Black code formatting of entire repository +da4cd7af618fea30ab54052f6ccaa137c5471d82 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c59f584 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.4 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.3.0 + hooks: + - id: black-jupyter + args: ["--skip-string-normalization"] + language_version: python3.11 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfc518..c27abfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v1.0.1 +### 2024-04-05 + +This version of the Swath Projector implements black code formatting across the +entire repository. There should be no functional changes to the service. + ## v1.0.0 ### 2023-11-16 diff --git a/README.md b/README.md index 3560494..77e4368 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,39 @@ newest release of the code (starting at the top of the file). ## vX.Y.Z ``` +### pre-commit hooks: + +This repository uses [pre-commit](https://pre-commit.com/) to enable pre-commit +checking the repository for some coding standard best practices. These include: + +* Removing trailing whitespaces. +* Removing blank lines at the end of a file. +* JSON files have valid formats. +* [ruff](https://github.com/astral-sh/ruff) Python linting checks. +* [black](https://black.readthedocs.io/en/stable/index.html) Python code + formatting checks. + +To enable these checks: + +```bash +# Install pre-commit Python package as part of test requirements: +pip install -r tests/pip_test_requirements.txt + +# Install the git hook scripts: +pre-commit install + +# (Optional) Run against all files: +pre-commit run --all-files +``` + +When you try to make a new commit locally, `pre-commit` will automatically run. +If any of the hooks detect non-compliance (e.g., trailing whitespace), that +hook will state it failed, and also try to fix the issue. You will need to +review and `git add` the changes before you can make a commit. + +It is planned to implement additional hooks, possibly including tools such as +`mypy`. + ## Get in touch: You can reach out to the maintainers of this repository via email: diff --git a/bin/project_local_granule.py b/bin/project_local_granule.py index 7662241..fcebab9 100644 --- a/bin/project_local_granule.py +++ b/bin/project_local_granule.py @@ -67,6 +67,7 @@ function below can be edited. """ + from os import environ from unittest.mock import patch @@ -77,10 +78,10 @@ def set_environment_variables(): - """ If the following environment variables are absent, the - `SwathProjectorAdapter` class will not allow the projector to run. Make - sure to run this script in a different environment (e.g. conda - environment) than any local instance of Harmony. + """If the following environment variables are absent, the + `SwathProjectorAdapter` class will not allow the projector to run. Make + sure to run this script in a different environment (e.g. conda + environment) than any local instance of Harmony. """ environ['ENV'] = 'dev' @@ -93,39 +94,48 @@ def set_environment_variables(): def rmtree_side_effect(workdir: str, ignore_errors=True) -> None: - """ A side effect for the `shutil.rmtree` mock that will print the - temporary working directory containing all output NetCDF-4 files. + """A side effect for the `shutil.rmtree` mock that will print the + temporary working directory containing all output NetCDF-4 files. """ print(f'\n\n\n\033[92mOutput files saved to: {workdir}\033[0m\n\n\n') -def project_granule(local_file_path: str, target_crs: str = 'EPSG:4326', - interpolation_method: str = 'near') -> None: - """ The `local_file_path` will need to be absolute, and prepended with - `file:///` to ensure that the `harmony-service-lib-py` package can - recognise it as a local file. +def project_granule( + local_file_path: str, + target_crs: str = 'EPSG:4326', + interpolation_method: str = 'near', +) -> None: + """The `local_file_path` will need to be absolute, and prepended with + `file:///` to ensure that the `harmony-service-lib-py` package can + recognise it as a local file. - The optional keyword arguments `target_crs` and `interpolation_method` - allow for a test that overrides the default message parameters of a - geographically projected output using nearest neighbour interpolation. + The optional keyword arguments `target_crs` and `interpolation_method` + allow for a test that overrides the default message parameters of a + geographically projected output using nearest neighbour interpolation. """ - message = Message({ - 'callback': 'https://example.com/callback', - 'stagingLocation': 's3://example-bucket/example-path', - 'sources': [{ - 'granules': [{ - 'url': local_file_path, - 'temporal': { - 'start': '2021-01-03T23:45:00.000Z', - 'end': '2020-01-04T00:00:00.000Z', - }, - 'bbox': [-180, -90, 180, 90], - }], - }], - 'format': {'crs': target_crs, 'interpolation': interpolation_method}, - }) + message = Message( + { + 'callback': 'https://example.com/callback', + 'stagingLocation': 's3://example-bucket/example-path', + 'sources': [ + { + 'granules': [ + { + 'url': local_file_path, + 'temporal': { + 'start': '2021-01-03T23:45:00.000Z', + 'end': '2020-01-04T00:00:00.000Z', + }, + 'bbox': [-180, -90, 180, 90], + } + ], + } + ], + 'format': {'crs': target_crs, 'interpolation': interpolation_method}, + } + ) set_environment_variables() diff --git a/docker/service_version.txt b/docker/service_version.txt index 3eefcb9..7dea76e 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.0.0 +1.0.1 diff --git a/docs/Swath Projector User Guide.ipynb b/docs/Swath Projector User Guide.ipynb index 096ec50..4425747 100644 --- a/docs/Swath Projector User Guide.ipynb +++ b/docs/Swath Projector User Guide.ipynb @@ -47,12 +47,20 @@ "from netCDF4 import Dataset\n", "\n", "import matplotlib.pyplot as plt\n", + "\n", "%matplotlib inline\n", "\n", "\n", - "def create_plot(variable_data, x_values, y_values, title=None, colourbar_units=None,\n", - " x_label=None, y_label=None):\n", - " \"\"\" This helper function will display a contour plot of the requested data. \"\"\"\n", + "def create_plot(\n", + " variable_data,\n", + " x_values,\n", + " y_values,\n", + " title=None,\n", + " colourbar_units=None,\n", + " x_label=None,\n", + " y_label=None,\n", + "):\n", + " \"\"\"This helper function will display a contour plot of the requested data.\"\"\"\n", " fig = plt.figure(figsize=(10, 10))\n", "\n", " if title is not None:\n", @@ -62,7 +70,7 @@ "\n", " # Plot masked data:\n", " colour_scale = ax.contourf(x_values[:], y_values[:], variable_data[0][:], levels=20)\n", - " \n", + "\n", " # Add colour bar for scaling\n", " colour_bar = plt.colorbar(colour_scale, ax=ax, orientation='horizontal', pad=0.05)\n", "\n", @@ -73,15 +81,30 @@ " plt.show()\n", "\n", "\n", - "def plot_variable(file_name, variable, x_variable, y_variable, title, colourbar_units,\n", - " x_label, y_label):\n", - " \"\"\" Open the requested NetCDF-4 file and pass the variables through to the `create_plot`\n", - " function.\n", + "def plot_variable(\n", + " file_name,\n", + " variable,\n", + " x_variable,\n", + " y_variable,\n", + " title,\n", + " colourbar_units,\n", + " x_label,\n", + " y_label,\n", + "):\n", + " \"\"\"Open the requested NetCDF-4 file and pass the variables through to the `create_plot`\n", + " function.\n", "\n", " \"\"\"\n", " with Dataset(file_name, 'r') as dataset:\n", - " create_plot(dataset[variable], dataset[x_variable], dataset[y_variable], title=title,\n", - " colourbar_units=colourbar_units, x_label=x_label, y_label=y_label)" + " create_plot(\n", + " dataset[variable],\n", + " dataset[x_variable],\n", + " dataset[y_variable],\n", + " title=title,\n", + " colourbar_units=colourbar_units,\n", + " x_label=x_label,\n", + " y_label=y_label,\n", + " )" ] }, { @@ -100,12 +123,14 @@ "outputs": [], "source": [ "collection_id = 'C1233860183-EEDTEST'\n", - "granule_ids = {'015_00_210_africa': 'G1233860549-EEDTEST',\n", - " '015_01_210_australia': 'G1233860551-EEDTEST',\n", - " '015_02_210_europe': 'G1233860553-EEDTEST',\n", - " '002_00_028_africa': 'G1233860481-EEDTEST',\n", - " '002_01_028_australia': 'G1233860484-EEDTEST',\n", - " '002_02_028_europe': 'G1233860486-EEDTEST'}" + "granule_ids = {\n", + " '015_00_210_africa': 'G1233860549-EEDTEST',\n", + " '015_01_210_australia': 'G1233860551-EEDTEST',\n", + " '015_02_210_europe': 'G1233860553-EEDTEST',\n", + " '002_00_028_africa': 'G1233860481-EEDTEST',\n", + " '002_01_028_australia': 'G1233860484-EEDTEST',\n", + " '002_02_028_europe': 'G1233860486-EEDTEST',\n", + "}" ] }, { @@ -169,8 +194,8 @@ "outputs": [], "source": [ "def wait_and_download_results(job_id):\n", - " \"\"\" A helper function that waits for a job to complete, and then\n", - " downloads the results.\n", + " \"\"\"A helper function that waits for a job to complete, and then\n", + " downloads the results.\n", "\n", " \"\"\"\n", " print(f'\\nWaiting for the job {job_id} to finish')\n", @@ -189,8 +214,16 @@ "downloaded_files = wait_and_download_results(job_id)\n", "\n", "# Plot results\n", - "plot_variable(downloaded_files[0], 'alpha_var', 'lon', 'lat', 'Default arguments, Europe', 'Land Mask',\n", - " 'Longitude (degrees east)', 'Latitude (degrees north)')" + "plot_variable(\n", + " downloaded_files[0],\n", + " 'alpha_var',\n", + " 'lon',\n", + " 'lat',\n", + " 'Default arguments, Europe',\n", + " 'Land Mask',\n", + " 'Longitude (degrees east)',\n", + " 'Latitude (degrees north)',\n", + ")" ] }, { @@ -231,16 +264,25 @@ "source": [ "interpolation_options = ['bilinear', 'ewa', 'ewa-nn', 'near']\n", "\n", - "nearest_neighbour_request = Request(collection=collection,\n", - " granule_id=[granule_ids['002_01_028_australia']],\n", - " interpolation='near')\n", + "nearest_neighbour_request = Request(\n", + " collection=collection,\n", + " granule_id=[granule_ids['002_01_028_australia']],\n", + " interpolation='near',\n", + ")\n", "\n", "nearest_neighbour_job_id = harmony_client.submit(nearest_neighbour_request)\n", "nearest_neighbour_results = wait_and_download_results(nearest_neighbour_job_id)\n", "\n", - "plot_variable(nearest_neighbour_results[0], 'alpha_var', 'lon', 'lat',\n", - " 'Nearest Neighbour interpolation, Australia', 'Land Mask',\n", - " 'Longitude (degrees east)', 'Latitude (degrees north)')" + "plot_variable(\n", + " nearest_neighbour_results[0],\n", + " 'alpha_var',\n", + " 'lon',\n", + " 'lat',\n", + " 'Nearest Neighbour interpolation, Australia',\n", + " 'Land Mask',\n", + " 'Longitude (degrees east)',\n", + " 'Latitude (degrees north)',\n", + ")" ] }, { @@ -263,16 +305,25 @@ "x_min, x_max = (13.3, 24.2)\n", "y_min, y_max = (7.4, 23.7)\n", "\n", - "scale_extent_request = Request(collection=collection,\n", - " granule_id=[granule_ids['002_00_028_africa']],\n", - " scale_extent=[x_min, y_min, x_max, y_max])\n", + "scale_extent_request = Request(\n", + " collection=collection,\n", + " granule_id=[granule_ids['002_00_028_africa']],\n", + " scale_extent=[x_min, y_min, x_max, y_max],\n", + ")\n", "\n", "scale_extent_job_id = harmony_client.submit(scale_extent_request)\n", "scale_extent_results = wait_and_download_results(scale_extent_job_id)\n", "\n", - "plot_variable(scale_extent_results[0], 'green_var', 'lon', 'lat',\n", - " 'Output grid surrounding Chad', 'green_var',\n", - " 'Longitude (degrees east)', 'Latitude (degrees north)')" + "plot_variable(\n", + " scale_extent_results[0],\n", + " 'green_var',\n", + " 'lon',\n", + " 'lat',\n", + " 'Output grid surrounding Chad',\n", + " 'green_var',\n", + " 'Longitude (degrees east)',\n", + " 'Latitude (degrees north)',\n", + ")" ] }, { @@ -290,17 +341,26 @@ "metadata": {}, "outputs": [], "source": [ - "dimensions_request = Request(collection=collection,\n", - " granule_id=[granule_ids['015_02_210_europe']],\n", - " height=250,\n", - " width=250)\n", + "dimensions_request = Request(\n", + " collection=collection,\n", + " granule_id=[granule_ids['015_02_210_europe']],\n", + " height=250,\n", + " width=250,\n", + ")\n", "\n", "dimensions_job_id = harmony_client.submit(dimensions_request)\n", "dimensions_results = wait_and_download_results(dimensions_job_id)\n", "\n", - "plot_variable(dimensions_results[0], 'green_var', 'lon', 'lat',\n", - " 'Specifying output grid dimensions', 'green_var',\n", - " 'Longitude (degrees east)', 'Latitude (degrees north)')" + "plot_variable(\n", + " dimensions_results[0],\n", + " 'green_var',\n", + " 'lon',\n", + " 'lat',\n", + " 'Specifying output grid dimensions',\n", + " 'green_var',\n", + " 'Longitude (degrees east)',\n", + " 'Latitude (degrees north)',\n", + ")" ] }, { @@ -321,16 +381,25 @@ "x_resolution = 0.5\n", "y_resolution = 0.5\n", "\n", - "scale_size_request = Request(collection=collection,\n", - " granule_id=[granule_ids['015_01_210_australia']],\n", - " scale_size=[x_resolution, y_resolution])\n", + "scale_size_request = Request(\n", + " collection=collection,\n", + " granule_id=[granule_ids['015_01_210_australia']],\n", + " scale_size=[x_resolution, y_resolution],\n", + ")\n", "\n", "scale_size_job_id = harmony_client.submit(scale_size_request)\n", "scale_size_results = wait_and_download_results(scale_size_job_id)\n", "\n", - "plot_variable(scale_size_results[0], 'green_var', 'lon', 'lat',\n", - " 'Specifying output pixel size', 'green_var',\n", - " 'Longitude (degrees east)', 'Latitude (degrees north)')" + "plot_variable(\n", + " scale_size_results[0],\n", + " 'green_var',\n", + " 'lon',\n", + " 'lat',\n", + " 'Specifying output pixel size',\n", + " 'green_var',\n", + " 'Longitude (degrees east)',\n", + " 'Latitude (degrees north)',\n", + ")" ] }, { @@ -350,16 +419,25 @@ "metadata": {}, "outputs": [], "source": [ - "crs_epsg_request = Request(collection=collection,\n", - " granule_id=[granule_ids['015_00_210_africa']],\n", - " crs='EPSG:6933')\n", + "crs_epsg_request = Request(\n", + " collection=collection,\n", + " granule_id=[granule_ids['015_00_210_africa']],\n", + " crs='EPSG:6933',\n", + ")\n", "\n", "crs_epsg_job_id = harmony_client.submit(crs_epsg_request)\n", "crs_epsg_results = wait_and_download_results(crs_epsg_job_id)\n", "\n", - "plot_variable(crs_epsg_results[0], 'blue_var', 'x', 'y',\n", - " 'Specifying output pixel size', 'blue_var',\n", - " 'x (projected metres)', 'y (projected metres)')" + "plot_variable(\n", + " crs_epsg_results[0],\n", + " 'blue_var',\n", + " 'x',\n", + " 'y',\n", + " 'Specifying output pixel size',\n", + " 'blue_var',\n", + " 'x (projected metres)',\n", + " 'y (projected metres)',\n", + ")" ] } ], diff --git a/swath_projector/__main__.py b/swath_projector/__main__.py index 798ed3c..44d5214 100644 --- a/swath_projector/__main__.py +++ b/swath_projector/__main__.py @@ -1,4 +1,5 @@ """ Run the Harmony Swath Projector adapter via the Harmony CLI. """ + from argparse import ArgumentParser from sys import argv @@ -8,12 +9,14 @@ def main(arguments: list[str]): - """ Parse command line arguments and invoke the appropriate method to - respond to them. + """Parse command line arguments and invoke the appropriate method to + respond to them. """ - parser = ArgumentParser(prog='harmony-swath-projector', - description='Run the Harmony Swath Projector Tool') + parser = ArgumentParser( + prog='harmony-swath-projector', + description='Run the Harmony Swath Projector Tool', + ) setup_cli(parser) harmony_arguments, _ = parser.parse_known_args(arguments[1:]) diff --git a/swath_projector/adapter.py b/swath_projector/adapter.py index 0b5326b..5492890 100644 --- a/swath_projector/adapter.py +++ b/swath_projector/adapter.py @@ -1,4 +1,5 @@ """ Data Services Swath Projector service for Harmony. """ + import mimetypes import os import shutil @@ -6,27 +7,26 @@ from harmony import BaseHarmonyAdapter from harmony.message import Source as HarmonySource -from harmony.util import (download, generate_output_filename, HarmonyException, - stage) +from harmony.util import download, generate_output_filename, HarmonyException, stage from pystac import Asset, Item from swath_projector.reproject import reproject class SwathProjectorAdapter(BaseHarmonyAdapter): - """ Data Services Swath Projector service for Harmony + """Data Services Swath Projector service for Harmony - This class uses the Harmony utility library for processing the - service input options. + This class uses the Harmony utility library for processing the + service input options. """ def invoke(self): - """ Adds validation to default process_item-based invocation + """Adds validation to default process_item-based invocation - Returns - ------- - pystac.Catalog - the output catalog + Returns + ------- + pystac.Catalog + the output catalog """ logger = self.logger logger.info('Starting Data Services Swath Projector Service') @@ -65,35 +65,45 @@ def process_item(self, item: Item, source: HarmonySource): asset = next(v for v in item.assets.values() if 'data' in (v.roles or [])) granule_url = asset.href - input_filename = download(granule_url, - workdir, - logger=logger, - access_token=self.message.accessToken, - cfg=self.config) + input_filename = download( + granule_url, + workdir, + logger=logger, + access_token=self.message.accessToken, + cfg=self.config, + ) logger.info('Granule data copied') # Call Reprojection utility - working_filename = reproject(self.message, source.shortName, - granule_url, input_filename, workdir, - logger) + working_filename = reproject( + self.message, + source.shortName, + granule_url, + input_filename, + workdir, + logger, + ) # Stage the output file with a conventional filename output_filename = generate_output_filename(asset.href, is_regridded=True) - mimetype, _ = ( - mimetypes.guess_type(output_filename, False) - or ('application/x-netcdf4', None) + mimetype, _ = mimetypes.guess_type(output_filename, False) or ( + 'application/x-netcdf4', + None, ) - url = stage(working_filename, - output_filename, - mimetype, - location=self.message.stagingLocation, - logger=self.logger) + url = stage( + working_filename, + output_filename, + mimetype, + location=self.message.stagingLocation, + logger=self.logger, + ) # Update the STAC record - asset = Asset(url, title=output_filename, media_type=mimetype, - roles=['data']) + asset = Asset( + url, title=output_filename, media_type=mimetype, roles=['data'] + ) result.assets['data'] = asset # Return the output file back to Harmony @@ -103,15 +113,17 @@ def process_item(self, item: Item, source: HarmonySource): except Exception as err: logger.error('Reprojection failed: ' + str(err), exc_info=1) - raise HarmonyException('Reprojection failed with error: ' + str(err)) from err + raise HarmonyException( + 'Reprojection failed with error: ' + str(err) + ) from err finally: # Clean up any intermediate resources shutil.rmtree(workdir, ignore_errors=True) def validate_message(self): - """ Check the service was triggered by a valid message containing - the expected number of granules. + """Check the service was triggered by a valid message containing + the expected number of granules. """ if not hasattr(self, 'message'): diff --git a/swath_projector/exceptions.py b/swath_projector/exceptions.py index 1aaf146..49f7d84 100644 --- a/swath_projector/exceptions.py +++ b/swath_projector/exceptions.py @@ -6,10 +6,11 @@ class CustomError(Exception): - """ Base class for exceptions in the Swath Projector. This base class could - be extended in the future to assign exit codes, for example. + """Base class for exceptions in the Swath Projector. This base class could + be extended in the future to assign exit codes, for example. """ + def __init__(self, exception_type, message): self.exception_type = exception_type self.message = message @@ -17,23 +18,28 @@ def __init__(self, exception_type, message): class MissingReprojectedDataError(CustomError): - """ This exception is raised when an expected single-band output file - containing reprojected data for a science variable is not found by - the `create_output` function in `nc_merge.py`. + """This exception is raised when an expected single-band output file + containing reprojected data for a science variable is not found by + the `create_output` function in `nc_merge.py`. """ + def __init__(self, missing_variable): - super().__init__('MissingReprojectedDataError', - ('Could not find reprojected output file for ' - f'{missing_variable}.')) + super().__init__( + 'MissingReprojectedDataError', + ('Could not find reprojected output file for ' f'{missing_variable}.'), + ) class MissingCoordinatesError(CustomError): - """ This exception is raised when for science variables an coordinate - variable is not found in dataset by the `get_coordinate_variable` - function in `utilities.py`. + """This exception is raised when for science variables an coordinate + variable is not found in dataset by the `get_coordinate_variable` + function in `utilities.py`. """ + def __init__(self, missing_coordinate): - super().__init__('MissingCoordinatesError', - f'Could not find coordinate {missing_coordinate}.') + super().__init__( + 'MissingCoordinatesError', + f'Could not find coordinate {missing_coordinate}.', + ) diff --git a/swath_projector/interpolation.py b/swath_projector/interpolation.py index 2540903..ddc85a7 100644 --- a/swath_projector/interpolation.py +++ b/swath_projector/interpolation.py @@ -2,6 +2,7 @@ datasets within a file, using the pyresample Python package. """ + from functools import partial from logging import Logger from typing import Dict, List, Optional, Tuple @@ -16,13 +17,10 @@ from varinfo import VarInfoFromNetCDF4 import numpy as np -from swath_projector.nc_single_band import ( - HARMONY_TARGET, - write_single_band_output -) +from swath_projector.nc_single_band import HARMONY_TARGET, write_single_band_output from swath_projector.swath_geometry import ( get_extents_from_perimeter, - get_projected_resolution + get_projected_resolution, ) from swath_projector.utilities import ( create_coordinates_key, @@ -31,7 +29,7 @@ get_variable_file_path, get_variable_numeric_fill_value, get_variable_values, - make_array_two_dimensional + make_array_two_dimensional, ) # In nearest neighbour interpolation, the distance to a found value is @@ -48,17 +46,19 @@ RADIUS_OF_INFLUENCE = 50000 -def resample_all_variables(message_parameters: Dict, - science_variables: List[str], - temp_directory: str, - logger: Logger, - var_info: VarInfoFromNetCDF4) -> List[str]: - """ Iterate through all science variables and reproject to the target - coordinate grid. +def resample_all_variables( + message_parameters: Dict, + science_variables: List[str], + temp_directory: str, + logger: Logger, + var_info: VarInfoFromNetCDF4, +) -> List[str]: + """Iterate through all science variables and reproject to the target + coordinate grid. - Returns: - output_variables: A list of names of successfully reprojected - variables. + Returns: + output_variables: A list of names of successfully reprojected + variables. """ output_extension = os.path.splitext(message_parameters['input_file'])[-1] reprojection_cache = get_reprojection_cache(message_parameters) @@ -68,16 +68,21 @@ def resample_all_variables(message_parameters: Dict, for variable in science_variables: try: - variable_output_path = get_variable_file_path(temp_directory, - variable, - output_extension) + variable_output_path = get_variable_file_path( + temp_directory, variable, output_extension + ) logger.info(f'Reprojecting variable "{variable}"') logger.info(f'Reprojected output: "{variable_output_path}"') - resample_variable(message_parameters, variable, - reprojection_cache, variable_output_path, - logger, var_info) + resample_variable( + message_parameters, + variable, + reprojection_cache, + variable_output_path, + logger, + var_info, + ) output_variables.append(variable) except Exception as error: @@ -89,22 +94,27 @@ def resample_all_variables(message_parameters: Dict, return output_variables -def resample_variable(message_parameters: Dict, full_variable: str, - reprojection_cache: Dict, variable_output_path: str, - logger: Logger, var_info: VarInfoFromNetCDF4) -> None: - """ A function to perform the reprojection of a single variable. The - reprojection information for each will be derived using interpolation - method specific functions, as will the calculation of reprojected - results. +def resample_variable( + message_parameters: Dict, + full_variable: str, + reprojection_cache: Dict, + variable_output_path: str, + logger: Logger, + var_info: VarInfoFromNetCDF4, +) -> None: + """A function to perform the reprojection of a single variable. The + reprojection information for each will be derived using interpolation + method specific functions, as will the calculation of reprojected + results. - Reprojection information will be stored in a cache, enabling it to be - recalled, rather than re-derived for subsequent science variables that - share the same coordinate variables. + Reprojection information will be stored in a cache, enabling it to be + recalled, rather than re-derived for subsequent science variables that + share the same coordinate variables. """ - interpolation_functions = ( - get_resampling_functions()[message_parameters['interpolation']] - ) + interpolation_functions = get_resampling_functions()[ + message_parameters['interpolation'] + ] dataset = Dataset(message_parameters['input_file']) variable = dataset[full_variable] # get variable with CF_Overrides and get real coordinates @@ -112,8 +122,9 @@ def resample_variable(message_parameters: Dict, full_variable: str, coordinates_key = create_coordinates_key(variable_cf) if coordinates_key in reprojection_cache: - logger.debug('Retrieving previous interpolation information for ' - f'{full_variable}') + logger.debug( + 'Retrieving previous interpolation information for ' f'{full_variable}' + ) reprojection_information = reprojection_cache[coordinates_key] else: logger.debug(f'Deriving interpolation information for {full_variable}') @@ -123,8 +134,9 @@ def resample_variable(message_parameters: Dict, full_variable: str, target_area = reprojection_cache[HARMONY_TARGET]['target_area'] else: logger.debug('Deriving target area from associated coordinates.') - target_area = get_target_area(message_parameters, dataset, - coordinates_key, logger) + target_area = get_target_area( + message_parameters, dataset, coordinates_key, logger + ) swath_definition = get_swath_definition(dataset, coordinates_key) @@ -148,46 +160,55 @@ def resample_variable(message_parameters: Dict, full_variable: str, 'fill_value': fill_value, } - results = interpolation_functions['get_results'](variable_information, - reprojection_information) + results = interpolation_functions['get_results']( + variable_information, reprojection_information + ) results = results.astype(variable.dtype) attributes = get_scale_and_offset(variable) - write_single_band_output(reprojection_information['target_area'], results, - full_variable, variable_output_path, - reprojection_cache, attributes) + write_single_band_output( + reprojection_information['target_area'], + results, + full_variable, + variable_output_path, + reprojection_cache, + attributes, + ) dataset.close() - logger.debug(f'Saved {full_variable} output to temporary file: ' - f'{variable_output_path}') + logger.debug( + f'Saved {full_variable} output to temporary file: ' f'{variable_output_path}' + ) -def get_bilinear_information(swath_definition: SwathDefinition, - target_area: AreaDefinition) -> Dict: - """ Return the necessary information to reproject a swath using the - bilinear interpolation method. This information will be stored in the - reprojection cache, for use with other science variables that share the - same coordinate variables. +def get_bilinear_information( + swath_definition: SwathDefinition, target_area: AreaDefinition +) -> Dict: + """Return the necessary information to reproject a swath using the + bilinear interpolation method. This information will be stored in the + reprojection cache, for use with other science variables that share the + same coordinate variables. """ - bilinear_information = get_bil_info(swath_definition, target_area, - radius=RADIUS_OF_INFLUENCE, - neighbours=NEIGHBOURS) + bilinear_information = get_bil_info( + swath_definition, target_area, radius=RADIUS_OF_INFLUENCE, neighbours=NEIGHBOURS + ) - return {'vertical_distances': bilinear_information[0], - 'horizontal_distances': bilinear_information[1], - 'valid_input_indices': bilinear_information[2], - 'valid_point_mapping': bilinear_information[3], - 'target_area': target_area} + return { + 'vertical_distances': bilinear_information[0], + 'horizontal_distances': bilinear_information[1], + 'valid_input_indices': bilinear_information[2], + 'valid_point_mapping': bilinear_information[3], + 'target_area': target_area, + } -def get_bilinear_results(variable: Dict, - bilinear_information: Dict) -> np.ndarray: - """ Use the derived information from the input swath and target area to - reproject variable data in the target area using the bilinear - interpolation method. Any pixels with NaN values after reprojection are - set to the fill value for the variable. +def get_bilinear_results(variable: Dict, bilinear_information: Dict) -> np.ndarray: + """Use the derived information from the input swath and target area to + reproject variable data in the target area using the bilinear + interpolation method. Any pixels with NaN values after reprojection are + set to the fill value for the variable. """ results = get_sample_from_bil_info( @@ -196,7 +217,7 @@ def get_bilinear_results(variable: Dict, bilinear_information['horizontal_distances'], bilinear_information['valid_input_indices'], bilinear_information['valid_point_mapping'], - output_shape=bilinear_information['target_area'].shape + output_shape=bilinear_information['target_area'].shape, ) if variable['fill_value'] is not None: @@ -205,35 +226,35 @@ def get_bilinear_results(variable: Dict, return results -def get_ewa_information(swath_definition: SwathDefinition, - target_area: AreaDefinition) -> Dict: - """ Return the necessary information to reproject a swath using the - Elliptically Weighted Average interpolation method. This information - will be stored in the reprojection cache, for use with other science - variables that share the same coordinate variables. +def get_ewa_information( + swath_definition: SwathDefinition, target_area: AreaDefinition +) -> Dict: + """Return the necessary information to reproject a swath using the + Elliptically Weighted Average interpolation method. This information + will be stored in the reprojection cache, for use with other science + variables that share the same coordinate variables. """ ewa_info = ll2cr(swath_definition, target_area) - return {'columns': ewa_info[1], - 'rows': ewa_info[2], - 'target_area': target_area} + return {'columns': ewa_info[1], 'rows': ewa_info[2], 'target_area': target_area} -def get_ewa_results(variable: Dict, ewa_information: Dict, - maximum_weight_mode: bool) -> np.ndarray: - """ Use the derived information from the input swath and target area to - reproject variable data in the target area using the Elliptically - Weighted Average interpolation. This also includes the flag for whether - to use the maximum weight mode. +def get_ewa_results( + variable: Dict, ewa_information: Dict, maximum_weight_mode: bool +) -> np.ndarray: + """Use the derived information from the input swath and target area to + reproject variable data in the target area using the Elliptically + Weighted Average interpolation. This also includes the flag for whether + to use the maximum weight mode. - If maximum_weight_mode is False, a weighted average of all swath cells - that map to a particular grid cell is used. If True, the swath cell - having the maximum weight of all swath cells that map to a particular - grid cell is used, instead of a weighted average. This is a - 'nearest-neighbour' style interpolation, but accounts for pixels within - the same scan line being more closely related than those from different - scans. + If maximum_weight_mode is False, a weighted average of all swath cells + that map to a particular grid cell is used. If True, the swath cell + having the maximum weight of all swath cells that map to a particular + grid cell is used, instead of a weighted average. This is a + 'nearest-neighbour' style interpolation, but accounts for pixels within + the same scan line being more closely related than those from different + scans. """ if np.issubdtype(variable['values'].dtype, np.integer): @@ -241,9 +262,13 @@ def get_ewa_results(variable: Dict, ewa_information: Dict, # This call falls back on the EWA rows_per_scan default of total input rows # and ignores the quality status return value - _, results = fornav(ewa_information['columns'], ewa_information['rows'], - ewa_information['target_area'], variable['values'], - maximum_weight_mode=maximum_weight_mode) + _, results = fornav( + ewa_information['columns'], + ewa_information['rows'], + ewa_information['target_area'], + variable['values'], + maximum_weight_mode=maximum_weight_mode, + ) if variable['fill_value'] is not None: np.nan_to_num(results, nan=variable['fill_value'], copy=False) @@ -251,38 +276,47 @@ def get_ewa_results(variable: Dict, ewa_information: Dict, return results -def get_near_information(swath_definition: SwathDefinition, - target_area: AreaDefinition) -> Dict: - """ Return the necessary information to reproject a swath using the - nearest neighbour interpolation method. This information will be stored - in the reprojection cache, for use with other science variables that - share the same coordinate variables. +def get_near_information( + swath_definition: SwathDefinition, target_area: AreaDefinition +) -> Dict: + """Return the necessary information to reproject a swath using the + nearest neighbour interpolation method. This information will be stored + in the reprojection cache, for use with other science variables that + share the same coordinate variables. """ - near_information = get_neighbour_info(swath_definition, target_area, - RADIUS_OF_INFLUENCE, - epsilon=EPSILON, neighbours=1) + near_information = get_neighbour_info( + swath_definition, + target_area, + RADIUS_OF_INFLUENCE, + epsilon=EPSILON, + neighbours=1, + ) - return {'valid_input_index': near_information[0], - 'valid_output_index': near_information[1], - 'index_array': near_information[2], - 'distance_array': near_information[3], - 'target_area': target_area} + return { + 'valid_input_index': near_information[0], + 'valid_output_index': near_information[1], + 'index_array': near_information[2], + 'distance_array': near_information[3], + 'target_area': target_area, + } def get_near_results(variable: Dict, near_information) -> np.ndarray: - """ Use the derived information from the input swath and target area to - reproject variable data in the target area using the nearest neighbour - interpolation method. + """Use the derived information from the input swath and target area to + reproject variable data in the target area using the nearest neighbour + interpolation method. """ results = get_sample_from_neighbour_info( - 'nn', near_information['target_area'].shape, variable['values'], + 'nn', + near_information['target_area'].shape, + variable['values'], near_information['valid_input_index'], near_information['valid_output_index'], near_information['index_array'], distance_array=near_information['distance_array'], - fill_value=variable['fill_value'] + fill_value=variable['fill_value'], ) if len(results.shape) == 3: @@ -293,56 +327,58 @@ def get_near_results(variable: Dict, near_information) -> np.ndarray: def get_resampling_functions() -> Dict: - """ Return a mapping of interpolation options to resampling functions. This - dictionary is an alternative to using a four branched if, elif, else - condition for both retrieving reprojection information and reprojected - data. + """Return a mapping of interpolation options to resampling functions. This + dictionary is an alternative to using a four branched if, elif, else + condition for both retrieving reprojection information and reprojected + data. """ return { 'bilinear': { 'get_information': get_bilinear_information, - 'get_results': get_bilinear_results + 'get_results': get_bilinear_results, }, 'ewa': { 'get_information': get_ewa_information, - 'get_results': partial(get_ewa_results, maximum_weight_mode=False) + 'get_results': partial(get_ewa_results, maximum_weight_mode=False), }, 'ewa-nn': { 'get_information': get_ewa_information, - 'get_results': partial(get_ewa_results, maximum_weight_mode=True) + 'get_results': partial(get_ewa_results, maximum_weight_mode=True), }, 'near': { 'get_information': get_near_information, - 'get_results': get_near_results - } + 'get_results': get_near_results, + }, } -def check_for_valid_interpolation(message_parameters: Dict, - logger: Logger) -> None: - """ Ensure the interpolation supplied in the message parameters is one of - the expected options. +def check_for_valid_interpolation(message_parameters: Dict, logger: Logger) -> None: + """Ensure the interpolation supplied in the message parameters is one of + the expected options. """ resampling_functions = get_resampling_functions() if message_parameters['interpolation'] not in resampling_functions: - valid_interpolations = ', '.join([f'"{interpolation}"' - for interpolation - in resampling_functions]) + valid_interpolations = ', '.join( + [f'"{interpolation}"' for interpolation in resampling_functions] + ) - logger.error(f'Interpolation option "{message_parameters["interpolation"]}" ' - f'must be one of {valid_interpolations}.') - raise ValueError('Invalid value for interpolation type: ' - f'"{message_parameters["interpolation"]}".') + logger.error( + f'Interpolation option "{message_parameters["interpolation"]}" ' + f'must be one of {valid_interpolations}.' + ) + raise ValueError( + 'Invalid value for interpolation type: ' + f'"{message_parameters["interpolation"]}".' + ) -def get_swath_definition(dataset: Dataset, - coordinates: Tuple[str]) -> SwathDefinition: - """ Define the swath as specified by the associated longitude and latitude - datasets. Note, the longitudes must be wrapped to the range: - -180 < longitude < 180. +def get_swath_definition(dataset: Dataset, coordinates: Tuple[str]) -> SwathDefinition: + """Define the swath as specified by the associated longitude and latitude + datasets. Note, the longitudes must be wrapped to the range: + -180 < longitude < 180. """ latitudes = get_coordinate_variable(dataset, coordinates, 'lat') @@ -359,23 +395,23 @@ def get_swath_definition(dataset: Dataset, def get_reprojection_cache(parameters: Dict) -> Dict: - """ Return a cache for information to be shared between all variables with - common coordinates. Additionally, check the input Harmony message for a - complete definition of the target area. If that is present, return it - in the initial cache under a key that should not be match a valid - variable name in the input granule. + """Return a cache for information to be shared between all variables with + common coordinates. Additionally, check the input Harmony message for a + complete definition of the target area. If that is present, return it + in the initial cache under a key that should not be match a valid + variable name in the input granule. """ reprojection_cache = {} - grid_extents = get_parameters_tuple(parameters, - ['x_min', 'y_min', 'x_max', 'y_max']) + grid_extents = get_parameters_tuple( + parameters, ['x_min', 'y_min', 'x_max', 'y_max'] + ) dimensions = get_parameters_tuple(parameters, ['height', 'width']) resolutions = get_parameters_tuple(parameters, ['xres', 'yres']) projection_string = parameters['projection'].definition_string() - if grid_extents is not None and (dimensions is not None - or resolutions is not None): + if grid_extents is not None and (dimensions is not None or resolutions is not None): x_range = grid_extents[2] - grid_extents[0] y_range = grid_extents[1] - grid_extents[3] @@ -387,25 +423,26 @@ def get_reprojection_cache(parameters: Dict) -> Dict: dimensions = (height, width) - target_area = AreaDefinition.from_extent(HARMONY_TARGET, - projection_string, - dimensions, - grid_extents) + target_area = AreaDefinition.from_extent( + HARMONY_TARGET, projection_string, dimensions, grid_extents + ) reprojection_cache[HARMONY_TARGET] = {'target_area': target_area} return reprojection_cache -def get_target_area(parameters: Dict, dataset: Dataset, - coordinates: Tuple[str], logger: Logger) -> AreaDefinition: - """ Define the target area as specified by either a complete set of message - parameters, or supplemented with coordinate variables as referred to in - the science variable metadata. +def get_target_area( + parameters: Dict, dataset: Dataset, coordinates: Tuple[str], logger: Logger +) -> AreaDefinition: + """Define the target area as specified by either a complete set of message + parameters, or supplemented with coordinate variables as referred to in + the science variable metadata. """ - grid_extents = get_parameters_tuple(parameters, - ['x_min', 'y_min', 'x_max', 'y_max']) + grid_extents = get_parameters_tuple( + parameters, ['x_min', 'y_min', 'x_max', 'y_max'] + ) dimensions = get_parameters_tuple(parameters, ['height', 'width']) resolutions = get_parameters_tuple(parameters, ['xres', 'yres']) projection_string = parameters['projection'].definition_string() @@ -413,10 +450,12 @@ def get_target_area(parameters: Dict, dataset: Dataset, longitudes = get_coordinate_variable(dataset, coordinates, 'lon') if grid_extents is not None: - logger.info(f'Message x extent: x_min: {grid_extents[0]}, x_max: ' - f'{grid_extents[2]}') - logger.info(f'Message y extent: y_min: {grid_extents[1]}, y_max: ' - f'{grid_extents[3]}') + logger.info( + f'Message x extent: x_min: {grid_extents[0]}, x_max: ' f'{grid_extents[2]}' + ) + logger.info( + f'Message y extent: y_min: {grid_extents[1]}, y_max: ' f'{grid_extents[3]}' + ) else: x_min, x_max, y_min, y_max = get_extents_from_perimeter( parameters['projection'], longitudes, latitudes @@ -432,15 +471,17 @@ def get_target_area(parameters: Dict, dataset: Dataset, if resolutions is None and dimensions is not None: resolutions = (x_range / dimensions[1], y_range / dimensions[0]) elif resolutions is None: - x_res = get_projected_resolution(parameters['projection'], longitudes, - latitudes) + x_res = get_projected_resolution( + parameters['projection'], longitudes, latitudes + ) # TODO: Determine sign of y resolution from projected y data. y_res = -1.0 * x_res resolutions = (x_res, y_res) logger.info(f'Calculated projected resolutions: ({x_res}, {y_res})') else: - logger.info(f'Resolutions from message: ({resolutions[0]}, ' - f'{resolutions[1]})') + logger.info( + f'Resolutions from message: ({resolutions[0]}, ' f'{resolutions[1]})' + ) if dimensions is None: width = abs(round(x_range / resolutions[0])) @@ -449,24 +490,27 @@ def get_target_area(parameters: Dict, dataset: Dataset, logger.info(f'Calculated height: {height}') dimensions = (height, width) - return AreaDefinition.from_extent(', '.join(coordinates), - projection_string, dimensions, - grid_extents) + return AreaDefinition.from_extent( + ', '.join(coordinates), projection_string, dimensions, grid_extents + ) -def get_parameters_tuple(input_parameters: Dict, - output_parameter_keys: List) -> Optional[Tuple]: - """ Search the input Harmony message for the listed keys. If all of them - are valid, return the parameter values, in the order originally listed. - If any of the parameters are invalid, return `None`. +def get_parameters_tuple( + input_parameters: Dict, output_parameter_keys: List +) -> Optional[Tuple]: + """Search the input Harmony message for the listed keys. If all of them + are valid, return the parameter values, in the order originally listed. + If any of the parameters are invalid, return `None`. - This is specifically used to check all extent parameters (e.g. `x_min`, - `x_max`, `y_min` and `y_max`), dimensions (e.g. `height` and `width`) - or resolutions (e.g. `xres` and `yres`) are *all* valid. + This is specifically used to check all extent parameters (e.g. `x_min`, + `x_max`, `y_min` and `y_max`), dimensions (e.g. `height` and `width`) + or resolutions (e.g. `xres` and `yres`) are *all* valid. """ - output_values = tuple(input_parameters[output_parameter_key] - for output_parameter_key in output_parameter_keys) + output_values = tuple( + input_parameters[output_parameter_key] + for output_parameter_key in output_parameter_keys + ) if any((output_value is None for output_value in output_values)): output_values = None diff --git a/swath_projector/nc_merge.py b/swath_projector/nc_merge.py index e678167..f6b2ae6 100644 --- a/swath_projector/nc_merge.py +++ b/swath_projector/nc_merge.py @@ -2,6 +2,7 @@ `pyresample`, back into a single output file with all the necessary attributes. """ + from datetime import datetime, timezone from typing import Dict, Optional, Set, Tuple, Union import json @@ -13,28 +14,33 @@ import numpy as np from swath_projector.exceptions import MissingReprojectedDataError -from swath_projector.utilities import ( - get_variable_file_path, - variable_in_dataset -) +from swath_projector.utilities import get_variable_file_path, variable_in_dataset # Values needed for history_json attribute -HISTORY_JSON_SCHEMA = 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json' +HISTORY_JSON_SCHEMA = ( + 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json' +) PROGRAM = 'sds/harmony-swath-projector' PROGRAM_REF = 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST' VERSION = '0.9.0' -def create_output(request_parameters: dict, output_file: str, temp_dir: str, - science_variables: Set[str], metadata_variables: Set[str], - logger: logging.Logger, var_info: VarInfoFromNetCDF4) -> None: - """ Merge the reprojected single-dataset NetCDF-4 files from `pyresample` - into a single file, copying global attributes and metadata - variables (those without coordinates, which therefore can't be - reprojected) from the original input file. Then for each listed science - variable, retrieve the single-band file and copy the reprojected - variables, and any accompanying CRS and coordinate variables. Note, the - coordinate datasets will only be copied once. +def create_output( + request_parameters: dict, + output_file: str, + temp_dir: str, + science_variables: Set[str], + metadata_variables: Set[str], + logger: logging.Logger, + var_info: VarInfoFromNetCDF4, +) -> None: + """Merge the reprojected single-dataset NetCDF-4 files from `pyresample` + into a single file, copying global attributes and metadata + variables (those without coordinates, which therefore can't be + reprojected) from the original input file. Then for each listed science + variable, retrieve the single-band file and copy the reprojected + variables, and any accompanying CRS and coordinate variables. Note, the + coordinate datasets will only be copied once. """ input_file = request_parameters.get('input_file') @@ -42,7 +48,7 @@ def create_output(request_parameters: dict, output_file: str, temp_dir: str, with ( Dataset(input_file) as input_dataset, - Dataset(output_file, 'w', format='NETCDF4') as output_dataset + Dataset(output_file, 'w', format='NETCDF4') as output_dataset, ): logger.info('Copying input file attributes to output file.') @@ -52,60 +58,71 @@ def create_output(request_parameters: dict, output_file: str, temp_dir: str, copy_time_dimension(input_dataset, output_dataset, logger) for metadata_variable in metadata_variables: - copy_metadata_variable(input_dataset, output_dataset, - metadata_variable, logger) + copy_metadata_variable( + input_dataset, output_dataset, metadata_variable, logger + ) output_extension = os.path.splitext(input_file)[1] for variable_name in science_variables: - dataset_file = get_variable_file_path(temp_dir, variable_name, - output_extension) + dataset_file = get_variable_file_path( + temp_dir, variable_name, output_extension + ) if os.path.isfile(dataset_file): with Dataset(dataset_file) as data: set_dimensions(data, output_dataset) - copy_science_variable(input_dataset, output_dataset, data, - variable_name, logger, var_info) + copy_science_variable( + input_dataset, + output_dataset, + data, + variable_name, + logger, + var_info, + ) # Copy supporting variables from the single band output: # the grid mapping, reprojected x and reprojected y. for variable_key in data.variables: if ( - variable_key not in output_dataset.variables - and variable_key != variable_name + variable_key not in output_dataset.variables + and variable_key != variable_name ): - copy_metadata_variable(data, output_dataset, - variable_key, logger) + copy_metadata_variable( + data, output_dataset, variable_key, logger + ) else: logger.error(f'Cannot find "{dataset_file}".') raise MissingReprojectedDataError(variable_name) -def set_output_attributes(input_dataset: Dataset, output_dataset: Dataset, - request_parameters: Dict) -> None: - """ Set the global attributes of the merged output file. These begin as the - global attributes of the input granule, but are updated to also include - the provenance data via an updated `history` CF attribute (or `History` - if that is already present), and a `history_json` attribute that is - compliant with the schema defined at the URL specified by - `HISTORY_JSON_SCHEMA`. +def set_output_attributes( + input_dataset: Dataset, output_dataset: Dataset, request_parameters: Dict +) -> None: + """Set the global attributes of the merged output file. These begin as the + global attributes of the input granule, but are updated to also include + the provenance data via an updated `history` CF attribute (or `History` + if that is already present), and a `history_json` attribute that is + compliant with the schema defined at the URL specified by + `HISTORY_JSON_SCHEMA`. - `projection` is not included in the output parameters, as this is not - an original message parameter. It is a derived `pyproj.Proj` instance - that is defined by the input `crs` parameter. + `projection` is not included in the output parameters, as this is not + an original message parameter. It is a derived `pyproj.Proj` instance + that is defined by the input `crs` parameter. - `x_extent` and `y_extent` are not serializable, and are instead - included by `x_min`, `x_max` and `y_min` `y_max` accordingly. + `x_extent` and `y_extent` are not serializable, and are instead + included by `x_min`, `x_max` and `y_min` `y_max` accordingly. """ output_attributes = read_attrs(input_dataset) - valid_request_parameters = {parameter_name: parameter_value - for parameter_name, parameter_value - in request_parameters.items() - if parameter_value is not None} + valid_request_parameters = { + parameter_name: parameter_value + for parameter_name, parameter_value in request_parameters.items() + if parameter_value is not None + } # Remove unnecessary and unserializable request parameters for surplus_key in ['projection', 'x_extent', 'y_extent']: @@ -123,8 +140,9 @@ def set_output_attributes(input_dataset: Dataset, output_dataset: Dataset, input_history = getattr(input_dataset, cf_att_name, None) # Create new history_json attribute - new_history_json_record = create_history_record(input_history, - valid_request_parameters) + new_history_json_record = create_history_record( + input_history, valid_request_parameters + ) # Extract existing `history_json` from input granule if hasattr(input_dataset, 'history_json'): @@ -142,15 +160,22 @@ def set_output_attributes(input_dataset: Dataset, output_dataset: Dataset, output_attributes['history_json'] = json.dumps(output_history_json) # Create history attribute - history_parameters = {parameter_name: parameter_value - for parameter_name, parameter_value - in new_history_json_record['parameters'].items() - if parameter_name != 'input_file'} + history_parameters = { + parameter_name: parameter_value + for parameter_name, parameter_value in new_history_json_record[ + 'parameters' + ].items() + if parameter_name != 'input_file' + } - new_history_line = ' '.join([new_history_json_record['date_time'], - new_history_json_record['program'], - new_history_json_record['version'], - json.dumps(history_parameters)]) + new_history_line = ' '.join( + [ + new_history_json_record['date_time'], + new_history_json_record['program'], + new_history_json_record['version'], + json.dumps(history_parameters), + ] + ) output_history = '\n'.join(filter(None, [input_history, new_history_line])) output_attributes[cf_att_name] = output_history @@ -159,8 +184,8 @@ def set_output_attributes(input_dataset: Dataset, output_dataset: Dataset, def create_history_record(input_history: str, request_parameters: dict) -> Dict: - """ Create a serializable dictionary for the `history_json` global - attribute in the merged output NetCDF-4 file. + """Create a serializable dictionary for the `history_json` global + attribute in the merged output NetCDF-4 file. """ history_record = { @@ -182,14 +207,15 @@ def create_history_record(input_history: str, request_parameters: dict) -> Dict: def read_attrs(dataset: Union[Dataset, Variable]) -> Dict: - """ Read attributes from either a NetCDF4 Dataset or variable object. """ + """Read attributes from either a NetCDF4 Dataset or variable object.""" return dataset.__dict__ -def copy_time_dimension(input_dataset: Dataset, output_dataset: Dataset, - logger: logging.Logger) -> None: - """ Add time dimension to the output file. This will first add a dimension, - before creating the corresponding variable in the output dataset. +def copy_time_dimension( + input_dataset: Dataset, output_dataset: Dataset, logger: logging.Logger +) -> None: + """Add time dimension to the output file. This will first add a dimension, + before creating the corresponding variable in the output dataset. """ logger.info('Adding "time" dimension.') @@ -199,8 +225,8 @@ def copy_time_dimension(input_dataset: Dataset, output_dataset: Dataset, def set_dimensions(input_dataset: Dataset, output_dataset: Dataset) -> None: - """ Read the dimensions in the single band intermediate file. Add each - dimension to the output dataset that is not already present. + """Read the dimensions in the single band intermediate file. Add each + dimension to the output dataset that is not already present. """ for name, dimension in input_dataset.dimensions.items(): @@ -208,31 +234,35 @@ def set_dimensions(input_dataset: Dataset, output_dataset: Dataset) -> None: output_dataset.createDimension(name, dimension.size) -def set_metadata_dimensions(metadata_variable: str, source_dataset: Dataset, - output_dataset: Dataset) -> None: - """ Iterate through the dimensions of the metadata variable, and ensure - that all are present in the reprojected output file. This function is - necessary if any of the metadata variables, that aren't to be projected - use the swath-based dimensions from the input granule. +def set_metadata_dimensions( + metadata_variable: str, source_dataset: Dataset, output_dataset: Dataset +) -> None: + """Iterate through the dimensions of the metadata variable, and ensure + that all are present in the reprojected output file. This function is + necessary if any of the metadata variables, that aren't to be projected + use the swath-based dimensions from the input granule. """ for dimension in source_dataset[metadata_variable].dimensions: if dimension not in output_dataset.dimensions: output_dataset.createDimension( - dimension, - source_dataset.dimensions[dimension].size + dimension, source_dataset.dimensions[dimension].size ) -def copy_metadata_variable(source_dataset: Dataset, output_dataset: Dataset, - variable_name: str, logger: logging.Logger) -> None: - """ Write a metadata variable directly from either the input dataset or a - single band dataset. The variables from the input dataset have not been - reprojected as they contain no references to coordinate datasets. The - variables from the single band datasets are the non-science variables, - e.g. the grid mapping and projected coordinates. In both instances, the - variable should be exactly copied from the source dataset to the - output. +def copy_metadata_variable( + source_dataset: Dataset, + output_dataset: Dataset, + variable_name: str, + logger: logging.Logger, +) -> None: + """Write a metadata variable directly from either the input dataset or a + single band dataset. The variables from the input dataset have not been + reprojected as they contain no references to coordinate datasets. The + variables from the single band datasets are the non-science variables, + e.g. the grid mapping and projected coordinates. In both instances, the + variable should be exactly copied from the source dataset to the + output. """ logger.info(f'Adding metadata variable "{variable_name}" to the output.') @@ -242,42 +272,53 @@ def copy_metadata_variable(source_dataset: Dataset, output_dataset: Dataset, fill_value = get_fill_value_from_attributes(attributes) output_dataset.createVariable( - variable_name, source_dataset[variable_name].datatype, + variable_name, + source_dataset[variable_name].datatype, dimensions=source_dataset[variable_name].dimensions, - fill_value=fill_value, zlib=True, complevel=6 + fill_value=fill_value, + zlib=True, + complevel=6, ) output_dataset[variable_name][:] = source_dataset[variable_name][:] output_dataset[variable_name].setncatts(attributes) -def copy_science_variable(input_dataset: Dataset, output_dataset: Dataset, - single_band_dataset: Dataset, variable_name: str, - logger: logging.Logger, var_info: VarInfoFromNetCDF4) -> None: - """ Write a reprojected variable from a single-band output file to the - merged output file. This will first obtain metadata (dimensions, - data type and attributes) from either the single-band output, or from - the original input file dataset. Then the variable values from the - single-band output are copied into the data arrays. If the dataset - attributes include a scale and offset, the output values are adjusted - accordingly. +def copy_science_variable( + input_dataset: Dataset, + output_dataset: Dataset, + single_band_dataset: Dataset, + variable_name: str, + logger: logging.Logger, + var_info: VarInfoFromNetCDF4, +) -> None: + """Write a reprojected variable from a single-band output file to the + merged output file. This will first obtain metadata (dimensions, + data type and attributes) from either the single-band output, or from + the original input file dataset. Then the variable values from the + single-band output are copied into the data arrays. If the dataset + attributes include a scale and offset, the output values are adjusted + accordingly. """ logger.info(f'Adding reprojected "{variable_name}" to the output') - dimensions = get_science_variable_dimensions(input_dataset, - single_band_dataset, - variable_name) - attributes = get_science_variable_attributes(input_dataset, - single_band_dataset, - variable_name, - var_info) + dimensions = get_science_variable_dimensions( + input_dataset, single_band_dataset, variable_name + ) + attributes = get_science_variable_attributes( + input_dataset, single_band_dataset, variable_name, var_info + ) fill_value = get_fill_value_from_attributes(attributes) variable = output_dataset.createVariable( - variable_name, input_dataset[variable_name].datatype, - dimensions=dimensions, fill_value=fill_value, zlib=True, complevel=6 + variable_name, + input_dataset[variable_name].datatype, + dimensions=dimensions, + fill_value=fill_value, + zlib=True, + complevel=6, ) # Extract the data from the single band image, and ensure it is correctly @@ -300,37 +341,37 @@ def copy_science_variable(input_dataset: Dataset, output_dataset: Dataset, variable.setncatts(attributes) -def get_science_variable_attributes(input_dataset: Dataset, - single_band_dataset: Dataset, - variable_name: str, - var_info: VarInfoFromNetCDF4) -> Dict: - """ Extract the attributes for a science variable, using a combination of - the original metadata from the unprojected input variable, and then - augmenting that with the grid_mapping of the reprojected data. Finally, - ensure the coordinate metadata are still valid. If not, remove that - metadata entry. +def get_science_variable_attributes( + input_dataset: Dataset, + single_band_dataset: Dataset, + variable_name: str, + var_info: VarInfoFromNetCDF4, +) -> Dict: + """Extract the attributes for a science variable, using a combination of + the original metadata from the unprojected input variable, and then + augmenting that with the grid_mapping of the reprojected data. Finally, + ensure the coordinate metadata are still valid. If not, remove that + metadata entry. """ variable_attributes = read_attrs(input_dataset[variable_name]) grid_mapping = single_band_dataset[variable_name].grid_mapping variable_attributes['grid_mapping'] = grid_mapping - if ( - 'coordinates' in variable_attributes - and not check_coor_valid(var_info, variable_name, - input_dataset, single_band_dataset) + if 'coordinates' in variable_attributes and not check_coor_valid( + var_info, variable_name, input_dataset, single_band_dataset ): del variable_attributes['coordinates'] return variable_attributes -def get_science_variable_dimensions(input_dataset: Dataset, - single_band_dataset: Dataset, - variable_name: str) -> Tuple[str]: - """ Retrieve the dimensions from the single-band reprojected dataset. If - the original input dataset has a 'time' dimension, then include that as - a dimension of the reprojected variable. +def get_science_variable_dimensions( + input_dataset: Dataset, single_band_dataset: Dataset, variable_name: str +) -> Tuple[str]: + """Retrieve the dimensions from the single-band reprojected dataset. If + the original input dataset has a 'time' dimension, then include that as + a dimension of the reprojected variable. """ if 'time' not in input_dataset.dimensions: @@ -341,20 +382,22 @@ def get_science_variable_dimensions(input_dataset: Dataset, return dimensions -def check_coor_valid(var_info: VarInfoFromNetCDF4, variable_name: str, - input_dataset: Dataset, - single_band_dataset: Dataset) -> bool: - """ Check if variables listed in the coordinates metadata attributes are - still valid after reprojection. Invalid coordinate reference cases: +def check_coor_valid( + var_info: VarInfoFromNetCDF4, + variable_name: str, + input_dataset: Dataset, + single_band_dataset: Dataset, +) -> bool: + """Check if variables listed in the coordinates metadata attributes are + still valid after reprojection. Invalid coordinate reference cases: - 1) Coordinate variable listed in attribute does not exist in single - band output dataset. - 2) Coordinate variable array shape in the reprojected, single-band - dataset does not match the input coordinate array shape. + 1) Coordinate variable listed in attribute does not exist in single + band output dataset. + 2) Coordinate variable array shape in the reprojected, single-band + dataset does not match the input coordinate array shape. """ - coords = var_info.get_variable(variable_name).references.get('coordinates', - []) + coords = var_info.get_variable(variable_name).references.get('coordinates', []) all_coordinates_in_single_band = all( variable_in_dataset(coord, single_band_dataset) for coord in coords @@ -365,16 +408,18 @@ def check_coor_valid(var_info: VarInfoFromNetCDF4, variable_name: str, # output (single band file). valid = False else: - valid = all(single_band_dataset[coord].shape == input_dataset[coord].shape - for coord in coords) + valid = all( + single_band_dataset[coord].shape == input_dataset[coord].shape + for coord in coords + ) return valid def get_fill_value_from_attributes(variable_attributes: Dict) -> Optional: - """ Check attributes for _FillValue. If present return the value and - remove the _FillValue attribute from the input dictionary. Otherwise - return None. + """Check attributes for _FillValue. If present return the value and + remove the _FillValue attribute from the input dictionary. Otherwise + return None. """ return variable_attributes.pop('_FillValue', None) diff --git a/swath_projector/nc_single_band.py b/swath_projector/nc_single_band.py index 3b447b1..a670918 100644 --- a/swath_projector/nc_single_band.py +++ b/swath_projector/nc_single_band.py @@ -22,6 +22,7 @@ - y (or lat if geographic). """ + from typing import Dict, Tuple from pyresample.geometry import AreaDefinition @@ -29,70 +30,87 @@ import numpy as np -DIMENSION_METADATA = {'lat': {'long_name': 'latitude', - 'standard_name': 'latitude', - 'units': 'degrees_north'}, - 'lon': {'long_name': 'longitude', - 'standard_name': 'longitude', - 'units': 'degrees_east'}, - 'x': {'long_name': 'x coordinate of projection', - 'standard_name': 'projection_x_coordinate', - 'units': 'm'}, - 'y': {'long_name': 'y coordinate of projection', - 'standard_name': 'projection_y_coordinate', - 'units': 'm'}} +DIMENSION_METADATA = { + 'lat': { + 'long_name': 'latitude', + 'standard_name': 'latitude', + 'units': 'degrees_north', + }, + 'lon': { + 'long_name': 'longitude', + 'standard_name': 'longitude', + 'units': 'degrees_east', + }, + 'x': { + 'long_name': 'x coordinate of projection', + 'standard_name': 'projection_x_coordinate', + 'units': 'm', + }, + 'y': { + 'long_name': 'y coordinate of projection', + 'standard_name': 'projection_y_coordinate', + 'units': 'm', + }, +} HARMONY_TARGET = 'harmony_message_target' -def write_single_band_output(target_area: AreaDefinition, - reprojected_data: np.ndarray, - variable_name: str, - variable_output_path: str, - reprojection_cache: Dict, - attributes: Dict) -> None: - """ The main interface for this module. Each single band output file - will contain the following properties: - - - A reprojected, 2-dimensional science variable. - - Projected x and y dimensions. - - A 1-dimensional variable for each of the associated dimensions. The - science variable should use these as its dimensions. - - A grid mapping variable, with metadata conforming to the CF - Conventions. The science variable should refer to this in its - metadata. +def write_single_band_output( + target_area: AreaDefinition, + reprojected_data: np.ndarray, + variable_name: str, + variable_output_path: str, + reprojection_cache: Dict, + attributes: Dict, +) -> None: + """The main interface for this module. Each single band output file + will contain the following properties: + + - A reprojected, 2-dimensional science variable. + - Projected x and y dimensions. + - A 1-dimensional variable for each of the associated dimensions. The + science variable should use these as its dimensions. + - A grid mapping variable, with metadata conforming to the CF + Conventions. The science variable should refer to this in its + metadata. """ with Dataset(variable_output_path, 'w', format='NETCDF4') as output_file: - dimensions = write_dimensions(output_file, target_area, - reprojection_cache) - grid_mapping_name = write_grid_mapping(output_file, target_area, - dimensions) - write_science_variable(output_file, reprojected_data, variable_name, - dimensions, grid_mapping_name, attributes) + dimensions = write_dimensions(output_file, target_area, reprojection_cache) + grid_mapping_name = write_grid_mapping(output_file, target_area, dimensions) + write_science_variable( + output_file, + reprojected_data, + variable_name, + dimensions, + grid_mapping_name, + attributes, + ) write_dimension_variables(output_file, dimensions, target_area) -def write_dimensions(dataset: Dataset, target_area: AreaDefinition, - cache: Dict) -> Tuple[str]: - """ Derive the dimension names using the target area definition and the - information available in the reprojection cache. Then write the - dimensions to the output dataset. Finally, return the dimension names - for later use; e.g. defining the grid mapping name and writing the - dimension variables themselves. - - Possible use-cases: - - - The Harmony message fully defines a target area. All science - variables will use this, and the dimensions can just be ('y', 'x') or - ('lat', 'lon'). - - The Harmony message does not fully define a target area, but the - reprojection cache only has one key. The dimensions should be - ('y', 'x') or ('lat', 'lon'). - - The Harmony message does not fully define a target area, and the - reprojection cache has multiple keys. This means there are multiple - target grids. The dimensions for all grids after the first should - have a suffix added, so that they can all be included in the merged - output. +def write_dimensions( + dataset: Dataset, target_area: AreaDefinition, cache: Dict +) -> Tuple[str]: + """Derive the dimension names using the target area definition and the + information available in the reprojection cache. Then write the + dimensions to the output dataset. Finally, return the dimension names + for later use; e.g. defining the grid mapping name and writing the + dimension variables themselves. + + Possible use-cases: + + - The Harmony message fully defines a target area. All science + variables will use this, and the dimensions can just be ('y', 'x') or + ('lat', 'lon'). + - The Harmony message does not fully define a target area, but the + reprojection cache only has one key. The dimensions should be + ('y', 'x') or ('lat', 'lon'). + - The Harmony message does not fully define a target area, and the + reprojection cache has multiple keys. This means there are multiple + target grids. The dimensions for all grids after the first should + have a suffix added, so that they can all be included in the merged + output. """ coordinates_key = tuple(target_area.area_id.split(', ')) @@ -138,24 +156,25 @@ def write_dimensions(dataset: Dataset, target_area: AreaDefinition, return (y_dim, x_dim) -def write_grid_mapping(dataset: Dataset, target_area: AreaDefinition, - dimensions: Tuple[str]) -> str: - """ Use the `pyresample.geometry.AreaDefition` instance associated with - the target area of reprojection to write out a `grid_mapping` - variable to the single band output `netCDF4.Dataset`. +def write_grid_mapping( + dataset: Dataset, target_area: AreaDefinition, dimensions: Tuple[str] +) -> str: + """Use the `pyresample.geometry.AreaDefition` instance associated with + the target area of reprojection to write out a `grid_mapping` + variable to the single band output `netCDF4.Dataset`. - Return the grid_mapping name for use as the `grid_mapping_name` of - the science variable. + Return the grid_mapping name for use as the `grid_mapping_name` of + the science variable. - In the instance that there are multiple grids in the output file, - which would result from multiple input grids, this function will - use the extended form of `grid_mapping_name`. This format should be: + In the instance that there are multiple grids in the output file, + which would result from multiple input grids, this function will + use the extended form of `grid_mapping_name`. This format should be: - ': []' + ': []' - See section 5.6 of the CF-Conventions for more information: + See section 5.6 of the CF-Conventions for more information: - http://cfconventions.org/cf-conventions/cf-conventions.html + http://cfconventions.org/cf-conventions/cf-conventions.html """ grid_mapping_attributes = target_area.crs.to_cf() @@ -166,9 +185,9 @@ def write_grid_mapping(dataset: Dataset, target_area: AreaDefinition, # Check if there are multiple grids and, if so, use the extended format of # grid mapping name. if dimensions not in [('lat', 'lon'), ('y', 'x')]: - grid_mapping_attributes['grid_mapping_name'] += ( - f'_{dimensions[0]}_{dimensions[1]}' - ) + grid_mapping_attributes[ + 'grid_mapping_name' + ] += f'_{dimensions[0]}_{dimensions[1]}' grid_mapping = dataset.createVariable( grid_mapping_attributes['grid_mapping_name'], 'S1' @@ -178,53 +197,61 @@ def write_grid_mapping(dataset: Dataset, target_area: AreaDefinition, return grid_mapping_attributes.get('grid_mapping_name') -def write_science_variable(dataset: Dataset, data_values: np.ndarray, - variable_full_name: str, dimensions: Tuple[str], - grid_mapping_name: str, attributes: Dict) -> None: - """ Add the science variable to the output `netCDF4.Dataset` instance. - This variable will require: +def write_science_variable( + dataset: Dataset, + data_values: np.ndarray, + variable_full_name: str, + dimensions: Tuple[str], + grid_mapping_name: str, + attributes: Dict, +) -> None: + """Add the science variable to the output `netCDF4.Dataset` instance. + This variable will require: - - The reprojected values to be assigned to the variable array. - - The reprojected dimensions to be associated with the variable. - - The `grid_mapping_name` to be included as an attribute. - - Scaling metadata attributes, if present on the input value. + - The reprojected values to be assigned to the variable array. + - The reprojected dimensions to be associated with the variable. + - The `grid_mapping_name` to be included as an attribute. + - Scaling metadata attributes, if present on the input value. - Note, the `netCDF4` library automatically applied the `add_offset` and - `scale_factor` keywords on reading and writing of `Variable` objects. + Note, the `netCDF4` library automatically applied the `add_offset` and + `scale_factor` keywords on reading and writing of `Variable` objects. """ - variable = dataset.createVariable(variable_full_name, data_values.dtype, - dimensions=dimensions) + variable = dataset.createVariable( + variable_full_name, data_values.dtype, dimensions=dimensions + ) attributes['grid_mapping'] = grid_mapping_name variable.setncatts(attributes) variable[:] = data_values[:] -def write_dimension_variables(dataset: Dataset, dimensions: Tuple[str], - target_area: AreaDefinition) -> None: - """ Write projected x and y coordinate information to the `netCDF4.Dataset` - instance, each as a `netCDF4.Variable`. Each dimension variable - should have: +def write_dimension_variables( + dataset: Dataset, dimensions: Tuple[str], target_area: AreaDefinition +) -> None: + """Write projected x and y coordinate information to the `netCDF4.Dataset` + instance, each as a `netCDF4.Variable`. Each dimension variable + should have: - - A 1-dimension array of the projected values of that dimension. - - A reference to itself as a dimension. - - Metadata that includes the dimension variable's name and units. + - A 1-dimension array of the projected values of that dimension. + - A reference to itself as a dimension. + - Metadata that includes the dimension variable's name and units. """ x_vector, y_vector = target_area.get_proj_vectors() dimension_data = {dimensions[0]: y_vector, dimensions[1]: x_vector} for dimension_name, dimension_vector in dimension_data.items(): - variable = dataset.createVariable(dimension_name, - dimension_vector.dtype, - dimensions=(dimension_name,)) + variable = dataset.createVariable( + dimension_name, dimension_vector.dtype, dimensions=(dimension_name,) + ) variable[:] = dimension_vector - attributes = next(attributes - for coordinate, attributes - in DIMENSION_METADATA.items() - if dimension_name.startswith(coordinate)) + attributes = next( + attributes + for coordinate, attributes in DIMENSION_METADATA.items() + if dimension_name.startswith(coordinate) + ) variable.setncatts(attributes) diff --git a/swath_projector/reproject.py b/swath_projector/reproject.py index a929cef..327e9ce 100644 --- a/swath_projector/reproject.py +++ b/swath_projector/reproject.py @@ -1,4 +1,5 @@ """ Data Services Swath Projector service for Harmony """ + from tempfile import mkdtemp from typing import Dict import functools @@ -13,19 +14,26 @@ from swath_projector.interpolation import resample_all_variables -RADIUS_EARTH_METRES = 6_378_137 # http://nssdc.gsfc.nasa.gov/planetary/factsheet/earthfact.html +RADIUS_EARTH_METRES = ( + 6_378_137 # http://nssdc.gsfc.nasa.gov/planetary/factsheet/earthfact.html +) CRS_DEFAULT = '+proj=longlat +ellps=WGS84' INTERPOLATION_DEFAULT = 'ewa-nn' CF_CONFIG_FILE = 'swath_projector/cf_config.json' -def reproject(message: Message, collection_short_name: str, granule_url: str, - local_filename: str, temp_dir: str, - logger: logging.Logger) -> str: - """ Derive reprojection parameters from the input Harmony message. Then - extract listing of science variables and coordinate variables from the - source granule. Then reproject all science variables. Finally merge all - individual output bands back into a single NetCDF-4 file. +def reproject( + message: Message, + collection_short_name: str, + granule_url: str, + local_filename: str, + temp_dir: str, + logger: logging.Logger, +) -> str: + """Derive reprojection parameters from the input Harmony message. Then + extract listing of science variables and coordinate variables from the + source granule. Then reproject all science variables. Finally merge all + individual output bands back into a single NetCDF-4 file. """ parameters = get_parameters_from_message(message, granule_url, local_filename) @@ -36,13 +44,17 @@ def reproject(message: Message, collection_short_name: str, granule_url: str, output_file = temp_dir + os.sep + root_ext[0] + '_repr' + root_ext[1] logger.info(f'Reprojecting file {parameters.get("input_file")} as {output_file}') - logger.info(f'Selected CRS: {parameters.get("crs")}\t' - f'Interpolation: {parameters.get("interpolation")}') + logger.info( + f'Selected CRS: {parameters.get("crs")}\t' + f'Interpolation: {parameters.get("interpolation")}' + ) try: - var_info = VarInfoFromNetCDF4(parameters['input_file'], - short_name=collection_short_name, - config_file=CF_CONFIG_FILE) + var_info = VarInfoFromNetCDF4( + parameters['input_file'], + short_name=collection_short_name, + config_file=CF_CONFIG_FILE, + ) except Exception as err: logger.error(f'Unable to parse input file variables: {str(err)}') raise Exception('Unable to parse input file variables') from err @@ -56,38 +68,48 @@ def reproject(message: Message, collection_short_name: str, granule_url: str, # Loop through each dataset and reproject logger.debug('Using pyresample for reprojection.') - outputs = resample_all_variables(parameters, science_variables, temp_dir, - logger, var_info) + outputs = resample_all_variables( + parameters, science_variables, temp_dir, logger, var_info + ) if not outputs: raise Exception('No variables could be reprojected') # Now merge outputs (unless we only have one) metadata_variables = var_info.get_metadata_variables() - nc_merge.create_output(parameters, output_file, temp_dir, - science_variables, metadata_variables, logger, var_info) + nc_merge.create_output( + parameters, + output_file, + temp_dir, + science_variables, + metadata_variables, + logger, + var_info, + ) # Return the output file back to Harmony return output_file -def get_parameters_from_message(message: Message, granule_url: str, - input_file: str) -> Dict: - """ A helper function to parse the input Harmony message and extract - required information. If the message is missing parameters, then - default values will be used. The `granule_url` is taken from the - inbound STAC `Item.asset.href`, denoting a URL to the original source - of the input granule. The `input_file` is the local path of that input - granule, as downloaded by `harmony-service-lib-py` utility functions - for transformation by this service. +def get_parameters_from_message( + message: Message, granule_url: str, input_file: str +) -> Dict: + """A helper function to parse the input Harmony message and extract + required information. If the message is missing parameters, then + default values will be used. The `granule_url` is taken from the + inbound STAC `Item.asset.href`, denoting a URL to the original source + of the input granule. The `input_file` is the local path of that input + granule, as downloaded by `harmony-service-lib-py` utility functions + for transformation by this service. """ parameters = { 'crs': rgetattr(message, 'format.crs', CRS_DEFAULT), 'granule_url': granule_url, 'input_file': input_file, - 'interpolation': rgetattr(message, 'format.interpolation', - INTERPOLATION_DEFAULT), + 'interpolation': rgetattr( + message, 'format.interpolation', INTERPOLATION_DEFAULT + ), 'x_extent': rgetattr(message, 'format.scaleExtent.x', None), 'y_extent': rgetattr(message, 'format.scaleExtent.y', None), 'width': rgetattr(message, 'format.width', None), @@ -102,12 +124,13 @@ def get_parameters_from_message(message: Message, granule_url: str, parameters['interpolation'] = INTERPOLATION_DEFAULT # ERROR 5: -tr and -ts options cannot be used at the same time. - if ( - (parameters['xres'] is not None or parameters['yres'] is not None) - and (parameters['height'] is not None or parameters['width'] is not None) + if (parameters['xres'] is not None or parameters['yres'] is not None) and ( + parameters['height'] is not None or parameters['width'] is not None ): - raise Exception('"scaleSize", "width" or/and "height" cannot ' - 'be used at the same time in the message.') + raise Exception( + '"scaleSize", "width" or/and "height" cannot ' + 'be used at the same time in the message.' + ) if not os.path.isfile(parameters['input_file']): raise Exception('Input file does not exist') @@ -130,17 +153,18 @@ def get_parameters_from_message(message: Message, granule_url: str, # Mark the properties that this service will use, so that downstream # services will not re-use them. - message.format.process('crs', 'interpolation', 'scaleExtent', 'scaleSize', - 'height', 'width') + message.format.process( + 'crs', 'interpolation', 'scaleExtent', 'scaleSize', 'height', 'width' + ) return parameters def rgetattr(obj, attr: str, *args): - """ Recursive get attribute. Returns attribute from an attribute hierarchy, - e.g. a.b.c, if it exists. If it doesn't exist, the default value will - be assigned. Even though the `args` is often optional, in this case the - default value *must* be defined. + """Recursive get attribute. Returns attribute from an attribute hierarchy, + e.g. a.b.c, if it exists. If it doesn't exist, the default value will + be assigned. Even though the `args` is often optional, in this case the + default value *must* be defined. """ diff --git a/swath_projector/swath_geometry.py b/swath_projector/swath_geometry.py index be6c7a8..83886c6 100644 --- a/swath_projector/swath_geometry.py +++ b/swath_projector/swath_geometry.py @@ -3,6 +3,7 @@ (CRS). """ + from typing import List, Tuple import functools @@ -11,19 +12,20 @@ import numpy as np -def get_projected_resolution(projection: Proj, longitudes: Variable, - latitudes: Variable) -> Tuple[float]: - """ Find the resolution of the target grid in the projected coordinates, x - and y. First the perimeter points are found. These are then projected - to the target CRS. Gauss' Area formula is then applied to find the area - of the swath in the target CRS. This is assumed to be equally shared - between input pixels. The pixels are also assumed to be square. +def get_projected_resolution( + projection: Proj, longitudes: Variable, latitudes: Variable +) -> Tuple[float]: + """Find the resolution of the target grid in the projected coordinates, x + and y. First the perimeter points are found. These are then projected + to the target CRS. Gauss' Area formula is then applied to find the area + of the swath in the target CRS. This is assumed to be equally shared + between input pixels. The pixels are also assumed to be square. """ coordinates_mask = get_valid_coordinates_mask(longitudes, latitudes) - x_values, y_values = get_projected_coordinates(coordinates_mask, - projection, longitudes, - latitudes) + x_values, y_values = get_projected_coordinates( + coordinates_mask, projection, longitudes, latitudes + ) if len(longitudes.shape) == 1: absolute_resolution = get_one_dimensional_resolution(x_values, y_values) @@ -31,141 +33,154 @@ def get_projected_resolution(projection: Proj, longitudes: Variable, ordered_x, ordered_y = sort_perimeter_points(x_values, y_values) projected_area = get_polygon_area(ordered_x, ordered_y) absolute_resolution = get_absolute_resolution( - projected_area, - coordinates_mask.count() # pylint: disable=E1101 + projected_area, coordinates_mask.count() # pylint: disable=E1101 ) return absolute_resolution -def get_extents_from_perimeter(projection: Proj, longitudes: Variable, - latitudes: Variable) -> Tuple[float]: - """ Find the swath extents in the target CRS. First the perimeter points of - unfilled valid pixels are found. These are then projected to the target - CRS. Finally the minimum and maximum values in the projected x and y - coordinates are returned. +def get_extents_from_perimeter( + projection: Proj, longitudes: Variable, latitudes: Variable +) -> Tuple[float]: + """Find the swath extents in the target CRS. First the perimeter points of + unfilled valid pixels are found. These are then projected to the target + CRS. Finally the minimum and maximum values in the projected x and y + coordinates are returned. """ coordinates_mask = get_valid_coordinates_mask(longitudes, latitudes) - x_values, y_values = get_projected_coordinates(coordinates_mask, - projection, longitudes, - latitudes) + x_values, y_values = get_projected_coordinates( + coordinates_mask, projection, longitudes, latitudes + ) - return (np.min(x_values), np.max(x_values), np.min(y_values), - np.max(y_values)) + return (np.min(x_values), np.max(x_values), np.min(y_values), np.max(y_values)) -def get_projected_coordinates(coordinates_mask: np.ma.core.MaskedArray, - projection: Proj, longitudes: Variable, - latitudes: Variable) -> Tuple[np.ndarray]: - """ Get the required coordinate points projected in the target Coordinate - Reference System (CRS). +def get_projected_coordinates( + coordinates_mask: np.ma.core.MaskedArray, + projection: Proj, + longitudes: Variable, + latitudes: Variable, +) -> Tuple[np.ndarray]: + """Get the required coordinate points projected in the target Coordinate + Reference System (CRS). """ if len(longitudes.shape) == 1: - coordinates = get_all_coordinates(longitudes[:], latitudes[:], - coordinates_mask) + coordinates = get_all_coordinates(longitudes[:], latitudes[:], coordinates_mask) else: - coordinates = get_perimeter_coordinates(longitudes[:], latitudes[:], - coordinates_mask) + coordinates = get_perimeter_coordinates( + longitudes[:], latitudes[:], coordinates_mask + ) return reproject_coordinates(coordinates, projection) -def reproject_coordinates(points: List[Tuple], - projection: Proj) -> Tuple[np.ndarray]: - """ Reproject a list of input perimeter points, in longitude and latitude - tuples, to the target CRS. +def reproject_coordinates(points: List[Tuple], projection: Proj) -> Tuple[np.ndarray]: + """Reproject a list of input perimeter points, in longitude and latitude + tuples, to the target CRS. - Returns: - x: numpy.ndarray of projected x coordinates. - y: numpy.ndarray of projected y coordinates. + Returns: + x: numpy.ndarray of projected x coordinates. + y: numpy.ndarray of projected y coordinates. """ perimeter_longitudes, perimeter_latitudes = zip(*points) return projection(perimeter_longitudes, perimeter_latitudes) -def get_one_dimensional_resolution(x_values: List[float], - y_values: List[float]) -> float: - """ Find the projected distance between each pair of consecutive points - and return the median average as the resolution. +def get_one_dimensional_resolution( + x_values: List[float], y_values: List[float] +) -> float: + """Find the projected distance between each pair of consecutive points + and return the median average as the resolution. """ - return np.median([euclidean_distance(x_values[ind + 1], x_values[ind], - y_values[ind + 1], y_values[ind]) - for ind in range(len(x_values) - 2)]) + return np.median( + [ + euclidean_distance( + x_values[ind + 1], x_values[ind], y_values[ind + 1], y_values[ind] + ) + for ind in range(len(x_values) - 2) + ] + ) -def euclidean_distance(x_one: float, x_two: float, y_one: float, - y_two: float) -> float: - """ Find the Euclidean distance between two points, in projected coordinate - space. +def euclidean_distance(x_one: float, x_two: float, y_one: float, y_two: float) -> float: + """Find the Euclidean distance between two points, in projected coordinate + space. """ - return np.sqrt((x_one - x_two)**2.0 + (y_one - y_two)**2.0) + return np.sqrt((x_one - x_two) ** 2.0 + (y_one - y_two) ** 2.0) def get_polygon_area(x_values: np.ndarray, y_values: np.ndarray) -> float: - """ Use the Gauss' Area Formula (a.k.a. Shoelace Formula) to calculate the - area of the input swath from its perimeter points. These points must - be sorted so consecutive points along the perimeter are consecutive in - the input lists. + """Use the Gauss' Area Formula (a.k.a. Shoelace Formula) to calculate the + area of the input swath from its perimeter points. These points must + be sorted so consecutive points along the perimeter are consecutive in + the input lists. """ - return 0.5 * np.abs(np.dot(x_values, np.roll(y_values, 1)) - - np.dot(y_values, np.roll(x_values, 1))) + return 0.5 * np.abs( + np.dot(x_values, np.roll(y_values, 1)) - np.dot(y_values, np.roll(x_values, 1)) + ) def get_absolute_resolution(polygon_area: float, n_pixels: int) -> float: - """ Find the absolute value of the resolution of the target CRS. This - assumes that all pixels are equal in area, and that they are square. + """Find the absolute value of the resolution of the target CRS. This + assumes that all pixels are equal in area, and that they are square. """ return np.sqrt(np.divide(polygon_area, n_pixels)) -def get_valid_coordinates_mask(longitudes: Variable, - latitudes: Variable) -> np.ma.core.MaskedArray: - """ Get a `numpy` N-d array containing boolean values (0 or 1) indicating - whether the elements of both longitude and latitude are valid at that - location. Validity of these elements means that an element must not be - a fill value, or contain a NaN. Note, a value of 1 means that the pixel - contains valid data. +def get_valid_coordinates_mask( + longitudes: Variable, latitudes: Variable +) -> np.ma.core.MaskedArray: + """Get a `numpy` N-d array containing boolean values (0 or 1) indicating + whether the elements of both longitude and latitude are valid at that + location. Validity of these elements means that an element must not be + a fill value, or contain a NaN. Note, a value of 1 means that the pixel + contains valid data. - When a `netCDF4.Variable` is loaded, the data will automatically be - read as a `numpy.ma.core.MaskedArray`. Values matching the `_FillValue` - as stored in the variable metadata will be masked. + When a `netCDF4.Variable` is loaded, the data will automatically be + read as a `numpy.ma.core.MaskedArray`. Values matching the `_FillValue` + as stored in the variable metadata will be masked. """ - valid_longitudes = np.logical_and(np.isfinite(longitudes), - np.logical_not(longitudes[:].mask)) - valid_latitudes = np.logical_and(np.isfinite(latitudes), - np.logical_not(latitudes[:].mask)) + valid_longitudes = np.logical_and( + np.isfinite(longitudes), np.logical_not(longitudes[:].mask) + ) + valid_latitudes = np.logical_and( + np.isfinite(latitudes), np.logical_not(latitudes[:].mask) + ) condition = np.logical_and(valid_longitudes, valid_latitudes) - return np.ma.masked_where(np.logical_not(condition), - np.ones(longitudes.shape)) + return np.ma.masked_where(np.logical_not(condition), np.ones(longitudes.shape)) -def get_perimeter_coordinates(longitudes: np.ndarray, latitudes: np.ndarray, - mask: np.ma.core.MaskedArray) -> List[Tuple[float]]: - """ Get the coordinates for all pixels in the input grid with non-fill, - non-NaN values for both longitude and latitude. Note, these points will - be in a random order, due to the use of the Python Set class. +def get_perimeter_coordinates( + longitudes: np.ndarray, latitudes: np.ndarray, mask: np.ma.core.MaskedArray +) -> List[Tuple[float]]: + """Get the coordinates for all pixels in the input grid with non-fill, + non-NaN values for both longitude and latitude. Note, these points will + be in a random order, due to the use of the Python Set class. """ - row_points = {point - for row_index, row in enumerate(mask) - if row.any() - for point in get_slice_edges(row.nonzero()[0], row_index)} - - column_points = {point - for column_index, column in enumerate(mask.T) - if column.any() - for point in get_slice_edges(column.nonzero()[0], - column_index, is_row=False)} + row_points = { + point + for row_index, row in enumerate(mask) + if row.any() + for point in get_slice_edges(row.nonzero()[0], row_index) + } + + column_points = { + point + for column_index, column in enumerate(mask.T) + if column.any() + for point in get_slice_edges(column.nonzero()[0], column_index, is_row=False) + } unordered_points = row_points.union(column_points) @@ -178,46 +193,57 @@ def get_perimeter_coordinates(longitudes: np.ndarray, latitudes: np.ndarray, # Most pixels are in the Eastern Hemisphere. longitudes[longitudes < 0] += 360.0 - return [(longitudes[point[0], point[1]], latitudes[point[0], point[1]]) - for point in unordered_points] + return [ + (longitudes[point[0], point[1]], latitudes[point[0], point[1]]) + for point in unordered_points + ] -def get_all_coordinates(longitudes: np.ndarray, latitudes: np.ndarray, - mask: np.ma.core.MaskedArray) -> List[Tuple[float]]: - """ Return coordinates of all valid pixels in (longitude, latitude) tuples. - These points will have non-fill values for both the longitude and - latitude, and are expected to be from a 1-D variable. +def get_all_coordinates( + longitudes: np.ndarray, latitudes: np.ndarray, mask: np.ma.core.MaskedArray +) -> List[Tuple[float]]: + """Return coordinates of all valid pixels in (longitude, latitude) tuples. + These points will have non-fill values for both the longitude and + latitude, and are expected to be from a 1-D variable. """ - return [(longitude, latitudes[array_index]) - for array_index, longitude in enumerate(longitudes) - if mask[array_index]] + return [ + (longitude, latitudes[array_index]) + for array_index, longitude in enumerate(longitudes) + if mask[array_index] + ] -def get_slice_edges(slice_valid_indices: np.ndarray, slice_index: int, - is_row: bool = True) -> List[Tuple[int]]: - """ Given a list of indices for all valid data in a single row or column of - an array, return the 2-dimensional indices of the first and last pixels - with valid data. This function relies of the `nonzero` method of a - `numpy` masked array returning array indices in ascending order. +def get_slice_edges( + slice_valid_indices: np.ndarray, slice_index: int, is_row: bool = True +) -> List[Tuple[int]]: + """Given a list of indices for all valid data in a single row or column of + an array, return the 2-dimensional indices of the first and last pixels + with valid data. This function relies of the `nonzero` method of a + `numpy` masked array returning array indices in ascending order. """ if is_row: - slice_edges = [(slice_index, slice_valid_indices[0]), - (slice_index, slice_valid_indices[-1])] + slice_edges = [ + (slice_index, slice_valid_indices[0]), + (slice_index, slice_valid_indices[-1]), + ] else: - slice_edges = [(slice_valid_indices[0], slice_index), - (slice_valid_indices[-1], slice_index)] + slice_edges = [ + (slice_valid_indices[0], slice_index), + (slice_valid_indices[-1], slice_index), + ] return slice_edges -def sort_perimeter_points(unordered_x: np.ndarray, - unordered_y: np.ndarray) -> Tuple[np.ndarray]: - """ Take arrays of x and y projected coordinates, combine into coordinate - pairs, then order them clockwise starting from the point nearest to a - reference vector originating at the polygon centroid. Finally, return - ordered arrays of the x and y coordinates separated once more. +def sort_perimeter_points( + unordered_x: np.ndarray, unordered_y: np.ndarray +) -> Tuple[np.ndarray]: + """Take arrays of x and y projected coordinates, combine into coordinate + pairs, then order them clockwise starting from the point nearest to a + reference vector originating at the polygon centroid. Finally, return + ordered arrays of the x and y coordinates separated once more. """ unordered_points = np.array(list(zip(unordered_x, unordered_y))) @@ -227,19 +253,18 @@ def sort_perimeter_points(unordered_x: np.ndarray, return zip(*ordered_points) -def clockwise_point_sort(origin: List[float], - point: List[float]) -> Tuple[float]: - """ A key function to be used with the internal Python sorted function. - This function should return a tuple of the clockwise angle and length - between the point and a reference vector, which is a vertical unit - vector. The origin argument should be the within the polygon for which - perimeter points are being sorted, to ensure correct ordering. For - simplicity, it is assumed this origin is the centroid of the polygon. +def clockwise_point_sort(origin: List[float], point: List[float]) -> Tuple[float]: + """A key function to be used with the internal Python sorted function. + This function should return a tuple of the clockwise angle and length + between the point and a reference vector, which is a vertical unit + vector. The origin argument should be the within the polygon for which + perimeter points are being sorted, to ensure correct ordering. For + simplicity, it is assumed this origin is the centroid of the polygon. - See: + See: - - https://stackoverflow.com/a/41856340 - - https://stackoverflow.com/a/35134034 + - https://stackoverflow.com/a/41856340 + - https://stackoverflow.com/a/35134034 """ reference_vector = np.array([0, 1]) @@ -260,9 +285,9 @@ def clockwise_point_sort(origin: List[float], def swath_crosses_international_date_line(longitudes: np.ndarray) -> bool: - """ Check if swath begins west of the International Date Line and ends to - the east of it. In this case there should be a discontinuity between - either two adjacent longitude columns or rows. + """Check if swath begins west of the International Date Line and ends to + the east of it. In this case there should be a discontinuity between + either two adjacent longitude columns or rows. """ longitudes_difference_row = np.diff(longitudes, n=1, axis=0) diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 7d48a6d..749bbd4 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -11,26 +11,27 @@ def create_coordinates_key(variable: VariableFromNetCDF4) -> Tuple[str]: - """ Create a unique, hashable entity from the coordinates - associated with a science variable. These coordinates - are derived using the `earthdata-varinfo` package, which - augments the CF-Convention `coordinates` metadata - attribute with supplements and overrides, where required. + """Create a unique, hashable entity from the coordinates + associated with a science variable. These coordinates + are derived using the `earthdata-varinfo` package, which + augments the CF-Convention `coordinates` metadata + attribute with supplements and overrides, where required. """ return tuple(sorted(list(variable.references.get('coordinates')))) -def get_variable_values(input_file: Dataset, variable: Variable, - fill_value: Optional) -> np.ndarray: - """ A helper function to retrieve the values of a specified dataset. This - function accounts for 2-D and 3-D datasets based on whether the time - variable is present in the dataset. +def get_variable_values( + input_file: Dataset, variable: Variable, fill_value: Optional +) -> np.ndarray: + """A helper function to retrieve the values of a specified dataset. This + function accounts for 2-D and 3-D datasets based on whether the time + variable is present in the dataset. - As the variable data are returned as a `numpy.ma.MaskedArray`, the will - return no data in the filled pixels. To ensure that the data are - correctly handled, the fill value is applied to masked pixels using the - `filled` method. + As the variable data are returned as a `numpy.ma.MaskedArray`, the will + return no data in the filled pixels. To ensure that the data are + correctly handled, the fill value is applied to masked pixels using the + `filled` method. """ # TODO: Remove in favour of apply2D or process_subdimension. @@ -48,33 +49,33 @@ def get_variable_values(input_file: Dataset, variable: Variable, return variable[:].filled(fill_value=fill_value) -def get_coordinate_variable(dataset: Dataset, coordinates_tuple: Tuple[str], - coordinate_substring) -> Optional[Variable]: - """ Search the coordinate dataset names for a match to the substring, - which will be either "lat" or "lon". Return the corresponding variable - from the dataset. Only the base variable name is used, as the group - path may contain either of the strings as part of other words. +def get_coordinate_variable( + dataset: Dataset, coordinates_tuple: Tuple[str], coordinate_substring +) -> Optional[Variable]: + """Search the coordinate dataset names for a match to the substring, + which will be either "lat" or "lon". Return the corresponding variable + from the dataset. Only the base variable name is used, as the group + path may contain either of the strings as part of other words. """ for coordinate in coordinates_tuple: - if ( - coordinate_substring in coordinate.split('/')[-1] - and variable_in_dataset(coordinate, dataset) + if coordinate_substring in coordinate.split('/')[-1] and variable_in_dataset( + coordinate, dataset ): return dataset[coordinate] raise MissingCoordinatesError(coordinates_tuple) def get_variable_numeric_fill_value(variable: Variable) -> FillValueType: - """ Retrieve the _FillValue attribute for a given variable. If there is no - _FillValue attribute, return None. The `pyresample` - `get_sample_from_neighbour_info` function will only accept numerical - inputs for `fill_value`. Non-numeric fill values are returned as None. + """Retrieve the _FillValue attribute for a given variable. If there is no + _FillValue attribute, return None. The `pyresample` + `get_sample_from_neighbour_info` function will only accept numerical + inputs for `fill_value`. Non-numeric fill values are returned as None. - This function also accounts for if the input variable is scaled, as the - fill value as stored in a NetCDF-4 file should match the nature of the - saved data (e.g., if the data are scaled, the fill value should also - be scaled). + This function also accounts for if the input variable is scaled, as the + fill value as stored in a NetCDF-4 file should match the nature of the + saved data (e.g., if the data are scaled, the fill value should also + be scaled). """ if '_FillValue' in variable.ncattrs(): @@ -82,32 +83,27 @@ def get_variable_numeric_fill_value(variable: Variable) -> FillValueType: else: fill_value = None - if not isinstance(fill_value, - (np.integer, np.longlong, np.floating, int, float)): + if not isinstance(fill_value, (np.integer, np.longlong, np.floating, int, float)): fill_value = None if fill_value is not None: scaling = get_scale_and_offset(variable) if {'add_offset', 'scale_factor'}.issubset(scaling): - fill_value = ( - (fill_value * scaling['scale_factor']) - + scaling['add_offset'] - ) + fill_value = (fill_value * scaling['scale_factor']) + scaling['add_offset'] return fill_value -def get_variable_file_path(temp_dir: str, variable_name: str, - extension: str) -> str: - """ Create a file name for the variable, that should be unique, even if - there are other variables of the same name in a different group, e.g.: +def get_variable_file_path(temp_dir: str, variable_name: str, extension: str) -> str: + """Create a file name for the variable, that should be unique, even if + there are other variables of the same name in a different group, e.g.: - /gt1r/land_segments/dem_h - /gt1l/land_segments/dem_h + /gt1r/land_segments/dem_h + /gt1l/land_segments/dem_h - Leading forward slashes will be stripped from the variable name, and - those within the string are replaced with underscores. + Leading forward slashes will be stripped from the variable name, and + those within the string are replaced with underscores. """ converted_variable_name = variable_name.lstrip('/').replace('/', '_') @@ -115,11 +111,11 @@ def get_variable_file_path(temp_dir: str, variable_name: str, def get_scale_and_offset(variable: Variable) -> Dict: - """ Check the input dataset for the `scale_factor` and `add_offset` - parameter. If those attributes are present, return a dictionary - containing those values, so the single band output can correctly scale - the data. The `netCDF4` package will automatically apply these - values upon reading and writing of the data. + """Check the input dataset for the `scale_factor` and `add_offset` + parameter. If those attributes are present, return a dictionary + containing those values, so the single band output can correctly scale + the data. The `netCDF4` package will automatically apply these + values upon reading and writing of the data. """ attributes = variable.ncattrs() @@ -127,7 +123,7 @@ def get_scale_and_offset(variable: Variable) -> Dict: if {'add_offset', 'scale_factor'}.issubset(attributes): scaling_attributes = { 'add_offset': variable.getncattr('add_offset'), - 'scale_factor': variable.getncattr('scale_factor') + 'scale_factor': variable.getncattr('scale_factor'), } else: scaling_attributes = {} @@ -136,25 +132,24 @@ def get_scale_and_offset(variable: Variable) -> Dict: def qualify_reference(raw_reference: str, variable: Variable) -> str: - """ Take a reference to a variable, as stored in the metadata of another - variable, and construct an absolute path to it. For example: - - * In '/group_one/var_one', reference: '/base_var' becomes '/base_var' - * In '/group_one/var_one', reference: '../base_var' becomes '/base_var' - * In '/group_one/var_one', reference './group_var' becomes - '/group_one/group_var' - * In '/group_one/var_one', reference: 'group_var' becomes - '/group_one/group_var' (if '/group_one' contains 'group_var') - * In '/group_one/var_one', reference: 'base_var' becomes - '/base_var' (if'/group_one' does not contain 'base_var') + """Take a reference to a variable, as stored in the metadata of another + variable, and construct an absolute path to it. For example: + + * In '/group_one/var_one', reference: '/base_var' becomes '/base_var' + * In '/group_one/var_one', reference: '../base_var' becomes '/base_var' + * In '/group_one/var_one', reference './group_var' becomes + '/group_one/group_var' + * In '/group_one/var_one', reference: 'group_var' becomes + '/group_one/group_var' (if '/group_one' contains 'group_var') + * In '/group_one/var_one', reference: 'base_var' becomes + '/base_var' (if'/group_one' does not contain 'base_var') """ referee_group = variable.group() if raw_reference.startswith('../'): # Reference is relative, and requires qualification - absolute_reference = construct_absolute_path(raw_reference, - referee_group.path) + absolute_reference = construct_absolute_path(raw_reference, referee_group.path) elif raw_reference.startswith('/'): # Reference is already absolute absolute_reference = raw_reference @@ -163,8 +158,7 @@ def qualify_reference(raw_reference: str, variable: Variable) -> str: absolute_reference = referee_group.path + raw_reference[1:] elif raw_reference in referee_group.variables: # e.g. 'variable_name' and in the referee's group - absolute_reference = construct_absolute_path(raw_reference, - referee_group.path) + absolute_reference = construct_absolute_path(raw_reference, referee_group.path) else: # e.g. 'variable_name', not in referee's group, assume root group. absolute_reference = construct_absolute_path(raw_reference, '') @@ -173,16 +167,16 @@ def qualify_reference(raw_reference: str, variable: Variable) -> str: def construct_absolute_path(reference: str, referee_group_path: str) -> str: - """ Construct an absolute path for a relative reference to another variable - (e.g. '../latitude'), by combining the reference with the group path of - the referee variable. + """Construct an absolute path for a relative reference to another variable + (e.g. '../latitude'), by combining the reference with the group path of + the referee variable. """ relative_prefix = '../' group_path_pieces = referee_group_path.split('/') while reference.startswith(relative_prefix): - reference = reference[len(relative_prefix):] + reference = reference[len(relative_prefix) :] group_path_pieces.pop() absolute_path = '/'.join(group_path_pieces + [reference]) @@ -191,10 +185,10 @@ def construct_absolute_path(reference: str, referee_group_path: str) -> str: def variable_in_dataset(variable_name: str, dataset: Dataset) -> bool: - """ Check if a nested variable exists in a NetCDF-4 dataset. This function - is necessary, as the `Dataset.variables` or `Group.variables` class - attribute only includes immediate children, not those within nested - groups. + """Check if a nested variable exists in a NetCDF-4 dataset. This function + is necessary, as the `Dataset.variables` or `Group.variables` class + attribute only includes immediate children, not those within nested + groups. """ variable_pieces = variable_name.lstrip('/').split('/') @@ -214,11 +208,11 @@ def variable_in_dataset(variable_name: str, dataset: Dataset) -> bool: def make_array_two_dimensional(one_dimensional_array: np.ndarray) -> np.ndarray: - """ Take a one dimensional array and make it a two-dimensional array, with - all values in the same column. + """Take a one dimensional array and make it a two-dimensional array, with + all values in the same column. - This is primarily required to allow processing of data with the EWA - interpolations method. + This is primarily required to allow processing of data with the EWA + interpolations method. """ return np.expand_dims(one_dimensional_array, 1) diff --git a/tests/__init__.py b/tests/__init__.py index a2c1e57..681f36a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,3 @@ import os + os.environ['ENV'] = os.environ.get('ENV') or 'test' diff --git a/tests/pip_test_requirements.txt b/tests/pip_test_requirements.txt index 0cf95be..4b2bb55 100644 --- a/tests/pip_test_requirements.txt +++ b/tests/pip_test_requirements.txt @@ -1,4 +1,5 @@ coverage~=7.2.2 +pre-commit~=3.7.0 pycodestyle~=2.10.0 pylint~=2.17.2 unittest-xml-reporting~=3.2.0 diff --git a/tests/test_code_format.py b/tests/test_code_format.py index c359a98..15daf14 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -5,16 +5,18 @@ class TestCodeFormat(TestCase): - """ This test class should ensure all Harmony service Python code adheres - to standard Python code styling. - - Ignored errors and warning: - - * E501: Line length, which defaults to 80 characters. This is a - preferred feature of the code, but not always easily achieved. - * W503: Break before binary operator. Have to ignore one of W503 or - W504 to allow for breaking of some long lines. PEP8 suggests - breaking the line before a binary operatore is more "Pythonic". + """This test class should ensure all Harmony service Python code adheres + to standard Python code styling. + + Ignored errors and warning: + + * E501: Line length, which defaults to 80 characters. This is a + preferred feature of the code, but not always easily achieved. + * W503: Break before binary operator. Have to ignore one of W503 or + W504 to allow for breaking of some long lines. PEP8 suggests + breaking the line before a binary operatore is more "Pythonic". + * E203, E701: This repository uses black code formatting, which deviates + from PEP8 for these errors. """ @classmethod @@ -22,10 +24,10 @@ def setUpClass(cls): cls.python_files = Path('swath_projector').rglob('*.py') def test_pycodestyle_adherence(self): - """ Ensure all code in the `swath_projector` directory adheres to PEP8 - defined standard. + """Ensure all code in the `swath_projector` directory adheres to PEP8 + defined standard. """ - style_guide = StyleGuide(ignore=['E501', 'W503']) + style_guide = StyleGuide(ignore=['E501', 'W503', 'E203', 'E701']) results = style_guide.check_files(self.python_files) self.assertEqual(results.total_errors, 0, 'Found code style issues.') diff --git a/tests/test_input_file_download.py b/tests/test_input_file_download.py index 575547e..45db8e6 100644 --- a/tests/test_input_file_download.py +++ b/tests/test_input_file_download.py @@ -8,38 +8,42 @@ class TestInputFileDownload(TestCase): - """ A test class to ensure that common failures arising from missing or - incorrect file details in the input Harmony message are well handled. + """A test class to ensure that common failures arising from missing or + incorrect file details in the input Harmony message are well handled. """ def test_message_has_no_granules_attribute(self): - """ Handle a harmony message that does not list any granules """ + """Handle a harmony message that does not list any granules""" reprojector = SwathProjectorAdapter( - Message({'format': {}, 'sources': [{}]}), - config=config(False) + Message({'format': {}, 'sources': [{}]}), config=config(False) ) with self.assertRaises(HarmonyException) as context: reprojector.invoke() - self.assertEqual(str(context.exception), - 'No granules specified for reprojection') + self.assertEqual( + str(context.exception), 'No granules specified for reprojection' + ) def test_local_file_does_not_exist(self): - """ Handle a harmony message that refers to a granule local file that - does not exist + """Handle a harmony message that refers to a granule local file that + does not exist """ reprojector = SwathProjectorAdapter( Message({'format': {}, 'sources': [{'granules': [{}]}]}), - config=config(False) + config=config(False), ) with self.assertRaises(Exception) as context: - reproject(reprojector.message, 'short name', - 'https://example.com/no_such_file.nc4', - 'test/data/no_such_file', '/no/such/dir', - reprojector.logger) + reproject( + reprojector.message, + 'short name', + 'https://example.com/no_such_file.nc4', + 'test/data/no_such_file', + '/no/such/dir', + reprojector.logger, + ) self.assertEqual(str(context.exception), 'Input file does not exist') diff --git a/tests/test_pyresample_reproject.py b/tests/test_pyresample_reproject.py index 383dbe9..342ce2a 100755 --- a/tests/test_pyresample_reproject.py +++ b/tests/test_pyresample_reproject.py @@ -15,6 +15,7 @@ class TestPyResampleReproject(TestCase): interpolation options. """ + def test_pyresample_interpolation(self, mock_download, mock_stage): """Ensure SwotRepr will successfully complete when using pyresample and each specified interpolation. @@ -24,28 +25,35 @@ def test_pyresample_interpolation(self, mock_download, mock_stage): for interpolation in valid_interpolations: with self.subTest(f'pyresample "{interpolation}" interpolation.'): - test_data = Message({ - 'accessToken': 'fake_token', - 'callback': 'https://example.com/callback', - 'stagingLocation': 's3://example-bucket/example-path/', - 'sources': [{ - 'granules': [{ - 'url': 'tests/data/VOL2PSST_2017.nc', - 'temporal': { - 'start': '2020-01-01T00:00:00.000Z', - 'end': '2020-01-02T00:00:00.000Z' - }, - 'bbox': [-180, -90, 180, 90] - }], - }], - 'format': {'crs': 'EPSG:32603', - 'interpolation': interpolation, - 'width': 1000, - 'height': 500} - }) + test_data = Message( + { + 'accessToken': 'fake_token', + 'callback': 'https://example.com/callback', + 'stagingLocation': 's3://example-bucket/example-path/', + 'sources': [ + { + 'granules': [ + { + 'url': 'tests/data/VOL2PSST_2017.nc', + 'temporal': { + 'start': '2020-01-01T00:00:00.000Z', + 'end': '2020-01-02T00:00:00.000Z', + }, + 'bbox': [-180, -90, 180, 90], + } + ], + } + ], + 'format': { + 'crs': 'EPSG:32603', + 'interpolation': interpolation, + 'width': 1000, + 'height': 500, + }, + } + ) - reprojector = SwathProjectorAdapter(test_data, - config=config(False)) + reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() mock_download.assert_called_once_with( @@ -53,14 +61,15 @@ def test_pyresample_interpolation(self, mock_download, mock_stage): ANY, logger=ANY, access_token='fake_token', - cfg=ANY + cfg=ANY, ) mock_stage.assert_called_once_with( StringContains('VOL2PSST_2017_repr.nc'), 'VOL2PSST_2017_regridded.nc', 'application/x-netcdf', location='s3://example-bucket/example-path/', - logger=ANY) + logger=ANY, + ) # Reset mock calls for next interpolation mock_download.reset_mock() diff --git a/tests/test_swath_projector.py b/tests/test_swath_projector.py index 0e267a6..973537a 100755 --- a/tests/test_swath_projector.py +++ b/tests/test_swath_projector.py @@ -18,14 +18,15 @@ @patch('swath_projector.adapter.stage', return_value='https://example.com/data') @patch('swath_projector.adapter.download', side_effect=download_side_effect) class TestSwathProjector(TestCase): - """ A test class that will run the full Swath Projector against a variety - of input files and Harmony messages. + """A test class that will run the full Swath Projector against a variety + of input files and Harmony messages. """ + @classmethod def setUpClass(cls): - """ Define class properties that do not need to be re-instantiated - between tests. + """Define class properties that do not need to be re-instantiated + between tests. """ cls.access_token = 'fake_token' @@ -33,22 +34,24 @@ def setUpClass(cls): cls.callback = 'http://example.com/callback' cls.mime_type = 'application/x-netcdf' cls.staging_location = 's3://example-bucket/example-path/' - cls.temporal = {'start': '2020-01-01T00:00:00.000Z', - 'end': '2020-01-02T00:00:00.000Z'} + cls.temporal = { + 'start': '2020-01-01T00:00:00.000Z', + 'end': '2020-01-02T00:00:00.000Z', + } cls.tmp_dir = 'tests/temp' def setUp(self): - """ Set properties of tests that need to be re-created every test. """ + """Set properties of tests that need to be re-created every test.""" makedirs(self.tmp_dir) copy('tests/data/africa.nc', self.tmp_dir) def tearDown(self): - """ Perform per-test teardown operations. """ + """Perform per-test teardown operations.""" rmtree(self.tmp_dir) def get_provenance(self, file_path): - """ Utility method to retrieve `history`, `History` and `history_json` - global attributes from a test output file. + """Utility method to retrieve `history`, `History` and `history_json` + global attributes from a test output file. """ with Dataset(file_path, 'r') as dataset: @@ -59,168 +62,205 @@ def get_provenance(self, file_path): return history, history_uppercase, history_json def test_single_band_input(self, mock_download, mock_stage, mock_datetime): - """ Nominal (successful) reprojection of a single band input file. """ + """Nominal (successful) reprojection of a single band input file.""" input_file_path = 'tests/data/VNL2_oneBand.nc' mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': {'height': 500, 'width': 1000} - }) + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': {'height': 500, 'width': 1000}, + } + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('VNL2_oneBand_repr.nc'), - 'VNL2_oneBand_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('VNL2_oneBand_repr.nc'), + 'VNL2_oneBand_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) def test_africa_input(self, mock_download, mock_stage, mock_datetime): - """ Nominal (successful) reprojection of tests/data/africa.nc, using - geographic coordinates, bilinear interpolation and specifying the - extent of the target area grid. + """Nominal (successful) reprojection of tests/data/africa.nc, using + geographic coordinates, bilinear interpolation and specifying the + extent of the target area grid. - Also ensure that the provenance of the output file includes a - record of the operation performed via the `history` and - `history_json` global attributes. + Also ensure that the provenance of the output file includes a + record of the operation performed via the `history` and + `history_json` global attributes. """ input_file_path = 'tests/data/africa.nc' mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': {'crs': 'EPSG:4326', - 'interpolation': 'bilinear', - 'scaleExtent': {'x': {'min': -20, 'max': 60}, - 'y': {'min': 10, 'max': 35}}} - }) + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': { + 'crs': 'EPSG:4326', + 'interpolation': 'bilinear', + 'scaleExtent': { + 'x': {'min': -20, 'max': 60}, + 'y': {'min': 10, 'max': 35}, + }, + }, + } + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('africa_repr.nc'), - 'africa_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('africa_repr.nc'), + 'africa_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) output_path = mock_stage.call_args[0][0] history, history_uppercase, history_json = self.get_provenance(output_path) - expected_history = ('2021-05-12T19:03:04+00:00 sds/harmony-swath-projector ' - '0.9.0 {"crs": "EPSG:4326", "interpolation": ' - '"bilinear", "x_min": -20, "x_max": 60, "y_min": ' - '10, "y_max": 35}') - - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'crs': 'EPSG:4326', - 'input_file': input_file_path, - 'interpolation': 'bilinear', - 'x_min': -20, - 'x_max': 60, - 'y_min': 10, - 'y_max': 35}, - 'derived_from': input_file_path, - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', - }] + expected_history = ( + '2021-05-12T19:03:04+00:00 sds/harmony-swath-projector ' + '0.9.0 {"crs": "EPSG:4326", "interpolation": ' + '"bilinear", "x_min": -20, "x_max": 60, "y_min": ' + '10, "y_max": 35}' + ) + + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'crs': 'EPSG:4326', + 'input_file': input_file_path, + 'interpolation': 'bilinear', + 'x_min': -20, + 'x_max': 60, + 'y_min': 10, + 'y_max': 35, + }, + 'derived_from': input_file_path, + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + } + ] self.assertEqual(history, expected_history) self.assertIsNone(history_uppercase) self.assertListEqual(json.loads(history_json), expected_history_json) - def test_africa_input_with_history_and_history_json(self, mock_download, - mock_stage, mock_datetime): - """ Ensure that an input file with existing `history` and - `history_json` global attributes include these metadata in the - output `history` and `history_json` global attributes. + def test_africa_input_with_history_and_history_json( + self, mock_download, mock_stage, mock_datetime + ): + """Ensure that an input file with existing `history` and + `history_json` global attributes include these metadata in the + output `history` and `history_json` global attributes. - This test will use a temporary copy of the `africa.nc` granule, - and update the global attributes to include values for `history` - and `history_json`. This updated file will then be used as input - to the service. + This test will use a temporary copy of the `africa.nc` granule, + and update the global attributes to include values for `history` + and `history_json`. This updated file will then be used as input + to the service. """ input_file_path = f'{self.tmp_dir}/africa.nc' old_history = '2000-01-02T03:04:05.123456+00.00 Swathinator v0.0.1' - old_history_json = json.dumps([{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'Swathinator', - 'version': '0.0.1', - 'parameters': {'input_file': 'africa.nc'}, - 'derived_from': 'africa.nc', - 'program_ref': 'Swathinator Reference' - }]) + old_history_json = json.dumps( + [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'Swathinator', + 'version': '0.0.1', + 'parameters': {'input_file': 'africa.nc'}, + 'derived_from': 'africa.nc', + 'program_ref': 'Swathinator Reference', + } + ] + ) with Dataset(input_file_path, 'a') as input_dataset: input_dataset.setncattr('history', old_history) input_dataset.setncattr('history_json', old_history_json) mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': {'crs': 'EPSG:4326', - 'interpolation': 'bilinear', - 'scaleExtent': {'x': {'min': -20, 'max': 60}, - 'y': {'min': 10, 'max': 35}}} - }) + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': { + 'crs': 'EPSG:4326', + 'interpolation': 'bilinear', + 'scaleExtent': { + 'x': {'min': -20, 'max': 60}, + 'y': {'min': 10, 'max': 35}, + }, + }, + } + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('africa_repr.nc'), - 'africa_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('africa_repr.nc'), + 'africa_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) output_path = mock_stage.call_args[0][0] history, history_uppercase, history_json = self.get_provenance(output_path) @@ -232,45 +272,51 @@ def test_africa_input_with_history_and_history_json(self, mock_download, '"x_max": 60, "y_min": 10, "y_max": 35}' ) - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'Swathinator', - 'version': '0.0.1', - 'parameters': {'input_file': 'africa.nc'}, - 'derived_from': 'africa.nc', - 'program_ref': 'Swathinator Reference' - }, { - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'crs': 'EPSG:4326', - 'input_file': input_file_path, - 'interpolation': 'bilinear', - 'x_min': -20, - 'x_max': 60, - 'y_min': 10, - 'y_max': 35}, - 'derived_from': input_file_path, - 'cf_history': ['2000-01-02T03:04:05.123456+00.00 Swathinator v0.0.1'], - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST' - }] + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'Swathinator', + 'version': '0.0.1', + 'parameters': {'input_file': 'africa.nc'}, + 'derived_from': 'africa.nc', + 'program_ref': 'Swathinator Reference', + }, + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'crs': 'EPSG:4326', + 'input_file': input_file_path, + 'interpolation': 'bilinear', + 'x_min': -20, + 'x_max': 60, + 'y_min': 10, + 'y_max': 35, + }, + 'derived_from': input_file_path, + 'cf_history': ['2000-01-02T03:04:05.123456+00.00 Swathinator v0.0.1'], + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + }, + ] self.assertEqual(history, expected_history) self.assertIsNone(history_uppercase) self.assertListEqual(json.loads(history_json), expected_history_json) - def test_africa_input_with_history_uppercase(self, mock_download, - mock_stage, mock_datetime): - """ Ensure that an input file with an existing `History` global - attribute can be successfully projected, and that this existing - metadata is included in the output `History` and `history_json` - global attributes. + def test_africa_input_with_history_uppercase( + self, mock_download, mock_stage, mock_datetime + ): + """Ensure that an input file with an existing `History` global + attribute can be successfully projected, and that this existing + metadata is included in the output `History` and `history_json` + global attributes. - This test will use a temporary copy of the `africa.nc` granule, - and update the global attributes to include values for `History`. - This updated file will then be used as input to the service. + This test will use a temporary copy of the `africa.nc` granule, + and update the global attributes to include values for `History`. + This updated file will then be used as input to the service. """ input_file_path = 'tests/data/africa_History.nc' @@ -281,36 +327,46 @@ def test_africa_input_with_history_uppercase(self, mock_download, input_dataset.setncattr('History', old_history) mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': {'crs': 'EPSG:4326', - 'interpolation': 'bilinear', - 'scaleExtent': {'x': {'min': -20, 'max': 60}, - 'y': {'min': 10, 'max': 35}}} - }) + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': { + 'crs': 'EPSG:4326', + 'interpolation': 'bilinear', + 'scaleExtent': { + 'x': {'min': -20, 'max': 60}, + 'y': {'min': 10, 'max': 35}, + }, + }, + } + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('africa_repr.nc'), - 'africa_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('africa_repr.nc'), + 'africa_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) output_path = mock_stage.call_args[0][0] history, history_uppercase, history_json = self.get_provenance(output_path) @@ -322,67 +378,82 @@ def test_africa_input_with_history_uppercase(self, mock_download, '"x_max": 60, "y_min": 10, "y_max": 35}' ) - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'crs': 'EPSG:4326', - 'input_file': input_file_path, - 'interpolation': 'bilinear', - 'x_min': -20, - 'x_max': 60, - 'y_min': 10, - 'y_max': 35}, - 'derived_from': input_file_path, - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', - 'cf_history': ['2000-01-02T03:04:05.123456+00.00 Swathinator v0.0.1'] - }] + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'crs': 'EPSG:4326', + 'input_file': input_file_path, + 'interpolation': 'bilinear', + 'x_min': -20, + 'x_max': 60, + 'y_min': 10, + 'y_max': 35, + }, + 'derived_from': input_file_path, + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + 'cf_history': ['2000-01-02T03:04:05.123456+00.00 Swathinator v0.0.1'], + } + ] self.assertIsNone(history) self.assertEqual(history_uppercase, expected_history_uppercase) self.assertListEqual(json.loads(history_json), expected_history_json) - def test_single_band_input_default_crs(self, mock_download, mock_stage, - mock_datetime): - """ Nominal (successful) reprojection of a single band input. This - will default to using a geographic coordinate system, and use the - Elliptically Weighted Average (EWA) interpolation method to derive - the reprojected variables. + def test_single_band_input_default_crs( + self, mock_download, mock_stage, mock_datetime + ): + """Nominal (successful) reprojection of a single band input. This + will default to using a geographic coordinate system, and use the + Elliptically Weighted Average (EWA) interpolation method to derive + the reprojected variables. """ input_file_path = 'tests/data/VNL2_oneBand.nc' mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': {'interpolation': 'ewa', - 'scaleExtent': {'x': {'min': -160, 'max': -159}, - 'y': {'min': 24, 'max': 25}}} - }) + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': { + 'interpolation': 'ewa', + 'scaleExtent': { + 'x': {'min': -160, 'max': -159}, + 'y': {'min': 24, 'max': 25}, + }, + }, + } + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('VNL2_oneBand_repr.nc'), - 'VNL2_oneBand_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('VNL2_oneBand_repr.nc'), + 'VNL2_oneBand_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) output_path = mock_stage.call_args[0][0] history, history_uppercase, history_json = self.get_provenance(output_path) @@ -398,74 +469,91 @@ def test_single_band_input_default_crs(self, mock_download, mock_stage, '"x_max": -159, "y_min": 24, "y_max": 25}' ) - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'crs': '+proj=longlat +ellps=WGS84', - 'input_file': input_file_path, - 'interpolation': 'ewa', - 'x_min': -160, - 'x_max': -159, - 'y_min': 24, - 'y_max': 25}, - 'derived_from': input_file_path, - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', - 'cf_history': [('Tue Nov 12 15:31:14 2019: ncks -v sea_surface_temperature ' - 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS' - '_NPP-v02.0-fv03.0.nc VNL2_oneBand.nc'), - 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT'] - }] + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'crs': '+proj=longlat +ellps=WGS84', + 'input_file': input_file_path, + 'interpolation': 'ewa', + 'x_min': -160, + 'x_max': -159, + 'y_min': 24, + 'y_max': 25, + }, + 'derived_from': input_file_path, + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + 'cf_history': [ + ( + 'Tue Nov 12 15:31:14 2019: ncks -v sea_surface_temperature ' + 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS' + '_NPP-v02.0-fv03.0.nc VNL2_oneBand.nc' + ), + 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT', + ], + } + ] self.assertEqual(history, expected_history) self.assertIsNone(history_uppercase) self.assertListEqual(json.loads(history_json), expected_history_json) - def test_single_band_input_reprojected_metres(self, mock_download, - mock_stage, mock_datetime): - """ Nominal (successful) reprojection of the single band input file, - specifying the UTM Zone 3N (EPSG:32603) target projection and the - Elliptically Weighted Average, Nearest Neighbour (EWA-NN) - interpolation algorithm to derive the reprojected variables. + def test_single_band_input_reprojected_metres( + self, mock_download, mock_stage, mock_datetime + ): + """Nominal (successful) reprojection of the single band input file, + specifying the UTM Zone 3N (EPSG:32603) target projection and the + Elliptically Weighted Average, Nearest Neighbour (EWA-NN) + interpolation algorithm to derive the reprojected variables. - Note: This choice of target CRS is due to the location of the - input data being near the Hawaiian islands. + Note: This choice of target CRS is due to the location of the + input data being near the Hawaiian islands. """ input_file_path = 'tests/data/VNL2_oneBand.nc' mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - test_data = Message({ - 'accessToken': self.access_token, - 'callback': self.callback, - 'stagingLocation': self.staging_location, - 'sources': [{ - 'granules': [{ - 'url': input_file_path, - 'temporal': self.temporal, - 'bbox': self.bounding_box, - }], - }], - 'format': { - 'crs': 'EPSG:32603', - 'interpolation': 'ewa-nn', - 'scaleExtent': {'x': {'min': 0, 'max': 1500000}, - 'y': {'min': 2500000, 'max': 3300000}} + test_data = Message( + { + 'accessToken': self.access_token, + 'callback': self.callback, + 'stagingLocation': self.staging_location, + 'sources': [ + { + 'granules': [ + { + 'url': input_file_path, + 'temporal': self.temporal, + 'bbox': self.bounding_box, + } + ], + } + ], + 'format': { + 'crs': 'EPSG:32603', + 'interpolation': 'ewa-nn', + 'scaleExtent': { + 'x': {'min': 0, 'max': 1500000}, + 'y': {'min': 2500000, 'max': 3300000}, + }, + }, } - }) + ) reprojector = SwathProjectorAdapter(test_data, config=config(False)) reprojector.invoke() - mock_download.assert_called_once_with(input_file_path, - ANY, - logger=ANY, - access_token=self.access_token, - cfg=ANY) - mock_stage.assert_called_once_with(StringContains('VNL2_oneBand_repr.nc'), - 'VNL2_oneBand_regridded.nc', - self.mime_type, - location=self.staging_location, - logger=ANY) + mock_download.assert_called_once_with( + input_file_path, ANY, logger=ANY, access_token=self.access_token, cfg=ANY + ) + mock_stage.assert_called_once_with( + StringContains('VNL2_oneBand_repr.nc'), + 'VNL2_oneBand_regridded.nc', + self.mime_type, + location=self.staging_location, + logger=ANY, + ) output_path = mock_stage.call_args[0][0] history, history_uppercase, history_json = self.get_provenance(output_path) @@ -480,25 +568,33 @@ def test_single_band_input_reprojected_metres(self, mock_download, '"x_max": 1500000, "y_min": 2500000, "y_max": 3300000}' ) - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'crs': 'EPSG:32603', - 'input_file': input_file_path, - 'interpolation': 'ewa-nn', - 'x_min': 0, - 'x_max': 1500000, - 'y_min': 2500000, - 'y_max': 3300000}, - 'derived_from': input_file_path, - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', - 'cf_history': [('Tue Nov 12 15:31:14 2019: ncks -v sea_surface_temperature ' - 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS' - '_NPP-v02.0-fv03.0.nc VNL2_oneBand.nc'), - 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT'] - }] + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'crs': 'EPSG:32603', + 'input_file': input_file_path, + 'interpolation': 'ewa-nn', + 'x_min': 0, + 'x_max': 1500000, + 'y_min': 2500000, + 'y_max': 3300000, + }, + 'derived_from': input_file_path, + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + 'cf_history': [ + ( + 'Tue Nov 12 15:31:14 2019: ncks -v sea_surface_temperature ' + 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS' + '_NPP-v02.0-fv03.0.nc VNL2_oneBand.nc' + ), + 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT', + ], + } + ] self.assertEqual(history, expected_history) self.assertIsNone(history_uppercase) diff --git a/tests/test_utils.py b/tests/test_utils.py index d7dd46c..2499b15 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,16 @@ """ Utility classes used to extend the unittest capabilities. """ + from os import sep from os.path import basename from shutil import copy class StringContains: - """ A custom matcher that can be used in `unittest` assertions, ensuring - a substring is contained in one of the expected arguments. + """A custom matcher that can be used in `unittest` assertions, ensuring + a substring is contained in one of the expected arguments. """ + def __init__(self, expected_substring): self.expected_substring = expected_substring @@ -17,10 +19,10 @@ def __eq__(self, string_to_check): def download_side_effect(file_path, working_dir, **kwargs): - """ A side effect to be used when mocking the `harmony.util.download` - function. This should copy the input file (assuming it is a local - file path) to the working directory, and then return the new file - path. + """A side effect to be used when mocking the `harmony.util.download` + function. This should copy the input file (assuming it is a local + file path) to the working directory, and then return the new file + path. """ file_base_name = basename(file_path) diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index 1a0359b..31a4d7f 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -2,7 +2,7 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from netCDF4 import Dataset, Variable +from netCDF4 import Dataset from pyproj import Proj from pyresample.geometry import AreaDefinition import numpy as np @@ -11,19 +11,13 @@ from swath_projector.interpolation import ( check_for_valid_interpolation, EPSILON, - get_bilinear_information, - get_bilinear_results, - get_ewa_information, - get_ewa_results, - get_near_information, - get_near_results, get_parameters_tuple, get_reprojection_cache, get_swath_definition, get_target_area, resample_all_variables, resample_variable, - RADIUS_OF_INFLUENCE + RADIUS_OF_INFLUENCE, ) from swath_projector.nc_single_band import HARMONY_TARGET from swath_projector.reproject import CF_CONFIG_FILE @@ -32,8 +26,7 @@ class TestInterpolation(TestCase): def setUp(self): - self.science_variables = ('/red_var', '/green_var', '/blue_var', - '/alpha_var') + self.science_variables = ('/red_var', '/green_var', '/blue_var', '/alpha_var') self.message_parameters = { 'crs': '+proj=longlat', 'input_file': 'tests/data/africa.nc', @@ -52,15 +45,17 @@ def setUp(self): } self.temp_directory = '/tmp/01234' self.logger = Logger('test') - self.var_info = VarInfoFromNetCDF4(self.message_parameters['input_file'], - short_name='harmony_example_l2', - config_file=CF_CONFIG_FILE) - self.mock_target_area = MagicMock(spec=AreaDefinition, - shape='ta_shape', - area_id='/lon, /lat') + self.var_info = VarInfoFromNetCDF4( + self.message_parameters['input_file'], + short_name='harmony_example_l2', + config_file=CF_CONFIG_FILE, + ) + self.mock_target_area = MagicMock( + spec=AreaDefinition, shape='ta_shape', area_id='/lon, /lat' + ) def assert_areadefinitions_equal(self, area_one, area_two): - """ Compare the properties of two AreaDefinitions. """ + """Compare the properties of two AreaDefinitions.""" # Check the ID is set, as it is used in nc_single_band: self.assertEqual(area_one.area_id, area_two.area_id) @@ -70,28 +65,31 @@ def assert_areadefinitions_equal(self, area_one, area_two): attributes = ['height', 'width', 'proj_str'] for attribute in attributes: - self.assertEqual(getattr(area_one, attribute), - getattr(area_two, attribute), attribute) + self.assertEqual( + getattr(area_one, attribute), getattr(area_two, attribute), attribute + ) @patch('swath_projector.interpolation.resample_variable') def test_resample_all_variables(self, mock_resample_variable): - """ Ensure resample_variable is called for each non-coordinate - variable, and those variables are all included in the list of - outputs. + """Ensure resample_variable is called for each non-coordinate + variable, and those variables are all included in the list of + outputs. - The default message being supplied does not have sufficient - information to construct a target area for all variables, so the - cache being sent to all variables should be empty. + The default message being supplied does not have sufficient + information to construct a target area for all variables, so the + cache being sent to all variables should be empty. """ parameters = {'interpolation': 'ewa-nn'} parameters.update(self.message_parameters) - output_variables = resample_all_variables(parameters, - self.science_variables, - self.temp_directory, - self.logger, - self.var_info) + output_variables = resample_all_variables( + parameters, + self.science_variables, + self.temp_directory, + self.logger, + self.var_info, + ) expected_output = ['/red_var', '/green_var', '/blue_var', '/alpha_var'] self.assertEqual(output_variables, expected_output) @@ -99,16 +97,19 @@ def test_resample_all_variables(self, mock_resample_variable): for variable in expected_output: variable_output_path = f'/tmp/01234{variable}.nc' - mock_resample_variable.assert_any_call(parameters, - variable, - {}, - variable_output_path, - self.logger, self.var_info) + mock_resample_variable.assert_any_call( + parameters, + variable, + {}, + variable_output_path, + self.logger, + self.var_info, + ) @patch('swath_projector.interpolation.resample_variable') def test_resample_single_exception(self, mock_resample_variable): - """ Ensure that if a single variable fails reprojection, the remaining - variables will still be reprojected. + """Ensure that if a single variable fails reprojection, the remaining + variables will still be reprojected. """ mock_resample_variable.side_effect = [KeyError('random'), None, None, None] @@ -116,11 +117,13 @@ def test_resample_single_exception(self, mock_resample_variable): parameters = {'interpolation': 'ewa-nn'} parameters.update(self.message_parameters) - output_variables = resample_all_variables(parameters, - self.science_variables, - self.temp_directory, - self.logger, - self.var_info) + output_variables = resample_all_variables( + parameters, + self.science_variables, + self.temp_directory, + self.logger, + self.var_info, + ) expected_output = ['/green_var', '/blue_var', '/alpha_var'] self.assertEqual(output_variables, expected_output) @@ -130,11 +133,14 @@ def test_resample_single_exception(self, mock_resample_variable): for variable in all_variables: variable_output_path = f'/tmp/01234{variable}.nc' - mock_resample_variable.assert_any_call(parameters, - variable, - {}, - variable_output_path, - self.logger, self.var_info) + mock_resample_variable.assert_any_call( + parameters, + variable, + {}, + variable_output_path, + self.logger, + self.var_info, + ) @patch('swath_projector.interpolation.write_single_band_output') @patch('swath_projector.interpolation.get_swath_definition') @@ -142,21 +148,31 @@ def test_resample_single_exception(self, mock_resample_variable): @patch('swath_projector.interpolation.get_variable_values') @patch('swath_projector.interpolation.get_sample_from_bil_info') @patch('swath_projector.interpolation.get_bil_info') - def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, - mock_get_values, mock_get_target_area, - mock_get_swath, mock_write_output): - """ The bilinear interpolation should call both get_bil_info and - get_sample_from_bil_info if there are no matching entries for the - coordinates in the reprojection information. If there is an entry, - then only get_sample_from_bil_info should be called. + def test_resample_bilinear( + self, + mock_get_bil_info, + mock_get_sample, + mock_get_values, + mock_get_target_area, + mock_get_swath, + mock_write_output, + ): + """The bilinear interpolation should call both get_bil_info and + get_sample_from_bil_info if there are no matching entries for the + coordinates in the reprojection information. If there is an entry, + then only get_sample_from_bil_info should be called. """ - mock_get_bil_info.return_value = ['vertical', 'horizontal', - 'input_indices', 'point_mapping'] + mock_get_bil_info.return_value = [ + 'vertical', + 'horizontal', + 'input_indices', + 'point_mapping', + ] results = np.array([4.0]) mock_get_sample.return_value = results mock_get_swath.return_value = 'swath' - ravel_data = np.ones((3, )) + ravel_data = np.ones((3,)) mock_values = MagicMock(**{'ravel.return_value': ravel_data}) mock_get_values.return_value = mock_values mock_get_target_area.return_value = self.mock_target_area @@ -167,33 +183,44 @@ def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, output_path = 'path/to/output' with self.subTest('No pre-existing bilinear information'): - resample_variable(message_parameters, variable_name, {}, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + {}, + output_path, + self.logger, + self.var_info, + ) expected_cache = { - ('/lat', '/lon'): {'vertical_distances': 'vertical', - 'horizontal_distances': 'horizontal', - 'valid_input_indices': 'input_indices', - 'valid_point_mapping': 'point_mapping', - 'target_area': self.mock_target_area}, + ('/lat', '/lon'): { + 'vertical_distances': 'vertical', + 'horizontal_distances': 'horizontal', + 'valid_input_indices': 'input_indices', + 'valid_point_mapping': 'point_mapping', + 'target_area': self.mock_target_area, + }, } - mock_get_bil_info.assert_called_once_with('swath', - self.mock_target_area, - radius=50000, - neighbours=16) - mock_get_sample.assert_called_once_with(ravel_data, - 'vertical', - 'horizontal', - 'input_indices', - 'point_mapping', - output_shape='ta_shape') - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_get_bil_info.assert_called_once_with( + 'swath', self.mock_target_area, radius=50000, neighbours=16 + ) + mock_get_sample.assert_called_once_with( + ravel_data, + 'vertical', + 'horizontal', + 'input_indices', + 'point_mapping', + output_shape='ta_shape', + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) with self.subTest('Pre-existing bilinear information'): mock_get_bil_info.reset_mock() @@ -210,23 +237,32 @@ def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, } } - resample_variable(message_parameters, variable_name, - bilinear_information, output_path, self.logger, - self.var_info) + resample_variable( + message_parameters, + variable_name, + bilinear_information, + output_path, + self.logger, + self.var_info, + ) mock_get_bil_info.assert_not_called() - mock_get_sample.assert_called_once_with(ravel_data, - 'vertical_old', - 'horizontal_old', - 'input_indices_old', - 'point_mapping_old', - output_shape='ta_shape') - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - bilinear_information, - {}) + mock_get_sample.assert_called_once_with( + ravel_data, + 'vertical_old', + 'horizontal_old', + 'input_indices_old', + 'point_mapping_old', + output_shape='ta_shape', + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + bilinear_information, + {}, + ) with self.subTest('Harmony message defines target area'): mock_get_target_area.reset_mock() @@ -234,16 +270,22 @@ def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, mock_get_sample.reset_mock() mock_write_output.reset_mock() - harmony_target_area = MagicMock(spec=AreaDefinition, - area_id=HARMONY_TARGET, - shape='harmony_shape') + harmony_target_area = MagicMock( + spec=AreaDefinition, area_id=HARMONY_TARGET, shape='harmony_shape' + ) input_cache = { HARMONY_TARGET: {'target_area': harmony_target_area}, } - resample_variable(message_parameters, variable_name, input_cache, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + input_cache, + output_path, + self.logger, + self.var_info, + ) # Check that there is a new entry in the cache, and that it only # contains references to the original Harmony target area object, @@ -256,30 +298,31 @@ def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, 'valid_input_indices': 'input_indices', 'valid_point_mapping': 'point_mapping', 'target_area': harmony_target_area, - } + }, } self.assertDictEqual(input_cache, expected_cache) - mock_get_bil_info.assert_called_once_with('swath', - harmony_target_area, - radius=50000, - neighbours=16) + mock_get_bil_info.assert_called_once_with( + 'swath', harmony_target_area, radius=50000, neighbours=16 + ) mock_get_sample.assert_called_once_with( ravel_data, 'vertical', 'horizontal', 'input_indices', 'point_mapping', - output_shape='harmony_shape' + output_shape='harmony_shape', ) # The Harmony target area should be given to the output function - mock_write_output.assert_called_once_with(harmony_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_write_output.assert_called_once_with( + harmony_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) mock_get_target_area.assert_not_called() @patch('swath_projector.interpolation.write_single_band_output') @@ -288,13 +331,19 @@ def test_resample_bilinear(self, mock_get_bil_info, mock_get_sample, @patch('swath_projector.interpolation.get_variable_values') @patch('swath_projector.interpolation.fornav') @patch('swath_projector.interpolation.ll2cr') - def test_resample_ewa(self, mock_ll2cr, mock_fornav, mock_get_values, - mock_get_target_area, mock_get_swath, - mock_write_output): - """ EWA interpolation should call both ll2cr and fornav if there are - no matching entries for the coordinates in the reprojection - information. If there is an entry, then only fornav should be - called. + def test_resample_ewa( + self, + mock_ll2cr, + mock_fornav, + mock_get_values, + mock_get_target_area, + mock_get_swath, + mock_write_output, + ): + """EWA interpolation should call both ll2cr and fornav if there are + no matching entries for the coordinates in the reprojection + information. If there is an entry, then only fornav should be + called. """ mock_ll2cr.return_value = ['swath_points_in_grid', 'columns', 'rows'] @@ -311,26 +360,39 @@ def test_resample_ewa(self, mock_ll2cr, mock_fornav, mock_get_values, output_path = 'path/to/output' with self.subTest('No pre-existing EWA information'): - resample_variable(message_parameters, variable_name, {}, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + {}, + output_path, + self.logger, + self.var_info, + ) expected_cache = { - ('/lat', '/lon'): {'columns': 'columns', - 'rows': 'rows', - 'target_area': self.mock_target_area} + ('/lat', '/lon'): { + 'columns': 'columns', + 'rows': 'rows', + 'target_area': self.mock_target_area, + } } mock_ll2cr.assert_called_once_with('swath', self.mock_target_area) - mock_fornav.assert_called_once_with('columns', 'rows', - self.mock_target_area, - mock_values, - maximum_weight_mode=False) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_fornav.assert_called_once_with( + 'columns', + 'rows', + self.mock_target_area, + mock_values, + maximum_weight_mode=False, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) with self.subTest('Pre-existing EWA information'): mock_ll2cr.reset_mock() @@ -338,26 +400,38 @@ def test_resample_ewa(self, mock_ll2cr, mock_fornav, mock_get_values, mock_write_output.reset_mock() ewa_information = { - ('/lat', '/lon'): {'columns': 'old_columns', - 'rows': 'old_rows', - 'target_area': self.mock_target_area} + ('/lat', '/lon'): { + 'columns': 'old_columns', + 'rows': 'old_rows', + 'target_area': self.mock_target_area, + } } - resample_variable(message_parameters, variable_name, - ewa_information, output_path, self.logger, - self.var_info) + resample_variable( + message_parameters, + variable_name, + ewa_information, + output_path, + self.logger, + self.var_info, + ) mock_ll2cr.assert_not_called() - mock_fornav.assert_called_once_with('old_columns', 'old_rows', - self.mock_target_area, - mock_values, - maximum_weight_mode=False) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - ewa_information, - {}) + mock_fornav.assert_called_once_with( + 'old_columns', + 'old_rows', + self.mock_target_area, + mock_values, + maximum_weight_mode=False, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + ewa_information, + {}, + ) @patch('swath_projector.interpolation.write_single_band_output') @patch('swath_projector.interpolation.get_swath_definition') @@ -365,13 +439,19 @@ def test_resample_ewa(self, mock_ll2cr, mock_fornav, mock_get_values, @patch('swath_projector.interpolation.get_variable_values') @patch('swath_projector.interpolation.fornav') @patch('swath_projector.interpolation.ll2cr') - def test_resample_ewa_nn(self, mock_ll2cr, mock_fornav, mock_get_values, - mock_get_target_area, mock_get_swath, - mock_write_output): - """ EWA-NN interpolation should call both ll2cr and fornav if there are - no matching entries for the coordinates in the reprojection - information. If there is an entry, then only fornav should - be called. + def test_resample_ewa_nn( + self, + mock_ll2cr, + mock_fornav, + mock_get_values, + mock_get_target_area, + mock_get_swath, + mock_write_output, + ): + """EWA-NN interpolation should call both ll2cr and fornav if there are + no matching entries for the coordinates in the reprojection + information. If there is an entry, then only fornav should + be called. """ mock_ll2cr.return_value = ['swath_points_in_grid', 'columns', 'rows'] results = np.array([5.0]) @@ -387,26 +467,39 @@ def test_resample_ewa_nn(self, mock_ll2cr, mock_fornav, mock_get_values, output_path = 'path/to/output' with self.subTest('No pre-existing EWA-NN information'): - resample_variable(message_parameters, variable_name, {}, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + {}, + output_path, + self.logger, + self.var_info, + ) expected_cache = { - ('/lat', '/lon'): {'columns': 'columns', - 'rows': 'rows', - 'target_area': self.mock_target_area} + ('/lat', '/lon'): { + 'columns': 'columns', + 'rows': 'rows', + 'target_area': self.mock_target_area, + } } mock_ll2cr.assert_called_once_with('swath', self.mock_target_area) - mock_fornav.assert_called_once_with('columns', 'rows', - self.mock_target_area, - mock_values, - maximum_weight_mode=True) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_fornav.assert_called_once_with( + 'columns', + 'rows', + self.mock_target_area, + mock_values, + maximum_weight_mode=True, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) with self.subTest('Pre-existing EWA-NN information'): mock_ll2cr.reset_mock() @@ -414,25 +507,38 @@ def test_resample_ewa_nn(self, mock_ll2cr, mock_fornav, mock_get_values, mock_write_output.reset_mock() ewa_nn_information = { - ('/lat', '/lon'): {'columns': 'old_columns', - 'rows': 'old_rows', - 'target_area': self.mock_target_area}} + ('/lat', '/lon'): { + 'columns': 'old_columns', + 'rows': 'old_rows', + 'target_area': self.mock_target_area, + } + } - resample_variable(message_parameters, variable_name, - ewa_nn_information, output_path, self.logger, - self.var_info) + resample_variable( + message_parameters, + variable_name, + ewa_nn_information, + output_path, + self.logger, + self.var_info, + ) mock_ll2cr.assert_not_called() - mock_fornav.assert_called_once_with('old_columns', 'old_rows', - self.mock_target_area, - mock_values, - maximum_weight_mode=True) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - ewa_nn_information, - {}) + mock_fornav.assert_called_once_with( + 'old_columns', + 'old_rows', + self.mock_target_area, + mock_values, + maximum_weight_mode=True, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + ewa_nn_information, + {}, + ) with self.subTest('Harmony message defines target area'): mock_get_target_area.reset_mock() @@ -440,39 +546,52 @@ def test_resample_ewa_nn(self, mock_ll2cr, mock_fornav, mock_get_values, mock_fornav.reset_mock() mock_write_output.reset_mock() - harmony_target_area = MagicMock(spec=AreaDefinition, - area_id=HARMONY_TARGET, - shape='harmony_shape') + harmony_target_area = MagicMock( + spec=AreaDefinition, area_id=HARMONY_TARGET, shape='harmony_shape' + ) cache = {HARMONY_TARGET: {'target_area': harmony_target_area}} - resample_variable(message_parameters, variable_name, cache, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + cache, + output_path, + self.logger, + self.var_info, + ) # Check that there is a new entry in the cache, and that it only # contains references to the original Harmony target area object, # not copies of those objects. expected_cache = { HARMONY_TARGET: {'target_area': harmony_target_area}, - ('/lat', '/lon'): {'columns': 'columns', - 'rows': 'rows', - 'target_area': harmony_target_area} + ('/lat', '/lon'): { + 'columns': 'columns', + 'rows': 'rows', + 'target_area': harmony_target_area, + }, } self.assertDictEqual(cache, expected_cache) mock_ll2cr.assert_called_once_with('swath', harmony_target_area) - mock_fornav.assert_called_once_with('columns', 'rows', - harmony_target_area, - mock_values, - maximum_weight_mode=True) + mock_fornav.assert_called_once_with( + 'columns', + 'rows', + harmony_target_area, + mock_values, + maximum_weight_mode=True, + ) # The Harmony target area should be given to the output function - mock_write_output.assert_called_once_with(harmony_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_write_output.assert_called_once_with( + harmony_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) mock_get_target_area.assert_not_called() @patch('swath_projector.interpolation.write_single_band_output') @@ -481,17 +600,27 @@ def test_resample_ewa_nn(self, mock_ll2cr, mock_fornav, mock_get_values, @patch('swath_projector.interpolation.get_variable_values') @patch('swath_projector.interpolation.get_sample_from_neighbour_info') @patch('swath_projector.interpolation.get_neighbour_info') - def test_resample_nearest(self, mock_get_info, mock_get_sample, - mock_get_values, mock_get_target_area, - mock_get_swath, mock_write_output): - """ Nearest neighbour interpolation should call both get_neighbour_info - and get_sample_from_neighbour_info if there are no matching entries - for the coordinates in the reprojection information. If there is an - entry, then only get_sample_from_neighbour_info should be called. + def test_resample_nearest( + self, + mock_get_info, + mock_get_sample, + mock_get_values, + mock_get_target_area, + mock_get_swath, + mock_write_output, + ): + """Nearest neighbour interpolation should call both get_neighbour_info + and get_sample_from_neighbour_info if there are no matching entries + for the coordinates in the reprojection information. If there is an + entry, then only get_sample_from_neighbour_info should be called. """ - mock_get_info.return_value = ['valid_input_index', 'valid_output_index', - 'index_array', 'distance_array'] + mock_get_info.return_value = [ + 'valid_input_index', + 'valid_output_index', + 'index_array', + 'distance_array', + ] results = np.array([4.0]) mock_get_sample.return_value = results mock_get_swath.return_value = 'swath' @@ -506,35 +635,50 @@ def test_resample_nearest(self, mock_get_info, mock_get_sample, alpha_var_fill = 0.0 with self.subTest('No pre-existing nearest neighbour information'): - resample_variable(message_parameters, variable_name, {}, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + {}, + output_path, + self.logger, + self.var_info, + ) expected_cache = { - ('/lat', '/lon'): {'valid_input_index': 'valid_input_index', - 'valid_output_index': 'valid_output_index', - 'index_array': 'index_array', - 'distance_array': 'distance_array', - 'target_area': self.mock_target_area} + ('/lat', '/lon'): { + 'valid_input_index': 'valid_input_index', + 'valid_output_index': 'valid_output_index', + 'index_array': 'index_array', + 'distance_array': 'distance_array', + 'target_area': self.mock_target_area, + } } - mock_get_info.assert_called_once_with('swath', - self.mock_target_area, - RADIUS_OF_INFLUENCE, - epsilon=EPSILON, - neighbours=1) - mock_get_sample.assert_called_once_with('nn', 'ta_shape', - mock_values, - 'valid_input_index', - 'valid_output_index', - 'index_array', - distance_array='distance_array', - fill_value=alpha_var_fill) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_get_info.assert_called_once_with( + 'swath', + self.mock_target_area, + RADIUS_OF_INFLUENCE, + epsilon=EPSILON, + neighbours=1, + ) + mock_get_sample.assert_called_once_with( + 'nn', + 'ta_shape', + mock_values, + 'valid_input_index', + 'valid_output_index', + 'index_array', + distance_array='distance_array', + fill_value=alpha_var_fill, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) with self.subTest('Pre-existing nearest neighbour information'): mock_get_info.reset_mock() @@ -551,23 +695,34 @@ def test_resample_nearest(self, mock_get_info, mock_get_sample, } } - resample_variable(message_parameters, variable_name, - nearest_information, output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + nearest_information, + output_path, + self.logger, + self.var_info, + ) mock_get_info.assert_not_called() - mock_get_sample.assert_called_once_with('nn', 'ta_shape', - mock_values, - 'old_valid_input', - 'old_valid_output', - 'old_index_array', - distance_array='old_distance', - fill_value=alpha_var_fill) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - nearest_information, - {}) + mock_get_sample.assert_called_once_with( + 'nn', + 'ta_shape', + mock_values, + 'old_valid_input', + 'old_valid_output', + 'old_index_array', + distance_array='old_distance', + fill_value=alpha_var_fill, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + nearest_information, + {}, + ) with self.subTest('Harmony message defines target area'): mock_get_target_area.reset_mock() @@ -575,14 +730,20 @@ def test_resample_nearest(self, mock_get_info, mock_get_sample, mock_get_sample.reset_mock() mock_write_output.reset_mock() - harmony_target_area = MagicMock(spec=AreaDefinition, - area_id=HARMONY_TARGET, - shape='harmony_shape') + harmony_target_area = MagicMock( + spec=AreaDefinition, area_id=HARMONY_TARGET, shape='harmony_shape' + ) cache = {HARMONY_TARGET: {'target_area': harmony_target_area}} - resample_variable(message_parameters, variable_name, cache, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + cache, + output_path, + self.logger, + self.var_info, + ) # Check that there is a new entry in the cache, and that it only # contains references to the original Harmony target area object, @@ -595,31 +756,38 @@ def test_resample_nearest(self, mock_get_info, mock_get_sample, 'index_array': 'index_array', 'distance_array': 'distance_array', 'target_area': harmony_target_area, - } + }, } self.assertDictEqual(cache, expected_cache) mock_get_target_area.assert_not_called() - mock_get_info.assert_called_once_with('swath', - harmony_target_area, - RADIUS_OF_INFLUENCE, - epsilon=EPSILON, - neighbours=1) - mock_get_sample.assert_called_once_with('nn', 'harmony_shape', - mock_values, - 'valid_input_index', - 'valid_output_index', - 'index_array', - distance_array='distance_array', - fill_value=alpha_var_fill) + mock_get_info.assert_called_once_with( + 'swath', + harmony_target_area, + RADIUS_OF_INFLUENCE, + epsilon=EPSILON, + neighbours=1, + ) + mock_get_sample.assert_called_once_with( + 'nn', + 'harmony_shape', + mock_values, + 'valid_input_index', + 'valid_output_index', + 'index_array', + distance_array='distance_array', + fill_value=alpha_var_fill, + ) # The Harmony target area should be given to the output function - mock_write_output.assert_called_once_with(harmony_target_area, - results, - variable_name, - output_path, - expected_cache, - {}) + mock_write_output.assert_called_once_with( + harmony_target_area, + results, + variable_name, + output_path, + expected_cache, + {}, + ) @patch('swath_projector.interpolation.write_single_band_output') @patch('swath_projector.interpolation.get_swath_definition') @@ -627,17 +795,27 @@ def test_resample_nearest(self, mock_get_info, mock_get_sample, @patch('swath_projector.interpolation.get_variable_values') @patch('swath_projector.interpolation.get_sample_from_neighbour_info') @patch('swath_projector.interpolation.get_neighbour_info') - def test_resample_scaled_variable(self, mock_get_info, mock_get_sample, - mock_get_values, mock_get_target_area, - mock_get_swath, mock_write_output): - """ Ensure that an input variable that contains scaling attributes, - `add_offset` and `scale_factor` passes those attributes to the - function that writes the intermediate output, so that the variable - in that dataset is also correctly scaled. + def test_resample_scaled_variable( + self, + mock_get_info, + mock_get_sample, + mock_get_values, + mock_get_target_area, + mock_get_swath, + mock_write_output, + ): + """Ensure that an input variable that contains scaling attributes, + `add_offset` and `scale_factor` passes those attributes to the + function that writes the intermediate output, so that the variable + in that dataset is also correctly scaled. """ - mock_get_info.return_value = ['valid_input_index', 'valid_output_index', - 'index_array', 'distance_array'] + mock_get_info.return_value = [ + 'valid_input_index', + 'valid_output_index', + 'index_array', + 'distance_array', + ] results = np.array([4.0]) mock_get_sample.return_value = results mock_get_swath.return_value = 'swath' @@ -651,39 +829,54 @@ def test_resample_scaled_variable(self, mock_get_info, mock_get_sample, output_path = 'path/to/output' blue_var_fill = 0.0 - resample_variable(message_parameters, variable_name, {}, - output_path, self.logger, self.var_info) + resample_variable( + message_parameters, + variable_name, + {}, + output_path, + self.logger, + self.var_info, + ) expected_cache = { - ('/lat', '/lon'): {'valid_input_index': 'valid_input_index', - 'valid_output_index': 'valid_output_index', - 'index_array': 'index_array', - 'distance_array': 'distance_array', - 'target_area': self.mock_target_area} + ('/lat', '/lon'): { + 'valid_input_index': 'valid_input_index', + 'valid_output_index': 'valid_output_index', + 'index_array': 'index_array', + 'distance_array': 'distance_array', + 'target_area': self.mock_target_area, + } } expected_scaling = {'add_offset': 0, 'scale_factor': 2} - mock_get_info.assert_called_once_with('swath', - self.mock_target_area, - RADIUS_OF_INFLUENCE, - epsilon=EPSILON, - neighbours=1) - mock_get_sample.assert_called_once_with('nn', 'ta_shape', - mock_values, - 'valid_input_index', - 'valid_output_index', - 'index_array', - distance_array='distance_array', - fill_value=blue_var_fill) - mock_write_output.assert_called_once_with(self.mock_target_area, - results, - variable_name, - output_path, - expected_cache, - expected_scaling) + mock_get_info.assert_called_once_with( + 'swath', + self.mock_target_area, + RADIUS_OF_INFLUENCE, + epsilon=EPSILON, + neighbours=1, + ) + mock_get_sample.assert_called_once_with( + 'nn', + 'ta_shape', + mock_values, + 'valid_input_index', + 'valid_output_index', + 'index_array', + distance_array='distance_array', + fill_value=blue_var_fill, + ) + mock_write_output.assert_called_once_with( + self.mock_target_area, + results, + variable_name, + output_path, + expected_cache, + expected_scaling, + ) def test_check_for_valid_interpolation(self): - """ Ensure all valid interpolations don't raise an exception. """ + """Ensure all valid interpolations don't raise an exception.""" interpolations = ['bilinear', 'ewa', 'ewa-nn', 'near'] for interpolation in interpolations: @@ -697,10 +890,10 @@ def test_check_for_valid_interpolation(self): check_for_valid_interpolation(parameters, self.logger) def test_get_swath_definition(self): - """ Ensure a valid SwathDefinition object can be created for a dataset - with coordinates. The shape of the swath definition should match - the shapes of the input coordinates, and the longitude and latitude - values should be correctly stored in the swath definition. + """Ensure a valid SwathDefinition object can be created for a dataset + with coordinates. The shape of the swath definition should match + the shapes of the input coordinates, and the longitude and latitude + values should be correctly stored in the swath definition. """ dataset = Dataset('tests/data/africa.nc') @@ -714,9 +907,9 @@ def test_get_swath_definition(self): np.testing.assert_array_equal(latitudes, swath_definition.lats) def test_get_swath_definition_wrapping_longitudes(self): - """ Ensure that a dataset with coordinates that have longitude ranging - from 0 to 360 degrees will produce a valid SwathDefinition object, - with the longitudes ranging from -180 degrees to 180 degrees. + """Ensure that a dataset with coordinates that have longitude ranging + from 0 to 360 degrees will produce a valid SwathDefinition object, + with the longitudes ranging from -180 degrees to 180 degrees. """ dataset = Dataset('test.nc', 'w', diskless=True) @@ -727,10 +920,10 @@ def test_get_swath_definition_wrapping_longitudes(self): dataset.createDimension('lat', size=2) dataset.createDimension('lon', size=2) - dataset.createVariable('latitude', lat_values.dtype, - dimensions=('lat', 'lon')) - dataset.createVariable('longitude', raw_lon_values.dtype, - dimensions=('lat', 'lon')) + dataset.createVariable('latitude', lat_values.dtype, dimensions=('lat', 'lon')) + dataset.createVariable( + 'longitude', raw_lon_values.dtype, dimensions=('lat', 'lon') + ) dataset['longitude'][:] = raw_lon_values[:] dataset['latitude'][:] = lat_values[:] @@ -743,9 +936,9 @@ def test_get_swath_definition_wrapping_longitudes(self): dataset.close() def test_get_swath_definition_one_dimensional_coordinates(self): - """ Ensure that if 1-D coordinate arrays are used to produce a swath, - they are converted to 2-D before being used to construct the - object. + """Ensure that if 1-D coordinate arrays are used to produce a swath, + they are converted to 2-D before being used to construct the + object. """ dataset = Dataset('test_1d.nc', 'w', diskless=True) @@ -757,10 +950,8 @@ def test_get_swath_definition_one_dimensional_coordinates(self): dataset.createDimension('lat', size=3) dataset.createDimension('lon', size=3) - dataset.createVariable('latitude', lat_values.dtype, - dimensions=('lat',)) - dataset.createVariable('longitude', lon_values.dtype, - dimensions=('lon',)) + dataset.createVariable('latitude', lat_values.dtype, dimensions=('lat',)) + dataset.createVariable('longitude', lon_values.dtype, dimensions=('lon',)) dataset['longitude'][:] = lon_values[:] dataset['latitude'][:] = lat_values[:] @@ -773,17 +964,16 @@ def test_get_swath_definition_one_dimensional_coordinates(self): dataset.close() def test_get_reprojection_cache_minimal(self): - """ If a Harmony message does not contain any target area information, - then an empty cache should be retrieved. + """If a Harmony message does not contain any target area information, + then an empty cache should be retrieved. """ - self.assertDictEqual(get_reprojection_cache(self.message_parameters), - {}) + self.assertDictEqual(get_reprojection_cache(self.message_parameters), {}) def test_get_cache_information_extents(self): - """ If a Harmony message defines the extents of a target area, but - neither dimensions nor resolutions, then an empty cache should be - retrieved. + """If a Harmony message defines the extents of a target area, but + neither dimensions nor resolutions, then an empty cache should be + retrieved. """ message_parameters = self.message_parameters @@ -795,9 +985,9 @@ def test_get_cache_information_extents(self): self.assertDictEqual(get_reprojection_cache(message_parameters), {}) def test_get_reprojection_cache_extents_resolutions(self): - """ If a Harmony message defines the target area extents and - resolutions, the returned cache should contain an entry that will - be used for all variables. + """If a Harmony message defines the target area extents and + resolutions, the returned cache should contain an entry that will + be used for all variables. """ message_parameters = self.message_parameters @@ -812,7 +1002,7 @@ def test_get_reprojection_cache_extents_resolutions(self): HARMONY_TARGET, message_parameters['projection'].definition_string(), (10, 20), - (-10, -5, 10, 5) + (-10, -5, 10, 5), ) cache = get_reprojection_cache(message_parameters) @@ -821,14 +1011,13 @@ def test_get_reprojection_cache_extents_resolutions(self): self.assertSetEqual(set(cache[HARMONY_TARGET].keys()), {'target_area'}) self.assert_areadefinitions_equal( - cache[HARMONY_TARGET]['target_area'], - expected_target_area + cache[HARMONY_TARGET]['target_area'], expected_target_area ) def test_get_reprojection_cache_extents_dimensions(self): - """ If the Harmony message defines the target area extents and - dimensions, the returned cache should contain an entry that will be - used for all variables. + """If the Harmony message defines the target area extents and + dimensions, the returned cache should contain an entry that will be + used for all variables. """ message_parameters = self.message_parameters @@ -843,7 +1032,7 @@ def test_get_reprojection_cache_extents_dimensions(self): HARMONY_TARGET, message_parameters['projection'].definition_string(), (10, 20), - (-10, -5, 10, 5) + (-10, -5, 10, 5), ) cache = get_reprojection_cache(message_parameters) @@ -851,12 +1040,13 @@ def test_get_reprojection_cache_extents_dimensions(self): self.assertIn(HARMONY_TARGET, cache) self.assertSetEqual(set(cache[HARMONY_TARGET].keys()), {'target_area'}) - self.assert_areadefinitions_equal(cache[HARMONY_TARGET]['target_area'], - expected_target_area) + self.assert_areadefinitions_equal( + cache[HARMONY_TARGET]['target_area'], expected_target_area + ) def test_get_reprojection_cache_dimensions(self): - """ If the Harmony message defines the target area dimensions, but not - the extents, then the retrieved cache should be empty. + """If the Harmony message defines the target area dimensions, but not + the extents, then the retrieved cache should be empty. """ message_parameters = self.message_parameters @@ -866,8 +1056,8 @@ def test_get_reprojection_cache_dimensions(self): self.assertDictEqual(get_reprojection_cache(message_parameters), {}) def test_get_reprojection_cache_resolutions(self): - """ If the Harmony message defines the target area resolutions, but not - the extents, then the retrieved cache should be empty. + """If the Harmony message defines the target area resolutions, but not + the extents, then the retrieved cache should be empty. """ message_parameters = self.message_parameters @@ -879,16 +1069,17 @@ def test_get_reprojection_cache_resolutions(self): @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_minimal(self, mock_get_coordinates, - mock_get_extents, mock_get_resolution): - """ If the Harmony message does not define a target area, then that - information should be derived from the coordinate variables - referred to in the variable metadata. - - Note: These unit tests are primarily to make sure the correct - combinations of message and variable-specific parameters are being - used. The full functional test comes from those in the main `test` - directory. + def test_get_target_area_minimal( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message does not define a target area, then that + information should be derived from the coordinate variables + referred to in the variable metadata. + + Note: These unit tests are primarily to make sure the correct + combinations of message and variable-specific parameters are being + used. The full functional test comes from those in the main `test` + directory. """ latitudes = 'lats' @@ -902,20 +1093,20 @@ def test_get_target_area_minimal(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (20, 20), - (-20, 0, 20, 40) + (-20, 0, 20, 40), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', ('/lat', '/lon'), - self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_called_once_with( self.message_parameters['projection'], longitudes, latitudes ) @@ -928,11 +1119,12 @@ def test_get_target_area_minimal(self, mock_get_coordinates, @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_extents(self, mock_get_coordinates, - mock_get_extents, mock_get_resolution): - """ If the Harmony message defines the target area extents, these - should be used, with the dimensions and resolution of the output - being defined by the coordinate data from the variable. + def test_get_target_area_extents( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message defines the target area extents, these + should be used, with the dimensions and resolution of the output + being defined by the coordinate data from the variable. """ latitudes = 'lats' @@ -952,20 +1144,20 @@ def test_get_target_area_extents(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (5, 10), - (-10, -5, 10, 5) + (-10, -5, 10, 5), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', ('/lat', '/lon'), - self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_not_called() mock_get_resolution.assert_called_once_with( self.message_parameters['projection'], longitudes, latitudes @@ -976,14 +1168,14 @@ def test_get_target_area_extents(self, mock_get_coordinates, @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_extents_resolutions(self, mock_get_coordinates, - mock_get_extents, - mock_get_resolution): - """ If the Harmony message defines the target area extents and - resolutions, these should be used for the target area definition. - Note, this shouldn't happen in practice, as it should result in a - global definition being defined when the reprojection cache is - instantiated. + def test_get_target_area_extents_resolutions( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message defines the target area extents and + resolutions, these should be used for the target area definition. + Note, this shouldn't happen in practice, as it should result in a + global definition being defined when the reprojection cache is + instantiated. """ latitudes = 'lats' @@ -1005,20 +1197,20 @@ def test_get_target_area_extents_resolutions(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (10, 20), - (-10, -5, 10, 5) + (-10, -5, 10, 5), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', ('/lat', '/lon'), - self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_not_called() mock_get_resolution.assert_not_called() @@ -1027,14 +1219,14 @@ def test_get_target_area_extents_resolutions(self, mock_get_coordinates, @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_extents_dimensions(self, mock_get_coordinates, - mock_get_extents, - mock_get_resolution): - """ If the Harmony message defines the target area extents and - dimensions, these should be used for the target area definition. - Note, this shouldn't happen in practice, as it should result in a - global definition being defined when the reprojection cache is - instantiated. + def test_get_target_area_extents_dimensions( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message defines the target area extents and + dimensions, these should be used for the target area definition. + Note, this shouldn't happen in practice, as it should result in a + global definition being defined when the reprojection cache is + instantiated. """ latitudes = 'lats' @@ -1055,20 +1247,20 @@ def test_get_target_area_extents_dimensions(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (10, 10), - (-10, -5, 10, 5) + (-10, -5, 10, 5), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', ('/lat', '/lon'), - self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_not_called() mock_get_resolution.assert_not_called() @@ -1077,11 +1269,12 @@ def test_get_target_area_extents_dimensions(self, mock_get_coordinates, @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_dimensions(self, mock_get_coordinates, - mock_get_extents, mock_get_resolution): - """ If the Harmony message defines the target area dimensions, then - that information should be used, along with the extents as - defined by the variables associated coordinates. + def test_get_target_area_dimensions( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message defines the target area dimensions, then + that information should be used, along with the extents as + defined by the variables associated coordinates. """ latitudes = 'lats' @@ -1098,20 +1291,20 @@ def test_get_target_area_dimensions(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (10, 10), - (-20, 0, 20, 40) + (-20, 0, 20, 40), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', ('/lat', '/lon'), - self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_called_once_with( message_parameters['projection'], longitudes, latitudes ) @@ -1122,12 +1315,12 @@ def test_get_target_area_dimensions(self, mock_get_coordinates, @patch('swath_projector.interpolation.get_projected_resolution') @patch('swath_projector.interpolation.get_extents_from_perimeter') @patch('swath_projector.interpolation.get_coordinate_variable') - def test_get_target_area_resolutions(self, mock_get_coordinates, - mock_get_extents, - mock_get_resolution): - """ If the Harmony message defines the target area resolutions, then - that information should be used, along with the extents as - defined by the variables associated coordinates. + def test_get_target_area_resolutions( + self, mock_get_coordinates, mock_get_extents, mock_get_resolution + ): + """If the Harmony message defines the target area resolutions, then + that information should be used, along with the extents as + defined by the variables associated coordinates. """ latitudes = 'lats' @@ -1145,20 +1338,20 @@ def test_get_target_area_resolutions(self, mock_get_coordinates, '/lat, /lon', self.message_parameters['projection'].definition_string(), (8, 10), - (-20, 0, 20, 40) + (-20, 0, 20, 40), ) - target_area = get_target_area(self.message_parameters, - 'coordinate_group', - ('/lat', '/lon'), self.logger) + target_area = get_target_area( + self.message_parameters, 'coordinate_group', ('/lat', '/lon'), self.logger + ) self.assertEqual(mock_get_coordinates.call_count, 2) - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lat') - mock_get_coordinates.assert_any_call('coordinate_group', - ('/lat', '/lon'), - 'lon') + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lat' + ) + mock_get_coordinates.assert_any_call( + 'coordinate_group', ('/lat', '/lon'), 'lon' + ) mock_get_extents.assert_called_once_with( message_parameters['projection'], longitudes, latitudes ) @@ -1167,21 +1360,24 @@ def test_get_target_area_resolutions(self, mock_get_coordinates, self.assert_areadefinitions_equal(target_area, expected_target_area) def test_get_parameters_tuple(self): - """ Ensure that the function behaves correctly when all, some or none - of the requested parameters are not `None` in the input dictionary. + """Ensure that the function behaves correctly when all, some or none + of the requested parameters are not `None` in the input dictionary. - If any of the requested parameters are `None`, then the function - should return `None`. Otherwise a tuple of the corresponding values - will be returned, in the requested order. + If any of the requested parameters are `None`, then the function + should return `None`. Otherwise a tuple of the corresponding values + will be returned, in the requested order. """ input_parameters = {'a': 1, 'b': 2, 'c': None} - test_args = [['All valid', ['a', 'b'], (1, 2)], - ['Some valid', ['a', 'b', 'c'], None], - ['None valid', ['c'], None]] + test_args = [ + ['All valid', ['a', 'b'], (1, 2)], + ['Some valid', ['a', 'b', 'c'], None], + ['None valid', ['c'], None], + ] for description, keys, expected_output in test_args: with self.subTest(description): - self.assertEqual(get_parameters_tuple(input_parameters, keys), - expected_output) + self.assertEqual( + get_parameters_tuple(input_parameters, keys), expected_output + ) diff --git a/tests/unit/test_nc_merge.py b/tests/unit/test_nc_merge.py index 137a397..f4c67dd 100644 --- a/tests/unit/test_nc_merge.py +++ b/tests/unit/test_nc_merge.py @@ -16,7 +16,7 @@ get_fill_value_from_attributes, get_science_variable_attributes, get_science_variable_dimensions, - read_attrs + read_attrs, ) from swath_projector.reproject import CF_CONFIG_FILE @@ -26,24 +26,37 @@ class TestNCMerge(TestCase): @classmethod def setUpClass(cls): cls.logger = logging.getLogger('nc_merge test') - cls.properties = {'input_file': 'tests/data/VNL2_test_data.nc', - 'granule_url': 'tests/data/VNL2_test_data.nc', - 'crs': 'EPSG:4326', - 'interpolation': 'bilinear'} + cls.properties = { + 'input_file': 'tests/data/VNL2_test_data.nc', + 'granule_url': 'tests/data/VNL2_test_data.nc', + 'crs': 'EPSG:4326', + 'interpolation': 'bilinear', + } cls.tmp_dir = 'tests/data/test_tmp/' cls.output_file = 'tests/data/VNL2_test_data_repr.nc' - cls.science_variables = {'/brightness_temperature_4um', - '/satellite_zenith_angle', - '/sea_surface_temperature', '/wind_speed'} + cls.science_variables = { + '/brightness_temperature_4um', + '/satellite_zenith_angle', + '/sea_surface_temperature', + '/wind_speed', + } cls.metadata_variables = set() - cls.var_info = VarInfoFromNetCDF4(cls.properties['input_file'], - short_name='VIIRS_NPP-NAVO-L2P-v3.0', - config_file=CF_CONFIG_FILE) - create_output(cls.properties, cls.output_file, cls.tmp_dir, - cls.science_variables, cls.metadata_variables, - cls.logger, cls.var_info) + cls.var_info = VarInfoFromNetCDF4( + cls.properties['input_file'], + short_name='VIIRS_NPP-NAVO-L2P-v3.0', + config_file=CF_CONFIG_FILE, + ) + create_output( + cls.properties, + cls.output_file, + cls.tmp_dir, + cls.science_variables, + cls.metadata_variables, + cls.logger, + cls.var_info, + ) @classmethod def tearDownClass(cls): @@ -51,12 +64,11 @@ def tearDownClass(cls): os.remove(cls.output_file) def test_output_has_all_variables(self): - """ Output file has all expected variables from the input file. """ + """Output file has all expected variables from the input file.""" with Dataset(self.output_file, 'r') as output_dataset: # Output has all projected science variables: for expected_variable in self.science_variables: - self.assertIn(expected_variable.lstrip('/'), - output_dataset.variables) + self.assertIn(expected_variable.lstrip('/'), output_dataset.variables) # Output also has a CRS grid_mapping variable, and three dimensions: self.assertIn('latitude_longitude', output_dataset.variables) @@ -64,28 +76,36 @@ def test_output_has_all_variables(self): self.assertIn(expected_dimension, output_dataset.variables) def test_same_dimensions(self): - """ Corresponding variables in input and output should have the same - number of dimensions. + """Corresponding variables in input and output should have the same + number of dimensions. """ test_dataset = 'sea_surface_temperature' in_dataset = Dataset(self.properties['input_file']) out_dataset = Dataset(self.output_file) - self.assertEqual(len(in_dataset[test_dataset].dimensions), - len(out_dataset[test_dataset].dimensions)) + self.assertEqual( + len(in_dataset[test_dataset].dimensions), + len(out_dataset[test_dataset].dimensions), + ) @patch('swath_projector.nc_merge.datetime') def test_output_global_attributes(self, mock_datetime): - """ The root group of the output files should contain the global - attributes of the input file, with the addition of `history` (if - not originally present) and `history_json`. + """The root group of the output files should contain the global + attributes of the input file, with the addition of `history` (if + not originally present) and `history_json`. """ mock_datetime.utcnow = Mock(return_value=datetime(2021, 5, 12, 19, 3, 4)) - create_output(self.properties, self.output_file, self.tmp_dir, - self.science_variables, self.metadata_variables, - self.logger, self.var_info) + create_output( + self.properties, + self.output_file, + self.tmp_dir, + self.science_variables, + self.metadata_variables, + self.logger, + self.var_info, + ) with Dataset(self.properties['input_file']) as in_dataset: input_attrs = read_attrs(in_dataset) @@ -114,31 +134,40 @@ def test_output_global_attributes(self, mock_datetime): self.assertEqual(output_attrs['history'], expected_history) self.assertNotIn('History', output_attrs.keys()) - expected_history_json = [{ - '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', - 'date_time': '2021-05-12T19:03:04+00:00', - 'program': 'sds/harmony-swath-projector', - 'version': '0.9.0', - 'parameters': {'input_file': 'tests/data/VNL2_test_data.nc', - 'crs': 'EPSG:4326', - 'interpolation': 'bilinear'}, - 'derived_from': 'tests/data/VNL2_test_data.nc', - 'cf_history': [('Mon Dec 9 11:22:11 2019: ncks -v ' - 'sea_surface_temperature,satellite_zenith_angle,' - 'brightness_temperature_4um,wind_speed ' - '/Users/yzhang29/Desktop/NCOTest/' - 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS_NPP-v02.0-fv03.0.nc ' - '/Users/yzhang29/Desktop/NCOTest/VNL2_test_data.nc'), - 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT'], - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST' - }] + expected_history_json = [ + { + '$schema': 'https://harmony.earthdata.nasa.gov/schemas/history/0.1.0/history-v0.1.0.json', + 'date_time': '2021-05-12T19:03:04+00:00', + 'program': 'sds/harmony-swath-projector', + 'version': '0.9.0', + 'parameters': { + 'input_file': 'tests/data/VNL2_test_data.nc', + 'crs': 'EPSG:4326', + 'interpolation': 'bilinear', + }, + 'derived_from': 'tests/data/VNL2_test_data.nc', + 'cf_history': [ + ( + 'Mon Dec 9 11:22:11 2019: ncks -v ' + 'sea_surface_temperature,satellite_zenith_angle,' + 'brightness_temperature_4um,wind_speed ' + '/Users/yzhang29/Desktop/NCOTest/' + 'VNL2PSST_20190109000457-NAVO-L2P_GHRSST-SST1m-VIIRS_NPP-v02.0-fv03.0.nc ' + '/Users/yzhang29/Desktop/NCOTest/VNL2_test_data.nc' + ), + 'Created with VIIRSseatemp on 2019/01/09 at 00:57:15 UT', + ], + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', + } + ] self.assertIn('history_json', output_attrs.keys()) - self.assertEqual(json.loads(output_attrs['history_json']), - expected_history_json) + self.assertEqual( + json.loads(output_attrs['history_json']), expected_history_json + ) def test_same_num_of_dataset_attributes(self): - """ Variables in input should have the same number of attributes. """ + """Variables in input should have the same number of attributes.""" test_variable = 'sea_surface_temperature' in_dataset = Dataset(self.properties['input_file']) out_dataset = Dataset(self.output_file) @@ -149,7 +178,7 @@ def test_same_num_of_dataset_attributes(self): self.assertEqual(len(input_attrs), len(output_attrs)) def test_same_data_type(self): - """ Variables in input and output should have same data type. """ + """Variables in input and output should have same data type.""" test_variable = 'sea_surface_temperature' in_dataset = Dataset(self.properties['input_file']) out_dataset = Dataset(self.output_file) @@ -158,43 +187,49 @@ def test_same_data_type(self): self.assertEqual(input_data_type, output_data_type, 'Should be equal') def test_missing_file_raises_error(self): - """ If a science variable should be included in the output, but there - is no associated output file, an exception should be raised. + """If a science variable should be included in the output, but there + is no associated output file, an exception should be raised. """ test_variables = {'missing_variable'} temporary_output_file = 'tests/data/unit_test.nc4' with self.assertRaises(MissingReprojectedDataError): - create_output(self.properties, temporary_output_file, self.tmp_dir, - test_variables, self.metadata_variables, self.logger, self.var_info) + create_output( + self.properties, + temporary_output_file, + self.tmp_dir, + test_variables, + self.metadata_variables, + self.logger, + self.var_info, + ) if os.path.exists(temporary_output_file): os.remove(temporary_output_file) def test_get_fill_value_from_attributes(self): - """ If a variable has a fill value it should be popped from the - dictionary and returned. Otherwise, the default value of `None` - should be returned. + """If a variable has a fill value it should be popped from the + dictionary and returned. Otherwise, the default value of `None` + should be returned. """ with self.subTest('_FillValue present in attributes'): fill_value = 123 attributes = {'_FillValue': fill_value} - self.assertEqual(get_fill_value_from_attributes(attributes), - fill_value) + self.assertEqual(get_fill_value_from_attributes(attributes), fill_value) self.assertNotIn('_FillValue', attributes) with self.subTest('_FillValue absent, returns None'): self.assertEqual(get_fill_value_from_attributes({}), None) def test_check_coord_valid(self): - """ If some of the listed coordinates are not in the single band - output, then the function should return `False`. If any of the - any of the coordinate variables have different shapes in the input - and the single band output, then the function should return - `False`. Otherwise, the function should return `True`. Also check - the case that no coordinates are listed. + """If some of the listed coordinates are not in the single band + output, then the function should return `False`. If any of the + any of the coordinate variables have different shapes in the input + and the single band output, then the function should return + `False`. Otherwise, the function should return `True`. Also check + the case that no coordinates are listed. """ test_dataset_name = 'sea_surface_temperature.nc' @@ -202,28 +237,37 @@ def test_check_coord_valid(self): input_dataset = Dataset(self.properties['input_file']) with self.subTest('No coordinate data returns True'): - self.assertTrue(check_coor_valid(self.var_info, '/lat', - input_dataset, single_band_dataset)) + self.assertTrue( + check_coor_valid( + self.var_info, '/lat', input_dataset, single_band_dataset + ) + ) with self.subTest('Reprojected data missing coordinates returns False'): - self.assertFalse(check_coor_valid(self.var_info, - '/brightness_temperature_4um', - input_dataset, - single_band_dataset)) + self.assertFalse( + check_coor_valid( + self.var_info, + '/brightness_temperature_4um', + input_dataset, + single_band_dataset, + ) + ) with self.subTest('Reprojected data with preserved coordinates returns True'): # To ensure a match, this uses two different reprojected output # files, as these are guaranteed to match coordinate shapes. second_dataset = Dataset(f'{self.tmp_dir}wind_speed.nc') - self.assertTrue(check_coor_valid(self.var_info, '/wind_speed', - second_dataset, - single_band_dataset)) + self.assertTrue( + check_coor_valid( + self.var_info, '/wind_speed', second_dataset, single_band_dataset + ) + ) def test_get_science_variable_dimensions(self): - """ Ensure that the retrieved dimensions match those in the single band - dataset. If the input dataset includes a time dimension, that - should be included in the returned tuple. + """Ensure that the retrieved dimensions match those in the single band + dataset. If the input dataset includes a time dimension, that + should be included in the returned tuple. """ variable_name = 'sea_surface_temperature' @@ -231,25 +275,25 @@ def test_get_science_variable_dimensions(self): input_dataset = Dataset(self.properties['input_file']) with self.subTest('Input dataset has time dimension.'): - dimensions = get_science_variable_dimensions(input_dataset, - single_band_dataset, - variable_name) + dimensions = get_science_variable_dimensions( + input_dataset, single_band_dataset, variable_name + ) self.assertTupleEqual(dimensions, ('time', 'lat', 'lon')) with self.subTest('Input dataset has no time dimension.'): # Using the single_band_dataset as input ensure no time dimension - dimensions = get_science_variable_dimensions(single_band_dataset, - single_band_dataset, - 'lat') + dimensions = get_science_variable_dimensions( + single_band_dataset, single_band_dataset, 'lat' + ) self.assertTupleEqual(dimensions, ('lat',)) @patch('swath_projector.nc_merge.check_coor_valid') def test_get_science_variable_attributes(self, mock_check_coord_valid): - """ The original input metadata should be mostly present. The - `grid_mapping` metadata attribute should be added from the single - band output. If the shapes of the variables listed as coordinates - have changed in reprojection, then the `coordinates` metadata - attribute not be present in the returned attributes. + """The original input metadata should be mostly present. The + `grid_mapping` metadata attribute should be added from the single + band output. If the shapes of the variables listed as coordinates + have changed in reprojection, then the `coordinates` metadata + attribute not be present in the returned attributes. """ variable_name = 'sea_surface_temperature' @@ -258,9 +302,9 @@ def test_get_science_variable_attributes(self, mock_check_coord_valid): with self.subTest('Coordinates remain valid.'): mock_check_coord_valid.return_value = True - attributes = get_science_variable_attributes(input_dataset, - single_band_dataset, - variable_name, self.var_info) + attributes = get_science_variable_attributes( + input_dataset, single_band_dataset, variable_name, self.var_info + ) input_attributes = input_dataset[variable_name].__dict__ single_band_attributes = single_band_dataset[variable_name].__dict__ @@ -271,14 +315,15 @@ def test_get_science_variable_attributes(self, mock_check_coord_valid): self.assertEqual(attributes[attribute_name], attribute_value) self.assertIn('grid_mapping', attributes) - self.assertEqual(attributes['grid_mapping'], - single_band_attributes['grid_mapping']) + self.assertEqual( + attributes['grid_mapping'], single_band_attributes['grid_mapping'] + ) with self.subTest('Coordinates are no longer valid.'): mock_check_coord_valid.return_value = False - attributes = get_science_variable_attributes(input_dataset, - single_band_dataset, - variable_name, self.var_info) + attributes = get_science_variable_attributes( + input_dataset, single_band_dataset, variable_name, self.var_info + ) input_attributes = input_dataset[variable_name].__dict__ single_band_attributes = single_band_dataset[variable_name].__dict__ @@ -288,25 +333,27 @@ def test_get_science_variable_attributes(self, mock_check_coord_valid): for attribute_name, attribute_value in input_attributes.items(): if attribute_name != 'coordinates': self.assertIn(attribute_name, attributes) - self.assertEqual(attributes[attribute_name], - attribute_value) + self.assertEqual(attributes[attribute_name], attribute_value) self.assertIn('grid_mapping', attributes) - self.assertEqual(attributes['grid_mapping'], - single_band_attributes['grid_mapping']) + self.assertEqual( + attributes['grid_mapping'], single_band_attributes['grid_mapping'] + ) @patch('swath_projector.nc_merge.datetime') def test_create_history_record(self, mock_datetime): - """ Ensure a history record is correctly constructed, and only contains - a `cf_history` attribute if there is valid a `history` (or - `History`) attribute specified from the input. + """Ensure a history record is correctly constructed, and only contains + a `cf_history` attribute if there is valid a `history` (or + `History`) attribute specified from the input. """ mock_datetime.utcnow = Mock(return_value=datetime(2001, 2, 3, 4, 5, 6)) granule_url = 'https://example.com/input.nc4' - request_parameters = {'crs': '+proj=longlat', - 'input_file': granule_url, - 'interpolation': 'near'} + request_parameters = { + 'crs': '+proj=longlat', + 'input_file': granule_url, + 'interpolation': 'near', + } with self.subTest('No specified history'): expected_output = { @@ -316,10 +363,11 @@ def test_create_history_record(self, mock_datetime): 'version': '0.9.0', 'parameters': request_parameters, 'derived_from': granule_url, - 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST' + 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', } - self.assertDictEqual(create_history_record(None, request_parameters), - expected_output) + self.assertDictEqual( + create_history_record(None, request_parameters), expected_output + ) string_history = '2000-12-31T00:00:00+00.00 Swathinator v0.0.1' list_history = [string_history] @@ -331,13 +379,17 @@ def test_create_history_record(self, mock_datetime): 'parameters': request_parameters, 'derived_from': granule_url, 'program_ref': 'https://cmr.uat.earthdata.nasa.gov/search/concepts/S1237974711-EEDTEST', - 'cf_history': list_history + 'cf_history': list_history, } with self.subTest('String history specified in input file'): - self.assertDictEqual(create_history_record(string_history, request_parameters), - expected_output_with_history) + self.assertDictEqual( + create_history_record(string_history, request_parameters), + expected_output_with_history, + ) with self.subTest('List history specified in input file'): - self.assertDictEqual(create_history_record(list_history, request_parameters), - expected_output_with_history) + self.assertDictEqual( + create_history_record(list_history, request_parameters), + expected_output_with_history, + ) diff --git a/tests/unit/test_nc_single_band.py b/tests/unit/test_nc_single_band.py index 2394032..68c7c0f 100644 --- a/tests/unit/test_nc_single_band.py +++ b/tests/unit/test_nc_single_band.py @@ -14,7 +14,7 @@ write_grid_mapping, write_science_variable, write_single_band_output, - HARMONY_TARGET + HARMONY_TARGET, ) @@ -24,10 +24,9 @@ class TestNCSingleBand(TestCase): def setUpClass(cls): cls.temp_dir = mkdtemp() cls.area_id = 'lat, lon' - cls.area_definition = AreaDefinition.from_extent(cls.area_id, - '+proj=longlat', - (2, 4), - (-5, 40, 5, 50)) + cls.area_definition = AreaDefinition.from_extent( + cls.area_id, '+proj=longlat', (2, 4), (-5, 40, 5, 50) + ) cls.lat_values = np.array([47.5, 42.5]) cls.lon_values = np.array([-3.75, -1.25, 1.25, 3.75]) @@ -48,8 +47,7 @@ def setUpClass(cls): 'horizontal_datum_name': 'World Geodetic System 1984', } cls.non_geographic_area = AreaDefinition.from_extent( - 'cea', '+proj=cea', (2, 4), - (-3_000_000, 2_000_000, 3_000_000, 4_000_000) + 'cea', '+proj=cea', (2, 4), (-3_000_000, 2_000_000, 3_000_000, 4_000_000) ) @classmethod @@ -57,47 +55,55 @@ def tearDownClass(cls): rmtree(cls.temp_dir) def test_write_single_band_output(self): - """ An overall test that an output file can be produced. This will test - that each varaible has the expected dimensions, values and - attributes, and that the overall `netCDF4.Dataset` contains the - expected dimensions. + """An overall test that an output file can be produced. This will test + that each varaible has the expected dimensions, values and + attributes, and that the overall `netCDF4.Dataset` contains the + expected dimensions. """ output_path = f'{self.temp_dir}/overall_test.nc' - write_single_band_output(self.area_definition, self.reprojected_data, - self.variable_name, output_path, self.cache, - {}) + write_single_band_output( + self.area_definition, + self.reprojected_data, + self.variable_name, + output_path, + self.cache, + {}, + ) with Dataset(output_path) as saved_output: # Check dimensions - self.assertTupleEqual(tuple(saved_output.dimensions.keys()), - ('lat', 'lon')) + self.assertTupleEqual(tuple(saved_output.dimensions.keys()), ('lat', 'lon')) self.assertEqual(saved_output.dimensions['lat'].size, 2) self.assertEqual(saved_output.dimensions['lon'].size, 4) # Check all variables are present self.assertSetEqual( set(saved_output.variables.keys()), - {'lat', 'lon', 'latitude_longitude', self.variable_name} + {'lat', 'lon', 'latitude_longitude', self.variable_name}, ) # Check science variable - np.testing.assert_array_equal(saved_output[self.variable_name][:], - self.reprojected_data) - self.assertTupleEqual(saved_output[self.variable_name].dimensions, - ('lat', 'lon')) - self.assertListEqual(saved_output[self.variable_name].ncattrs(), - ['grid_mapping']) + np.testing.assert_array_equal( + saved_output[self.variable_name][:], self.reprojected_data + ) + self.assertTupleEqual( + saved_output[self.variable_name].dimensions, ('lat', 'lon') + ) + self.assertListEqual( + saved_output[self.variable_name].ncattrs(), ['grid_mapping'] + ) self.assertEqual( saved_output[self.variable_name].getncattr('grid_mapping'), - 'latitude_longitude' + 'latitude_longitude', ) # Check grid_mapping: grid_attributes = saved_output['latitude_longitude'].__dict__ for attribute_name, attribute_value in grid_attributes.items(): - self.assertEqual(attribute_value, - self.geographic_mapping_attributes[attribute_name]) + self.assertEqual( + attribute_value, self.geographic_mapping_attributes[attribute_name] + ) # Check dimension variables self.assertTupleEqual(saved_output['lat'].dimensions, ('lat',)) @@ -106,84 +112,78 @@ def test_write_single_band_output(self): np.testing.assert_array_equal(saved_output['lon'][:], self.lon_values) def test_write_dimensions(self): - """ Ensure dimensions are written with the correct names. The subtests - should establish whether geographic projections are identified, - whether pre-existing information in the cache is used and, if - needed, whether suffices are added to the dimension names. + """Ensure dimensions are written with the correct names. The subtests + should establish whether geographic projections are identified, + whether pre-existing information in the cache is used and, if + needed, whether suffices are added to the dimension names. """ with self.subTest('Geographic, Harmony defined area.'): cache = {HARMONY_TARGET: {'reprojection': 'information'}} with Dataset('test.nc', 'w', diskless=True) as dataset: - dimensions = write_dimensions(dataset, self.area_definition, - cache) + dimensions = write_dimensions(dataset, self.area_definition, cache) self.assertTupleEqual(dimensions, ('lat', 'lon')) - self.assertSetEqual(set(dataset.dimensions.keys()), - {'lat', 'lon'}) + self.assertSetEqual(set(dataset.dimensions.keys()), {'lat', 'lon'}) with self.subTest('Non-geographic, Harmony defined area.'): cache = {HARMONY_TARGET: {'reprojection': 'information'}} with Dataset('test.nc', 'w', diskless=True) as dataset: - dimensions = write_dimensions(dataset, - self.non_geographic_area, cache) + dimensions = write_dimensions(dataset, self.non_geographic_area, cache) self.assertTupleEqual(dimensions, ('y', 'x')) - self.assertSetEqual(set(dataset.dimensions.keys()), - {'y', 'x'}) + self.assertSetEqual(set(dataset.dimensions.keys()), {'y', 'x'}) with self.subTest('Geographic, retrieve dimensions from cache.'): cache = {('lat', 'lon'): {'dimensions': ('saved_lat', 'saved_lon')}} with Dataset('test.nc', 'w', diskless=True) as dataset: - dimensions = write_dimensions(dataset, self.area_definition, - cache) + dimensions = write_dimensions(dataset, self.area_definition, cache) self.assertTupleEqual(dimensions, ('saved_lat', 'saved_lon')) - self.assertSetEqual(set(dataset.dimensions.keys()), - {'saved_lat', 'saved_lon'}) + self.assertSetEqual( + set(dataset.dimensions.keys()), {'saved_lat', 'saved_lon'} + ) with self.subTest('Geographic, no Harmony, but no saved dimensions.'): cache = {('lat', 'lon'): {'reprojection': 'information'}} with Dataset('test.nc', 'w', diskless=True) as dataset: - dimensions = write_dimensions(dataset, self.area_definition, - cache) + dimensions = write_dimensions(dataset, self.area_definition, cache) self.assertTupleEqual(dimensions, ('lat', 'lon')) - self.assertSetEqual(set(dataset.dimensions.keys()), - {'lat', 'lon'}) + self.assertSetEqual(set(dataset.dimensions.keys()), {'lat', 'lon'}) with self.subTest('Geographic, no Harmony, multiple target grids.'): cache = { ('first_lat', 'first_lon'): {'dimensions': ('lat', 'lon')}, ('second_lat', 'second_lon'): {'dimensions': ('lat_1', 'lon_1')}, - ('lat', 'lon'): {}} + ('lat', 'lon'): {}, + } with Dataset('test.nc', 'w', diskless=True) as dataset: - dimensions = write_dimensions(dataset, self.area_definition, - cache) + dimensions = write_dimensions(dataset, self.area_definition, cache) self.assertTupleEqual(dimensions, ('lat_2', 'lon_2')) - self.assertSetEqual(set(dataset.dimensions.keys()), - {'lat_2', 'lon_2'}) + self.assertSetEqual(set(dataset.dimensions.keys()), {'lat_2', 'lon_2'}) def test_write_grid_mapping(self): - """ Check that the grid mapping attributes from the target area are - saved to the metadata of an appropriately named variable. + """Check that the grid mapping attributes from the target area are + saved to the metadata of an appropriately named variable. - The name of the variable should be returned. If the grid mapping is - non-standard, and does not include a name, "crs" should be used. - If the dimensions associated with the mapping do not conform to - either ('lat', 'lon') or ('y', 'x'), then the extended form of the - naming schema should be used. + The name of the variable should be returned. If the grid mapping is + non-standard, and does not include a name, "crs" should be used. + If the dimensions associated with the mapping do not conform to + either ('lat', 'lon') or ('y', 'x'), then the extended form of the + naming schema should be used. """ with self.subTest('Geographic mapping, regular names.'): with Dataset('test.nc', 'w', diskless=True) as dataset: - grid_mapping_name = write_grid_mapping(dataset, self.area_definition, - ('lat', 'lon')) + grid_mapping_name = write_grid_mapping( + dataset, self.area_definition, ('lat', 'lon') + ) self.assertEqual(grid_mapping_name, 'latitude_longitude') self.assertIn('latitude_longitude', dataset.variables) @@ -191,32 +191,31 @@ def test_write_grid_mapping(self): for attribute_name, attribute_value in grid_attributes.items(): self.assertEqual( attribute_value, - self.geographic_mapping_attributes[attribute_name] + self.geographic_mapping_attributes[attribute_name], ) with self.subTest('Non-geographic mapping, regular names.'): with Dataset('test.nc', 'w', diskless=True) as dataset: - grid_mapping_name = write_grid_mapping(dataset, - self.non_geographic_area, - ('y', 'x')) + grid_mapping_name = write_grid_mapping( + dataset, self.non_geographic_area, ('y', 'x') + ) - self.assertEqual(grid_mapping_name, - 'lambert_cylindrical_equal_area') + self.assertEqual(grid_mapping_name, 'lambert_cylindrical_equal_area') self.assertIn('lambert_cylindrical_equal_area', dataset.variables) grid_attributes = dataset['lambert_cylindrical_equal_area'].__dict__ - self.assertIn(grid_attributes['grid_mapping_name'], - 'lambert_cylindrical_equal_area') + self.assertIn( + grid_attributes['grid_mapping_name'], + 'lambert_cylindrical_equal_area', + ) with self.subTest('Geographic, extended mapping name.'): with Dataset('test.nc', 'w', diskless=True) as dataset: - grid_mapping_name = write_grid_mapping(dataset, - self.area_definition, - ('lat_1', 'lon_1')) + grid_mapping_name = write_grid_mapping( + dataset, self.area_definition, ('lat_1', 'lon_1') + ) - self.assertEqual(grid_mapping_name, - 'latitude_longitude_lat_1_lon_1') - self.assertIn('latitude_longitude_lat_1_lon_1', - dataset.variables) + self.assertEqual(grid_mapping_name, 'latitude_longitude_lat_1_lon_1') + self.assertIn('latitude_longitude_lat_1_lon_1', dataset.variables) with self.subTest('A custom CRS, with no name specified.'): crs = Mock(spec=CRS) @@ -225,21 +224,24 @@ def test_write_grid_mapping(self): pixel_size_x = 0.1 pixel_size_y = 0.1 - target_area = Mock(spec=AreaDefinition, area_extent=area_extent, - crs=crs, pixel_size_x=pixel_size_x, - pixel_size_y=pixel_size_y) + target_area = Mock( + spec=AreaDefinition, + area_extent=area_extent, + crs=crs, + pixel_size_x=pixel_size_x, + pixel_size_y=pixel_size_y, + ) with Dataset('test.nc', 'w', diskless=True) as dataset: - grid_mapping_name = write_grid_mapping(dataset, target_area, - ('y', 'x')) + grid_mapping_name = write_grid_mapping(dataset, target_area, ('y', 'x')) self.assertEqual(grid_mapping_name, 'crs') self.assertIn('crs', dataset.variables) def test_write_science_variable(self): - """ Ensure that the values, dimensions, datatype and attributes are all - correctly set of a science variable. This should also include the - grid mapping name. + """Ensure that the values, dimensions, datatype and attributes are all + correctly set of a science variable. This should also include the + grid mapping name. """ attributes = {'add_offset': 10, 'scale_factor': 0.1} @@ -247,13 +249,20 @@ def test_write_science_variable(self): dataset.createDimension('lat', size=2) dataset.createDimension('lon', size=4) - write_science_variable(dataset, self.reprojected_data, - 'science_name', ('lat', 'lon'), - 'mapping_name', attributes) + write_science_variable( + dataset, + self.reprojected_data, + 'science_name', + ('lat', 'lon'), + 'mapping_name', + attributes, + ) - expected_attributes = {'add_offset': 10, - 'grid_mapping': 'mapping_name', - 'scale_factor': 0.1} + expected_attributes = { + 'add_offset': 10, + 'grid_mapping': 'mapping_name', + 'scale_factor': 0.1, + } # The science variable exists. self.assertIn('science_name', dataset.variables) @@ -262,29 +271,27 @@ def test_write_science_variable(self): self.assertEqual(dataset['science_name'].datatype, np.int64) # The science variable has the expected dimensions. - self.assertTupleEqual(dataset['science_name'].dimensions, - ('lat', 'lon')) + self.assertTupleEqual(dataset['science_name'].dimensions, ('lat', 'lon')) # The science variable array contains the correct values. - np.testing.assert_array_equal(dataset['science_name'][:], - self.reprojected_data) + np.testing.assert_array_equal( + dataset['science_name'][:], self.reprojected_data + ) # The science variable metadata attributes are correct. - self.assertDictEqual(dataset['science_name'].__dict__, - expected_attributes) + self.assertDictEqual(dataset['science_name'].__dict__, expected_attributes) def test_write_dimension_variables(self): - """ Ensure that dimension variables that have suffices are successfully - saved to a `netCDF4.Dataset`, and still include the expected - metadata attributes. + """Ensure that dimension variables that have suffices are successfully + saved to a `netCDF4.Dataset`, and still include the expected + metadata attributes. """ with Dataset('test.nc', 'w', diskless=True) as dataset: dataset.createDimension('lat_1', size=2) dataset.createDimension('lon_1', size=4) - write_dimension_variables(dataset, ('lat_1', 'lon_1'), - self.area_definition) + write_dimension_variables(dataset, ('lat_1', 'lon_1'), self.area_definition) # The dimension variables exist. self.assertIn('lat_1', dataset.variables) @@ -297,15 +304,19 @@ def test_write_dimension_variables(self): # The expected attributes are present. self.assertDictEqual( dataset['lat_1'].__dict__, - {'long_name': 'latitude', - 'standard_name': 'latitude', - 'units': 'degrees_north'} + { + 'long_name': 'latitude', + 'standard_name': 'latitude', + 'units': 'degrees_north', + }, ) self.assertDictEqual( dataset['lon_1'].__dict__, - {'long_name': 'longitude', - 'standard_name': 'longitude', - 'units': 'degrees_east'} + { + 'long_name': 'longitude', + 'standard_name': 'longitude', + 'units': 'degrees_east', + }, ) # The data values are correct. diff --git a/tests/unit/test_reproject.py b/tests/unit/test_reproject.py index 352e136..e3b566f 100644 --- a/tests/unit/test_reproject.py +++ b/tests/unit/test_reproject.py @@ -4,18 +4,14 @@ from harmony.message import Message from pyproj import Proj -from swath_projector.reproject import ( - CRS_DEFAULT, - get_parameters_from_message, - rgetattr -) +from swath_projector.reproject import CRS_DEFAULT, get_parameters_from_message, rgetattr class TestReproject(TestCase): @classmethod def setUpClass(cls): - """ Class properties that only need to be set once. """ + """Class properties that only need to be set once.""" cls.logger = Logger('Reproject test') cls.granule = 'tests/data/africa.nc' cls.granule_url = 'https://example.com/africa.nc' @@ -29,7 +25,7 @@ def setUpClass(cls): cls.y_res = -1.0 def setUp(self): - """ Define properties that should be refreshed on each test. """ + """Define properties that should be refreshed on each test.""" self.default_parameters = { 'crs': CRS_DEFAULT, 'granule_url': self.granule_url, @@ -43,61 +39,66 @@ def setUp(self): 'xres': None, 'y_min': None, 'y_max': None, - 'yres': None + 'yres': None, } def assert_parameters_equal(self, parameters, expected_parameters): - """ A helper method to check that the parameters retrieved from the - input Harmony message are all as expected. Note, this does not - compare the file_data parameter, which probably should not be part - of the output, (the individual parameters within this dictionary - are transferred to the top level of the parameters). There is a - note in the code for clean-up to occur! + """A helper method to check that the parameters retrieved from the + input Harmony message are all as expected. Note, this does not + compare the file_data parameter, which probably should not be part + of the output, (the individual parameters within this dictionary + are transferred to the top level of the parameters). There is a + note in the code for clean-up to occur! """ for key, expected_value in expected_parameters.items(): - self.assertEqual(parameters[key], expected_value, - f'Failing parameter: {key}') + self.assertEqual( + parameters[key], expected_value, f'Failing parameter: {key}' + ) def test_get_parameters_from_message_interpolation(self): - """ Ensure that various input messages can be correctly parsed, and - that those missing raise the expected exceptions. + """Ensure that various input messages can be correctly parsed, and + that those missing raise the expected exceptions. """ - test_args = [['No interpolation', {}, self.default_interpolation], - ['Non default', {'interpolation': 'ewa'}, 'ewa'], - ['None interpolation', {'interpolation': None}, 'ewa-nn'], - ['String None', {'interpolation': 'None'}, 'ewa-nn'], - ['Empty string', {'interpolation': ''}, 'ewa-nn']] - + test_args = [ + ['No interpolation', {}, self.default_interpolation], + ['Non default', {'interpolation': 'ewa'}, 'ewa'], + ['None interpolation', {'interpolation': None}, 'ewa-nn'], + ['String None', {'interpolation': 'None'}, 'ewa-nn'], + ['Empty string', {'interpolation': ''}, 'ewa-nn'], + ] for description, format_attribute, expected_interpolation in test_args: with self.subTest(description): - message_content = {'format': format_attribute, - 'granules': self.granules} + message_content = { + 'format': format_attribute, + 'granules': self.granules, + } message = Message(message_content) - parameters = get_parameters_from_message(message, - self.granule_url, - self.granule) + parameters = get_parameters_from_message( + message, self.granule_url, self.granule + ) - self.assertEqual(parameters['interpolation'], - expected_interpolation) + self.assertEqual(parameters['interpolation'], expected_interpolation) def test_get_parameters_error_5(self): - """ Ensure that, if parameters are set for the resolution and the - dimensions, an exception is raised. + """Ensure that, if parameters are set for the resolution and the + dimensions, an exception is raised. """ exception_snippet = 'cannot be used at the same time in the message.' - test_args = [['height and scaleSize', True, False, True, True], - ['width and scaleSize', False, True, True, True], - ['x_res and dimensions', True, True, True, False], - ['y_res and dimensions', True, True, False, True], - ['x_res and width', False, True, True, False], - ['y_res and height', True, False, False, True], - ['x_res and height', True, False, True, False], - ['y_res and width', False, True, False, True]] + test_args = [ + ['height and scaleSize', True, False, True, True], + ['width and scaleSize', False, True, True, True], + ['x_res and dimensions', True, True, True, False], + ['y_res and dimensions', True, True, False, True], + ['x_res and width', False, True, True, False], + ['y_res and height', True, False, False, True], + ['x_res and height', True, False, True, False], + ['y_res and width', False, True, False, True], + ] for description, has_height, has_width, has_x_res, has_y_res in test_args: with self.subTest(description): @@ -119,52 +120,50 @@ def test_get_parameters_error_5(self): message = Message(message_content) with self.assertRaises(Exception) as context: - get_parameters_from_message(message, self.granule_url, - self.granule) + get_parameters_from_message(message, self.granule_url, self.granule) self.assertTrue(str(context.exception).endswith(exception_snippet)) def test_get_parameters_missing_extents_or_dimensions(self): - """ Ensure that an exception is raised if there is only one of either - x_extent and y_extent or height and width set. + """Ensure that an exception is raised if there is only one of either + x_extent and y_extent or height and width set. """ test_args = [ ['height not width', {'height': self.height}], ['width not height', {'width': self.width}], ['x_extent not y_extent', {'scaleExtent': {'x': self.x_extent}}], - ['y_extent not x_extent', {'scaleExtent': {'y': self.y_extent}}] + ['y_extent not x_extent', {'scaleExtent': {'y': self.y_extent}}], ] for description, format_content in test_args: with self.subTest(description): - message_content = {'granules': self.granules, - 'format': format_content} + message_content = {'granules': self.granules, 'format': format_content} message = Message(message_content) with self.assertRaises(Exception) as context: - get_parameters_from_message(message, self.granule_url, - self.granule) + get_parameters_from_message(message, self.granule_url, self.granule) self.assertTrue('Missing' in str(context.exception)) def test_get_parameters_from_message_defaults(self): - """ Ensure that if the most minimal Harmony message is supplied to the - SWOT Reprojection tool, sensible defaults are assigned for the - extracted message parameters. + """Ensure that if the most minimal Harmony message is supplied to the + SWOT Reprojection tool, sensible defaults are assigned for the + extracted message parameters. """ expected_parameters = self.default_parameters message = Message({'granules': self.granules, 'format': {}}) - parameters = get_parameters_from_message(message, self.granule_url, - self.granule) + parameters = get_parameters_from_message( + message, self.granule_url, self.granule + ) self.assert_parameters_equal(parameters, expected_parameters) def test_get_parameters_from_message_extents(self): - """ Ensure that if the `scaleExtent` is specified in the input - Harmony message, the non-default extents are used. + """Ensure that if the `scaleExtent` is specified in the input + Harmony message, the non-default extents are used. """ expected_parameters = self.default_parameters @@ -173,20 +172,19 @@ def test_get_parameters_from_message_extents(self): expected_parameters['y_min'] = self.y_extent['min'] expected_parameters['y_max'] = self.y_extent['max'] - extents_format = {'scaleExtent': {'x': self.x_extent, - 'y': self.y_extent}} - message = Message({'granules': self.granules, - 'format': extents_format}) + extents_format = {'scaleExtent': {'x': self.x_extent, 'y': self.y_extent}} + message = Message({'granules': self.granules, 'format': extents_format}) expected_parameters['x_extent'] = message.format.scaleExtent.x expected_parameters['y_extent'] = message.format.scaleExtent.y - parameters = get_parameters_from_message(message, self.granule_url, - self.granule) + parameters = get_parameters_from_message( + message, self.granule_url, self.granule + ) self.assert_parameters_equal(parameters, expected_parameters) def test_get_parameters_from_message_resolutions(self): - """ Ensure that if the `scaleSize` is specified in the input Harmony - message, the non-default resolutions are used. + """Ensure that if the `scaleSize` is specified in the input Harmony + message, the non-default resolutions are used. """ expected_parameters = self.default_parameters @@ -194,15 +192,15 @@ def test_get_parameters_from_message_resolutions(self): expected_parameters['yres'] = self.y_res resolutions_format = {'scaleSize': {'x': self.x_res, 'y': self.y_res}} - message = Message({'granules': self.granules, - 'format': resolutions_format}) - parameters = get_parameters_from_message(message, self.granule_url, - self.granule) + message = Message({'granules': self.granules, 'format': resolutions_format}) + parameters = get_parameters_from_message( + message, self.granule_url, self.granule + ) self.assert_parameters_equal(parameters, expected_parameters) def test_get_parameters_from_message_dimensions(self): - """ Ensure that if the `height` and `width` are specified in the input - Harmony message, the non-default dimensions are used. + """Ensure that if the `height` and `width` are specified in the input + Harmony message, the non-default dimensions are used. """ expected_parameters = self.default_parameters @@ -210,17 +208,18 @@ def test_get_parameters_from_message_dimensions(self): expected_parameters['width'] = self.height extents_format = {'height': self.height, 'width': self.width} - message = Message({'granules': self.granules, - 'format': extents_format}) - parameters = get_parameters_from_message(message, self.granule_url, - self.granule) + message = Message({'granules': self.granules, 'format': extents_format}) + parameters = get_parameters_from_message( + message, self.granule_url, self.granule + ) self.assert_parameters_equal(parameters, expected_parameters) def test_rgetattr(self): - """ Ensure the utility function to recursively retrieve a class - attribute will work as expected. + """Ensure the utility function to recursively retrieve a class + attribute will work as expected. """ + class ExampleInnerClass: def __init__(self): self.interpolation = 'bilinear' @@ -234,17 +233,22 @@ def __init__(self): example_object = ExampleOuterClass() default = 'default' - test_args = [['Single depth property', 'user', 'jglenn'], - ['Nested property', 'inner.interpolation', 'bilinear'], - ['Property is None, default', 'none', default], - ['Absent attribute uses default', 'absent', default], - ['Absent nested attribute uses default', 'inner.absent', default], - ['Absent outer for nested uses default', 'absent.interpolation', default], - ['Outer present, but not object, uses default', 'user.interpolation', default]] + test_args = [ + ['Single depth property', 'user', 'jglenn'], + ['Nested property', 'inner.interpolation', 'bilinear'], + ['Property is None, default', 'none', default], + ['Absent attribute uses default', 'absent', default], + ['Absent nested attribute uses default', 'inner.absent', default], + ['Absent outer for nested uses default', 'absent.interpolation', default], + [ + 'Outer present, but not object, uses default', + 'user.interpolation', + default, + ], + ] for description, attribute_path, expected_value in test_args: with self.subTest(description): self.assertEqual( - rgetattr(example_object, attribute_path, default), - expected_value + rgetattr(example_object, attribute_path, default), expected_value ) diff --git a/tests/unit/test_swath_geometry.py b/tests/unit/test_swath_geometry.py index 46a76a0..30325bf 100644 --- a/tests/unit/test_swath_geometry.py +++ b/tests/unit/test_swath_geometry.py @@ -21,7 +21,7 @@ get_valid_coordinates_mask, reproject_coordinates, sort_perimeter_points, - swath_crosses_international_date_line + swath_crosses_international_date_line, ) @@ -35,12 +35,20 @@ def setUpClass(cls): cls.test_dir = mkdtemp() cls.test_path = f'{cls.test_dir}/geometry.nc' - cls.lat_data = np.array([[25.0, 25.0, 25.0, 25.0], - [20.0, 20.0, 20.0, 20.0], - [15.0, 15.0, 15.0, 15.0]]) - cls.lon_data = np.array([[40.0, 45.0, 50.0, 55.0], - [40.0, 45.0, 50.0, 55.0], - [40.0, 45.0, 50.0, 55.0]]) + cls.lat_data = np.array( + [ + [25.0, 25.0, 25.0, 25.0], + [20.0, 20.0, 20.0, 20.0], + [15.0, 15.0, 15.0, 15.0], + ] + ) + cls.lon_data = np.array( + [ + [40.0, 45.0, 50.0, 55.0], + [40.0, 45.0, 50.0, 55.0], + [40.0, 45.0, 50.0, 55.0], + ] + ) cls.test_file = Dataset(cls.test_path, 'w') cls.test_file.createDimension('nj', size=3) @@ -68,36 +76,40 @@ def tearDownClass(cls): rmtree(cls.test_dir, ignore_errors=True) def test_euclidean_distance(self): - """ Ensure the Euclidean distance is correctly calculated. """ + """Ensure the Euclidean distance is correctly calculated.""" self.assertEqual(euclidean_distance(2.3, 5.3, 6.8, 2.8), 5.0) def test_get_projected_resolution(self): - """ Ensure the calculated resolution from the input longitudes and - latitudes is as expected. Resolution is large for metres, because - grid is in 5 degree increments. + """Ensure the calculated resolution from the input longitudes and + latitudes is as expected. Resolution is large for metres, because + grid is in 5 degree increments. """ - test_args = [['Geographic', self.geographic_projection, 3.536], - ['Projected metres', self.ease_projection, 380302.401]] + test_args = [ + ['Geographic', self.geographic_projection, 3.536], + ['Projected metres', self.ease_projection, 380302.401], + ] for description, projection, expected_resolution in test_args: with self.subTest(description): - resolution = get_projected_resolution(projection, - self.longitudes, - self.latitudes) + resolution = get_projected_resolution( + projection, self.longitudes, self.latitudes + ) self.assertAlmostEqual(resolution, expected_resolution, places=3) def test_get_projected_resolution_1d(self): - """ Ensure the calculated one-dimensional resolution is correct. """ - resolution = get_projected_resolution(self.geographic_projection, - self.test_dataset['lon_1d'], - self.test_dataset['lat_1d']) + """Ensure the calculated one-dimensional resolution is correct.""" + resolution = get_projected_resolution( + self.geographic_projection, + self.test_dataset['lon_1d'], + self.test_dataset['lat_1d'], + ) self.assertAlmostEqual(resolution, 5.0) def test_get_extents_from_perimeter(self): - """ Get the maximum and minimum values from the perimeter data - points. + """Get the maximum and minimum values from the perimeter data + points. """ with self.subTest('Geographic coordinates'): @@ -121,8 +133,9 @@ def test_get_extents_from_perimeter(self): with self.subTest('Geographic, 1-D'): x_min, x_max, y_min, y_max = get_extents_from_perimeter( - self.geographic_projection, self.test_dataset['lon_1d'], - self.test_dataset['lat_1d'] + self.geographic_projection, + self.test_dataset['lon_1d'], + self.test_dataset['lat_1d'], ) self.assertAlmostEqual(x_min, 2.0, places=7) self.assertAlmostEqual(x_max, 14.0, places=7) @@ -130,33 +143,38 @@ def test_get_extents_from_perimeter(self): self.assertAlmostEqual(y_max, 9.0, places=7) def test_get_perimeter_coordinates(self): - """ Ensure a full list of longitude, latitude points are returned for - a given coordinate mask. These points will be unordered. + """Ensure a full list of longitude, latitude points are returned for + a given coordinate mask. These points will be unordered. """ - valid_pixels = [[False, True, True, False], - [True, True, True, True], - [True, True, False, False]] + valid_pixels = [ + [False, True, True, False], + [True, True, True, True], + [True, True, False, False], + ] - expected_points = [(self.longitudes[0][1], self.latitudes[0][1]), - (self.longitudes[0][2], self.latitudes[0][2]), - (self.longitudes[1][0], self.latitudes[1][0]), - (self.longitudes[1][2], self.latitudes[1][2]), - (self.longitudes[1][3], self.latitudes[1][3]), - (self.longitudes[2][0], self.latitudes[2][0]), - (self.longitudes[2][1], self.latitudes[2][1])] + expected_points = [ + (self.longitudes[0][1], self.latitudes[0][1]), + (self.longitudes[0][2], self.latitudes[0][2]), + (self.longitudes[1][0], self.latitudes[1][0]), + (self.longitudes[1][2], self.latitudes[1][2]), + (self.longitudes[1][3], self.latitudes[1][3]), + (self.longitudes[2][0], self.latitudes[2][0]), + (self.longitudes[2][1], self.latitudes[2][1]), + ] - mask = np.ma.masked_where(np.logical_not(valid_pixels), - np.ones(self.longitudes.shape)) + mask = np.ma.masked_where( + np.logical_not(valid_pixels), np.ones(self.longitudes.shape) + ) - coordinates = get_perimeter_coordinates(self.longitudes[:], - self.latitudes[:], - mask) + coordinates = get_perimeter_coordinates( + self.longitudes[:], self.latitudes[:], mask + ) self.assertCountEqual(coordinates, expected_points) def test_reproject_coordinates(self): - """ Ensure a set of points will be correctly projected. """ + """Ensure a set of points will be correctly projected.""" proj = Proj('EPSG:32603') input_points = [(10.0, 2.5), (15.0, 3.0), (20.0, 3.5)] expected_x = np.array([1056557.724, 500000.000, -56049.659]) @@ -169,7 +187,7 @@ def test_reproject_coordinates(self): np.testing.assert_allclose(y_values, expected_y, atol=0.001, rtol=0) def test_get_polygon_area(self): - """ Ensure area is correctly calculated for some known shapes. """ + """Ensure area is correctly calculated for some known shapes.""" triangle_points_x = [1.0, 3.0, 1.0] triangle_points_y = [1.0, 1.0, 3.0] triangle_area = get_polygon_area(triangle_points_x, triangle_points_y) @@ -181,7 +199,7 @@ def test_get_polygon_area(self): self.assertEqual(square_area, 4.0) def test_get_absolute_resolution(self): - """ Ensure the expected resolution value is returned. """ + """Ensure the expected resolution value is returned.""" area = 16.0 n_pixels = 4 resolution = get_absolute_resolution(area, n_pixels) @@ -189,8 +207,8 @@ def test_get_absolute_resolution(self): self.assertEqual(resolution, 2.0) def test_get_one_dimensional_resolution(self): - """ Ensure the 1-D resolution is calculated as expected from the input - data. + """Ensure the 1-D resolution is calculated as expected from the input + data. """ x_values = list(self.test_dataset['lon_1d'][:]) @@ -199,7 +217,7 @@ def test_get_one_dimensional_resolution(self): self.assertAlmostEqual(resolution, 5.0) def test_swath_crosses_international_date_line(self): - """ Ensure the International Date Line is correctly identified. """ + """Ensure the International Date Line is correctly identified.""" not_crossing_lon = np.array([[10, 20, 30], [10, 20, 30]]) crossing_lon = np.array([[165, 175, -175], [165, 175, -175]]) crossing_vertical = np.array([[101.0, 101.0], [10.0, 10.0]]) @@ -217,32 +235,40 @@ def test_swath_crosses_international_date_line(self): self.assertTrue(crosses) def test_clockwise_point_sort(self): - """ Ensure the correct lengths and angles are calculated. """ + """Ensure the correct lengths and angles are calculated.""" test_args = [ ['Point is at origin', [0, 0], [0, 0], (-np.pi, 0)], ['Point is in vertical direction', [0, 0], [0, 30], (0.0, 30)], - ['Point at 45 degrees', [0, 0], [3, 3], (np.pi / 4.0, np.sqrt(18.0))] + ['Point at 45 degrees', [0, 0], [3, 3], (np.pi / 4.0, np.sqrt(18.0))], ] for description, origin, point, expected_results in test_args: with self.subTest(description): - self.assertEqual(clockwise_point_sort(origin, point), - expected_results) + self.assertEqual(clockwise_point_sort(origin, point), expected_results) def test_sort_perimeter_points(self): - """ Ensure unsorted x and y coordinates are returned in order. - The points in the `square_points` and `polygon_points` lists are - ordered to be the expected output. + """Ensure unsorted x and y coordinates are returned in order. + The points in the `square_points` and `polygon_points` lists are + ordered to be the expected output. """ - square_points = [[0, 0], [0, 1], [0, 2], [1, 2], [2, 2], [2, 1], - [2, 0], [1, 0]] - polygon_points = [[20, 10], [10, 30], [20, 40], [10, 50], [20, 60], - [30, 60], [40, 50], [50, 60], [50, 40], [60, 40], - [50, 30], [60, 10]] + square_points = [[0, 0], [0, 1], [0, 2], [1, 2], [2, 2], [2, 1], [2, 0], [1, 0]] + polygon_points = [ + [20, 10], + [10, 30], + [20, 40], + [10, 50], + [20, 60], + [30, 60], + [40, 50], + [50, 60], + [50, 40], + [60, 40], + [50, 30], + [60, 10], + ] - test_args = [['Simple square', square_points], - ['Polygon', polygon_points]] + test_args = [['Simple square', square_points], ['Polygon', polygon_points]] for description, ordered_points in test_args: with self.subTest(description): @@ -252,13 +278,12 @@ def test_sort_perimeter_points(self): shuffle(disordered_points) disordered_x, disordered_y = zip(*disordered_points) - ordered_x, ordered_y = sort_perimeter_points(disordered_x, - disordered_y) + ordered_x, ordered_y = sort_perimeter_points(disordered_x, disordered_y) self.assertEqual(ordered_x, expected_x) self.assertEqual(ordered_y, expected_y) def test_get_valid_coordinates_mask(self): - """ Ensure all logical conditions are respected. """ + """Ensure all logical conditions are respected.""" fill_value = -9999.0 valid_lon = np.array([[1.0, 2.0], [3.0, 4.0]]) @@ -279,7 +304,7 @@ def test_get_valid_coordinates_mask(self): ['Longitude fill', fill_lon, valid_lat, [[1, 1], [0, 1]]], ['Latitude NaN', valid_lon, nan_lat, [[1, 0], [1, 1]]], ['Latitude fill', valid_lon, fill_lat, [[1, 1], [1, 0]]], - ['Combination', combined_lon, combined_lat, [[0, 0], [0, 0]]] + ['Combination', combined_lon, combined_lat, [[0, 0], [0, 0]]], ] for description, lon_data, lat_data, expected_mask in test_args: @@ -288,10 +313,12 @@ def test_get_valid_coordinates_mask(self): test_file = Dataset(test_path, 'w') test_file.createDimension('nj', size=2) test_file.createDimension('ni', size=2) - test_file.createVariable('lat', float, dimensions=('nj', 'ni'), - fill_value=fill_value) - test_file.createVariable('lon', float, dimensions=('nj', 'ni'), - fill_value=fill_value) + test_file.createVariable( + 'lat', float, dimensions=('nj', 'ni'), fill_value=fill_value + ) + test_file.createVariable( + 'lon', float, dimensions=('nj', 'ni'), fill_value=fill_value + ) test_file['lat'][:] = lat_data test_file['lon'][:] = lon_data test_file.close() @@ -299,16 +326,14 @@ def test_get_valid_coordinates_mask(self): dataset = Dataset(test_path) np.testing.assert_array_equal( get_valid_coordinates_mask(dataset['lon'], dataset['lat']), - expected_mask + expected_mask, ) dataset.close() - - def test_get_slice_edges(self): - """ Ensure the pixel coordinates for exterior points are returned, - this should order the elements based on whether the input slice - is a row or a column. + """Ensure the pixel coordinates for exterior points are returned, + this should order the elements based on whether the input slice + is a row or a column. """ data_slice = np.array([2, 3, 4, 5, 6, 7, 8, 9]) @@ -317,16 +342,19 @@ def test_get_slice_edges(self): expected_row_results = [(6, 2), (6, 9)] expected_column_results = [(2, 6), (9, 6)] - test_args = [['Row', True, expected_row_results], - ['Column', False, expected_column_results]] + test_args = [ + ['Row', True, expected_row_results], + ['Column', False, expected_column_results], + ] for description, is_row, expected_results in test_args: with self.subTest(description): self.assertEqual( get_slice_edges(data_slice, slice_index, is_row=is_row), - expected_results + expected_results, ) with self.subTest('Default (to row)'): - self.assertEqual(get_slice_edges(data_slice, slice_index), - expected_row_results) + self.assertEqual( + get_slice_edges(data_slice, slice_index), expected_row_results + ) diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 9a53011..a8b264b 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -1,4 +1,3 @@ -from logging import getLogger from unittest import TestCase from unittest.mock import Mock @@ -17,25 +16,24 @@ get_variable_numeric_fill_value, make_array_two_dimensional, qualify_reference, - variable_in_dataset + variable_in_dataset, ) class TestUtilities(TestCase): def test_create_coordinates_key(self): - """ Extract the coordinates from a `VariableFromNetCDF4` instance and - return an alphabetically sorted tuple. The ordering prevents any - shuffling due to earthdata-varinfo storing CF-Convention attribute - references as a Python Set. + """Extract the coordinates from a `VariableFromNetCDF4` instance and + return an alphabetically sorted tuple. The ordering prevents any + shuffling due to earthdata-varinfo storing CF-Convention attribute + references as a Python Set. """ data = np.ones((2, 4)) dimensions = ('lat', 'lon') expected_output = ('/lat', '/lon') - test_args = [['comma-space', ['/lon, /lat']], - ['reverse order', ['/lat, /lon']]] + test_args = [['comma-space', ['/lon, /lat']], ['reverse order', ['/lat, /lon']]] for description, coordinates in test_args: with self.subTest(description): @@ -46,21 +44,20 @@ def test_create_coordinates_key(self): nc4_variable = dataset.createVariable( '/group/variable', data.dtype, dimensions=dimensions ) - dataset.createVariable('/lat', data.dtype, - dimensions=dimensions) - dataset.createVariable('/lon', data.dtype, - dimensions=dimensions) + dataset.createVariable('/lat', data.dtype, dimensions=dimensions) + dataset.createVariable('/lon', data.dtype, dimensions=dimensions) nc4_variable.setncattr('coordinates', coordinates) varinfo = VarInfoFromNetCDF4('test.nc') varinfo_variable = varinfo.get_variable('/group/variable') - self.assertEqual(create_coordinates_key(varinfo_variable), - expected_output) + self.assertEqual( + create_coordinates_key(varinfo_variable), expected_output + ) def test_get_variable_values(self): - """ Ensure values for a variable are retrieved, respecting the absence - or presence of a time variable in the dataset. + """Ensure values for a variable are retrieved, respecting the absence + or presence of a time variable in the dataset. """ @@ -91,15 +88,17 @@ def test_get_variable_values(self): with Dataset('mock_data.nc', 'w', diskless=True) as dataset: dataset.createDimension('y', size=2) dataset.createDimension('x', size=2) - dataset.createVariable('data', np.uint8, dimensions=('y', 'x'), - fill_value=fill_value) + dataset.createVariable( + 'data', np.uint8, dimensions=('y', 'x'), fill_value=fill_value + ) dataset['data'][:] = input_data # Ensure the raw variable data is masked in the expected cell. self.assertTrue(dataset['data'][:].mask[0, 1]) - returned_data = get_variable_values(dataset, dataset['data'], - fill_value) + returned_data = get_variable_values( + dataset, dataset['data'], fill_value + ) # Check the output is an array, not a masked array. self.assertIsInstance(returned_data, np.ndarray) @@ -112,8 +111,9 @@ def test_get_variable_values(self): dataset.createDimension('lat', size=2) dataset.createDimension('lon', size=2) input_data = np.array([[1, 2], [3, 4]]) - variable = dataset.createVariable('data', input_data.dtype, - dimensions=('lat', 'lon')) + variable = dataset.createVariable( + 'data', input_data.dtype, dimensions=('lat', 'lon') + ) variable[:] = input_data[:] returned_data = get_variable_values(dataset, variable, None) @@ -122,8 +122,8 @@ def test_get_variable_values(self): np.testing.assert_array_equal(input_data, returned_data) def test_get_coordinate_variables(self): - """ Ensure the longitude or latitude coordinate variable, is retrieved - when requested. + """Ensure the longitude or latitude coordinate variable, is retrieved + when requested. """ dataset = Dataset('tests/data/africa.nc') @@ -131,62 +131,65 @@ def test_get_coordinate_variables(self): for coordinate in coordinates_tuple: with self.subTest(coordinate): - coordinates = get_coordinate_variable(dataset, - coordinates_tuple, - coordinate) + coordinates = get_coordinate_variable( + dataset, coordinates_tuple, coordinate + ) self.assertIsInstance(coordinates, Variable) - with self.subTest('Non existent coordinate variable "latitude" returns MissingCoordinatesError'): + with self.subTest( + 'Non existent coordinate variable "latitude" returns MissingCoordinatesError' + ): absent_coordinates_tuple = ['latitude'] with self.assertRaises(MissingCoordinatesError): coordinates = get_coordinate_variable( - dataset, - absent_coordinates_tuple, - absent_coordinates_tuple[0] + dataset, absent_coordinates_tuple, absent_coordinates_tuple[0] ) def test_get_variable_numeric_fill_value(self): - """ Ensure a fill value is retrieved from a variable that has a vaild - numeric value, and is cast as either an integer or a float. If no - fill value is present on the variable, or the fill value is non- - numeric, the function should return None. This is because - pyresample explicitly checks for float or int fill values in - get_sample_from_neighbour_info. + """Ensure a fill value is retrieved from a variable that has a vaild + numeric value, and is cast as either an integer or a float. If no + fill value is present on the variable, or the fill value is non- + numeric, the function should return None. This is because + pyresample explicitly checks for float or int fill values in + get_sample_from_neighbour_info. """ variable = Mock(spec=Variable) - test_args = [['np.float128', np.float128, 4.0, 4.0], - ['np.float16', np.float16, 4.0, 4.0], - ['np.float32', np.float32, 4.0, 4.0], - ['np.float64', np.float64, 4.0, 4.0], - ['np.float_', np.float_, 4.0, 4.0], - ['int', int, 5, 5], - ['np.int0', np.int0, 5, 5], - ['np.int16', np.int16, 5, 5], - ['np.int32', np.int32, 5, 5], - ['np.int64', np.int64, 5, 5], - ['np.int8', np.int8, 5, 5], - ['np.uint', np.uint, 5, 5], - ['np.uint0', np.uint0, 5, 5], - ['np.uint16', np.uint16, 5, 5], - ['np.uint32', np.uint32, 5, 5], - ['np.uint64', np.uint64, 5, 5], - ['np.uint8', np.uint8, 5, 5], - ['np.uintc', np.uintc, 5, 5], - ['np.uintp', np.uintp, 5, 5], - ['np.longlong', np.longlong, 5, 5], - ['float', float, 4.0, 4.0], - ['int', int, 5, 5], - ['str', str, '1235', None]] + test_args = [ + ['np.float128', np.float128, 4.0, 4.0], + ['np.float16', np.float16, 4.0, 4.0], + ['np.float32', np.float32, 4.0, 4.0], + ['np.float64', np.float64, 4.0, 4.0], + ['np.float_', np.float_, 4.0, 4.0], + ['int', int, 5, 5], + ['np.int0', np.int0, 5, 5], + ['np.int16', np.int16, 5, 5], + ['np.int32', np.int32, 5, 5], + ['np.int64', np.int64, 5, 5], + ['np.int8', np.int8, 5, 5], + ['np.uint', np.uint, 5, 5], + ['np.uint0', np.uint0, 5, 5], + ['np.uint16', np.uint16, 5, 5], + ['np.uint32', np.uint32, 5, 5], + ['np.uint64', np.uint64, 5, 5], + ['np.uint8', np.uint8, 5, 5], + ['np.uintc', np.uintc, 5, 5], + ['np.uintp', np.uintp, 5, 5], + ['np.longlong', np.longlong, 5, 5], + ['float', float, 4.0, 4.0], + ['int', int, 5, 5], + ['str', str, '1235', None], + ] for description, caster, fill_value, expected_output in test_args: with self.subTest(description): variable.ncattrs.return_value = ['_FillValue'] variable.getncattr.return_value = caster(fill_value) - self.assertEqual(get_variable_numeric_fill_value(variable), - expected_output) + self.assertEqual( + get_variable_numeric_fill_value(variable), expected_output + ) with self.subTest('Missing fill value attribute returns `None`.'): variable.ncattrs.return_value = ['other_attribute'] @@ -196,44 +199,43 @@ def test_get_variable_numeric_fill_value(self): raw_fill_value = 1 add_offset = 210 scale_factor = 2 - variable.ncattrs.return_value = ['add_offset', '_FillValue', - 'scale_factor'] - variable.getncattr.side_effect = [raw_fill_value, add_offset, - scale_factor] + variable.ncattrs.return_value = ['add_offset', '_FillValue', 'scale_factor'] + variable.getncattr.side_effect = [raw_fill_value, add_offset, scale_factor] self.assertEqual(get_variable_numeric_fill_value(variable), 212) def test_get_variable_file_path(self): - """ Ensure that a file path is correctly constructed from a variable - name. This should also handle a variable within a group, not just - at the root level of the dataset. + """Ensure that a file path is correctly constructed from a variable + name. This should also handle a variable within a group, not just + at the root level of the dataset. """ temporary_directory = '/tmp_dir' file_extension = '.nc' - test_args = [['Root variable', 'var_one', '/tmp_dir/var_one.nc'], - ['Nested variable', '/group/var_two', - '/tmp_dir/group_var_two.nc']] + test_args = [ + ['Root variable', 'var_one', '/tmp_dir/var_one.nc'], + ['Nested variable', '/group/var_two', '/tmp_dir/group_var_two.nc'], + ] for description, variable_name, expected_path in test_args: with self.subTest(description): - variable_path = get_variable_file_path(temporary_directory, - variable_name, - file_extension) + variable_path = get_variable_file_path( + temporary_directory, variable_name, file_extension + ) self.assertEqual(variable_path, expected_path) def test_get_scale_and_offset(self): - """ Ensure that the scaling attributes can be correctly returned from - the input variable attributes, or an empty dictionary if both - add_offset` and `scale_factor` are not present. + """Ensure that the scaling attributes can be correctly returned from + the input variable attributes, or an empty dictionary if both + add_offset` and `scale_factor` are not present. """ variable = Mock(spec=Variable) false_tests = [ ['Neither attribute present.', {'other_key': 123}], ['Only scale_factor is present.', {'scale_factor': 0.01}], - ['Only add_offset is present.', {'add_offset': 123.456}] + ['Only add_offset is present.', {'add_offset': 123.456}], ] for description, attributes in false_tests: @@ -244,18 +246,22 @@ def test_get_scale_and_offset(self): variable.getncattr.assert_not_called() with self.subTest('Contains both required attributes'): - attributes = {'add_offset': 123.456, - 'scale_factor': 0.01, - 'other_key': 'abc'} + attributes = { + 'add_offset': 123.456, + 'scale_factor': 0.01, + 'other_key': 'abc', + } variable.ncattrs.return_value = set(attributes.keys()) variable.getncattr.side_effect = [123.456, 0.01] - self.assertDictEqual(get_scale_and_offset(variable), - {'add_offset': 123.456, 'scale_factor': 0.01}) + self.assertDictEqual( + get_scale_and_offset(variable), + {'add_offset': 123.456, 'scale_factor': 0.01}, + ) def test_construct_absolute_path(self): - """ Ensure that an absolute path can be constructed from a relative one - and the supplied group path. + """Ensure that an absolute path can be constructed from a relative one + and the supplied group path. """ test_args = [ @@ -267,15 +273,14 @@ def test_construct_absolute_path(self): for description, reference, group_path, abs_reference in test_args: with self.subTest(description): self.assertEqual( - construct_absolute_path(reference, group_path), - abs_reference + construct_absolute_path(reference, group_path), abs_reference ) def test_qualify_reference(self): - """ Ensure that a reference within a variable's metadata is correctly - qualified to an absolute variable path, using the nature of the - reference (e.g. prefix of "../" or "./") and the group of the - referee variable. + """Ensure that a reference within a variable's metadata is correctly + qualified to an absolute variable path, using the nature of the + reference (e.g. prefix of "../" or "./") and the group of the + referee variable. """ dataset = Dataset('test.nc', 'w', diskless=True) @@ -283,31 +288,32 @@ def test_qualify_reference(self): dataset.createDimension('lon', size=4) data = np.ones((2, 4)) - variable = dataset.createVariable('/group/variable', data.dtype, - dimensions=('lat', 'lon')) + variable = dataset.createVariable( + '/group/variable', data.dtype, dimensions=('lat', 'lon') + ) - dataset.createVariable('/group/sibling', data.dtype, - dimensions=('lat', 'lon')) + dataset.createVariable('/group/sibling', data.dtype, dimensions=('lat', 'lon')) test_args = [ ['In /group/variable, ref /base_var', '/base_var', '/base_var'], ['In /group/variable, ref ../base_var', '../base_var', '/base_var'], ['In /group/variable, ref ./group_var', './group_var', '/group/group_var'], ['In /group/variable, ref sibling', 'sibling', '/group/sibling'], - ['In /group/variable, ref non-sibling', 'non_sibling', '/non_sibling'] + ['In /group/variable, ref non-sibling', 'non_sibling', '/non_sibling'], ] for description, raw_reference, absolute_reference in test_args: with self.subTest(description): - self.assertEqual(qualify_reference(raw_reference, variable), - absolute_reference) + self.assertEqual( + qualify_reference(raw_reference, variable), absolute_reference + ) dataset.close() def test_variable_in_dataset(self): - """ Ensure that a variable will be correctly identified as belonging - to the dataset. Also, the function should successfully handle - absent intervening groups. + """Ensure that a variable will be correctly identified as belonging + to the dataset. Also, the function should successfully handle + absent intervening groups. """ dataset = Dataset('test.nc', 'w', diskless=True) @@ -316,12 +322,11 @@ def test_variable_in_dataset(self): data = np.ones((2, 4)) - dataset.createVariable('/group/variable', data.dtype, - dimensions=('lat', 'lon')) - dataset.createVariable('/group/group_two/variable_two', data.dtype, - dimensions=('lat', 'lon')) - dataset.createVariable('/base_variable', data.dtype, - dimensions=('lat', 'lon')) + dataset.createVariable('/group/variable', data.dtype, dimensions=('lat', 'lon')) + dataset.createVariable( + '/group/group_two/variable_two', data.dtype, dimensions=('lat', 'lon') + ) + dataset.createVariable('/base_variable', data.dtype, dimensions=('lat', 'lon')) test_args = [ ['Root variable', '/base_variable', True], @@ -331,19 +336,20 @@ def test_variable_in_dataset(self): ['Non existant base variable', '/missing', False], ['Non existant nested variable', '/group/missing', False], ['Non existant group', '/group_three/variable', False], - ['Over nested variable', '/group/group_two/group_three/var', False] + ['Over nested variable', '/group/group_two/group_three/var', False], ] for description, variable_name, expected_result in test_args: with self.subTest(description): - self.assertEqual(variable_in_dataset(variable_name, dataset), - expected_result) + self.assertEqual( + variable_in_dataset(variable_name, dataset), expected_result + ) dataset.close() def test_make_array_two_dimensional(self): - """ Ensure a 1-D array is expaned to be a 2-D array with elements all - in the same column, + """Ensure a 1-D array is expaned to be a 2-D array with elements all + in the same column, """ input_array = np.array([1, 2, 3])