diff --git a/data/config/Point2GridConfig_default b/data/config/Point2GridConfig_default index 8361e871c9..3b8625023d 100644 --- a/data/config/Point2GridConfig_default +++ b/data/config/Point2GridConfig_default @@ -22,11 +22,12 @@ obs_window = { //////////////////////////////////////////////////////////////////////////////// - // -// Observation message type +// Point observation filtering options // -message_type = []; +message_type = []; +obs_quality_inc = []; +obs_quality_exc = []; //////////////////////////////////////////////////////////////////////////////// @@ -72,10 +73,6 @@ var_name_map = [ //////////////////////////////////////////////////////////////////////////////// -quality_mark_thresh = 2; - -//////////////////////////////////////////////////////////////////////////////// - tmp_dir = "/tmp"; version = "V12.0.0"; diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index d9ad8695c1..9be547fad1 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -41,51 +41,39 @@ pb2nc has both required and optional arguments. Required Arguments for pb2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. -The **prepbufr_file** argument is the input PrepBUFR file to be processed. +1. The **prepbufr_file** argument is the input PrepBUFR file to be processed. -2. -The **netcdf_file** argument is the output NetCDF file to be written. +2. The **netcdf_file** argument is the output NetCDF file to be written. -3. -The **config_file** argument is the configuration file to be used. The contents of the configuration file are discussed below. +3. The **config_file** argument is the configuration file to be used. The contents of the configuration file are discussed below. Optional Arguments for pb2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. -The **-pbfile prepbufr_file** option is used to pass additional input PrepBUFR files. +1. The **-pbfile prepbufr_file** option is used to pass additional input PrepBUFR files. -2. -The **-valid_beg** time option in YYYYMMDD[_HH[MMSS]] format sets the beginning of the retention time window. +2. The **-valid_beg** time option in YYYYMMDD[_HH[MMSS]] format sets the beginning of the retention time window. -3. -The **-valid_end** time option in YYYYMMDD[_HH[MMSS]] format sets the end of the retention time window. +3. The **-valid_end** time option in YYYYMMDD[_HH[MMSS]] format sets the end of the retention time window. -4. -The **-nmsg num_messages** option may be used for testing purposes. This argument indicates that only the first "num_messages" PrepBUFR messages should be processed rather than the whole file. This option is provided to speed up testing because running the PB2NC tool can take a few minutes for each file. Most users will not need this option. +4. The **-nmsg num_messages** option may be used for testing purposes. This argument indicates that only the first "num_messages" PrepBUFR messages should be processed rather than the whole file. This option is provided to speed up testing because running the PB2NC tool can take a few minutes for each file. Most users will not need this option. -5. -The **-dump path** option may be used to dump the entire contents of the PrepBUFR file to several ASCII files written to the directory specified by "path". The user may use this option to view a human-readable version of the input PrepBUFR file, although writing the contents to ASCII files can be slow. +5. The **-dump path** option may be used to dump the entire contents of the PrepBUFR file to several ASCII files written to the directory specified by "path". The user may use this option to view a human-readable version of the input PrepBUFR file, although writing the contents to ASCII files can be slow. -6. -The **-index** option shows the available variables with valid data from the BUFR input. It collects the available variable list from BUFR input and checks the existence of valid data and directs the variable names with valid data to the screen. The NetCDF output won't be generated. +6. The **-index** option shows the available variables with valid data from the BUFR input. It collects the available variable list from BUFR input and checks the existence of valid data and directs the variable names with valid data to the screen. The NetCDF output won't be generated. -7. -The **-log** file option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. +7. The **-log** file option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. -8. -The **-v level** option indicates the desired level of verbosity. The value of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. +8. The **-v level** option indicates the desired level of verbosity. The value of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. -9. -The **-compress level** option indicates the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. The value of "level" will override the default setting of 0 from the configuration file or the environment variable MET_NC_COMPRESS. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression. +9. The **-compress level** option indicates the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. The value of "level" will override the default setting of 0 from the configuration file or the environment variable MET_NC_COMPRESS. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression. An example of the pb2nc calling sequence is shown below: .. code-block:: none - - pb2nc sample_pb.blk \ - sample_pb.nc \ - PB2NCConfig + + pb2nc sample_pb.blk \ + sample_pb.nc \ + PB2NCConfig In this example, the PB2NC tool will process the input **sample_pb.blk** file applying the configuration specified in the **PB2NCConfig** file and write the output to a file named **sample_pb.nc**. @@ -101,11 +89,11 @@ Note that environment variables may be used when editing configuration files, as ____________________ .. code-block:: none - - obs_window = { beg = -5400; end = 5400; } - mask = { grid = ""; poly = ""; } - tmp_dir = "/tmp"; - version = "VN.N"; + + obs_window = { beg = -5400; end = 5400; } + mask = { grid = ""; poly = ""; } + tmp_dir = "/tmp"; + version = "VN.N"; The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. The use of temporary files in PB2NC is described in :numref:`Contributor's Guide Section %s `. @@ -113,47 +101,45 @@ The use of temporary files in PB2NC is described in :numref:`Contributor's Guide _____________________ .. code-block:: none - - message_type = []; + + message_type = []; Each PrepBUFR message is tagged with one of eighteen message types as listed in the :numref:`config_options` file. The **message_type** refers to the type of observation from which the observation value (or 'report') was derived. The user may specify a comma-separated list of message types to be retained. Providing an empty list indicates that all message types should be retained. _____________________ -.. code-block:: none +.. code-block:: none - message_type_map = [ { key = "AIRCAR"; val = "AIRCAR_PROFILES"; } ]; + message_type_map = [ { key = "AIRCAR"; val = "AIRCAR_PROFILES"; } ]; The **message_type_map** entry is an array of dictionaries, each containing a **key** string and **val** string. This defines a mapping of input PrepBUFR message types to output message types. This provides a method for renaming input PrepBUFR message types. _____________________ .. code-block:: none - - message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } - ]; + message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } + ]; The **message_type_group_map** entry is an array of dictionaries, each containing a **key** string and **val** string. This defines a mapping of message type group names to a comma-separated list of values. This map is defined in the config files for PB2NC, Point-Stat, or Ensemble-Stat. Modify this map to define sets of message types that should be processed together as a group. The **SURFACE** entry must be present to define message types for which surface verification logic should be applied. _____________________ .. code-block:: none - - station_id = []; + + station_id = []; Each PrepBUFR message has a station identification string associated with it. The user may specify a comma-separated list of station IDs to be retained. Providing an empty list indicates that messages from all station IDs will be retained. It can be a file name containing a list of stations. _____________________ .. code-block:: none - - elevation_range = { beg = -1000; end = 100000; } + elevation_range = { beg = -1000; end = 100000; } The **beg** and **end** variables are used to stratify the elevation (in meters) of the observations to be retained. The range shown above is set to -1000 to 100000 meters, which essentially retains every observation. @@ -161,11 +147,10 @@ _____________________ .. code-block:: none - pb_report_type = []; - in_report_type = []; - instrument_type = []; + pb_report_type = []; + in_report_type = []; + instrument_type = []; - The **pb_report_type, in_report_type**, and **instrument_type** variables are used to specify comma-separated lists of PrepBUFR report types, input report types, and instrument types to be retained, respectively. If left empty, all PrepBUFR report types, input report types, and instrument types will be retained. See the following for more details: `Code table for PrepBUFR report types used by Regional NAM GSI analyses. `_ @@ -175,19 +160,16 @@ The **pb_report_type, in_report_type**, and **instrument_type** variables are us _____________________ .. code-block:: none - - level_range = { beg = 1; end = 255; } - level_category = []; + level_range = { beg = 1; end = 255; } + level_category = []; The **beg** and **end** variables are used to stratify the model level of observations to be retained. The range shown above is 1 to 255. - The **level_category** variable is used to specify a comma-separated list of PrepBUFR data level categories to retain. An empty string indicates that all level categories should be retained. Accepted values and their meanings are described in :numref:`table_reformat-point_pb2nc_level_category`. See the following for more details: `PrepBUFR mnemonic table. `_ - .. _table_reformat-point_pb2nc_level_category: .. list-table:: Values for the level_category option. @@ -212,17 +194,15 @@ The **level_category** variable is used to specify a comma-separated list of Pre - Reports on a single level * - 7 - Auxiliary levels generated via interpolation from spanning levels - + _____________________ .. code-block:: none - - obs_bufr_var = [ 'QOB', 'TOB', 'ZOB', 'UOB', 'VOB' ]; + obs_bufr_var = [ 'QOB', 'TOB', 'ZOB', 'UOB', 'VOB' ]; Each PrepBUFR message will likely contain multiple observation variables. The **obs_bufr_var** variable is used to specify which observation variables should be retained or derived. The observation variable names are retrieved from the BUFR table embedded within the file. Users can run PB2NC with the **-index** command line argument to list out the variable names present in the file, and those names can be listed in this setting. If the list is empty, all BUFR variables present in the file are retained. This setting replaces the deprecated **obs_grib_code**. - The example **obs_bufr_var** setting above retains observations of QOB, TOB, ZOB, UOB, and VOB for specific humidity, temperature, height, and the u and v components of winds. Observations of those types are reported at the corresponding POB pressure level. In addition, PB2NC can derive several other variables from these observations. By convention, all observations that are derivable are named with a **D_** prefix: • **D_DPT** for dew point (from POB and QOB) @@ -243,39 +223,37 @@ The example **obs_bufr_var** setting above retains observations of QOB, TOB, ZOB • **D_MLCAPE** for mixed layer convective available potential energy (from POB, QOB, and TOB) - In BUFR, lower quality mark values indicate higher quality observations. The quality marks for derived observations are computed as the maximum of the quality marks for its components. For example, **D_DPT** derived from **POB** with quality mark 1 and **QOB** with quality mark 2 is assigned a quality mark value of 2. **D_PBL**, **D_CAPE**, and **D_MLCAPE** are derived using data from multiple vertical levels. Their quality marks are computed as the maximum of their components over all vertical levels. _____________________ .. code-block:: none - - obs_bufr_map = [ - { key = 'POB'; val = 'PRES'; }, - { key = 'QOB'; val = 'SPFH'; }, - { key = 'TOB'; val = 'TMP'; }, - { key = 'ZOB'; val = 'HGT'; }, - { key = 'UOB'; val = 'UGRD'; }, - { key = 'VOB'; val = 'VGRD'; }, - { key = 'D_DPT'; val = 'DPT'; }, - { key = 'D_WDIR'; val = 'WDIR'; }, - { key = 'D_WIND'; val = 'WIND'; }, - { key = 'D_RH'; val = 'RH'; }, - { key = 'D_MIXR'; val = 'MIXR'; }, - { key = 'D_PRMSL'; val = 'PRMSL'; }, - { key = 'D_PBL'; val = 'PBL'; }, - { key = 'D_CAPE'; val = 'CAPE'; } - { key = 'D_MLCAPE'; val = 'MLCAPE'; } - ]; + obs_bufr_map = [ + { key = 'POB'; val = 'PRES'; }, + { key = 'QOB'; val = 'SPFH'; }, + { key = 'TOB'; val = 'TMP'; }, + { key = 'ZOB'; val = 'HGT'; }, + { key = 'UOB'; val = 'UGRD'; }, + { key = 'VOB'; val = 'VGRD'; }, + { key = 'D_DPT'; val = 'DPT'; }, + { key = 'D_WDIR'; val = 'WDIR'; }, + { key = 'D_WIND'; val = 'WIND'; }, + { key = 'D_RH'; val = 'RH'; }, + { key = 'D_MIXR'; val = 'MIXR'; }, + { key = 'D_PRMSL'; val = 'PRMSL'; }, + { key = 'D_PBL'; val = 'PBL'; }, + { key = 'D_CAPE'; val = 'CAPE'; } + { key = 'D_MLCAPE'; val = 'MLCAPE'; } + ]; The BUFR variable names are not shared with other forecast data. This map is used to convert the BUFR name to the common name, like GRIB2. It allows to share the configuration for forecast data with PB2NC observation data. If there is no mapping, the BUFR variable name will be saved to output NetCDF file. _____________________ .. code-block:: none - - quality_mark_thresh = 2; + + quality_mark_thresh = 2; Each observation has a quality mark value associated with it. The **quality_mark_thresh** is used to stratify out which quality marks will be retained. The value shown above indicates that only observations with quality marks less than or equal to 2 will be retained. @@ -283,51 +261,45 @@ Each observation has a quality mark value associated with it. The **quality_mark _____________________ .. code-block:: none - - event_stack_flag = TOP; + event_stack_flag = TOP; A PrepBUFR message may contain duplicate observations with different quality mark values. The **event_stack_flag** indicates whether to use the observations at the top of the event stack (observation values have had more quality control processing applied) or the bottom of the event stack (observation values have had no quality control processing applied). The flag value of **TOP** listed above indicates the observations with the most amount of quality control processing should be used, the **BOTTOM** option uses the data closest to raw values. _____________________ .. code-block:: none - - time_summary = { - flag = FALSE; - raw_data = FALSE; - beg = "000000"; - end = "235959"; - step = 300; - width = 600; - // width = { beg = -300; end = 300; } - grib_code = []; - obs_var = [ "TMP", "WDIR", "RH" ]; - type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ]; - vld_freq = 0; - vld_thresh = 0.0; - } + time_summary = { + flag = FALSE; + raw_data = FALSE; + beg = "000000"; + end = "235959"; + step = 300; + width = 600; + // width = { beg = -300; end = 300; } + grib_code = []; + obs_var = [ "TMP", "WDIR", "RH" ]; + type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ]; + vld_freq = 0; + vld_thresh = 0.0; + } The **time_summary** dictionary enables additional processing for observations with high temporal resolution. The **flag** entry toggles the **time_summary** on (**TRUE**) and off (**FALSE**). If the **raw_data** flag is set to TRUE, then both the individual observation values and the derived time summary value will be written to the output. If FALSE, only the summary values are written. Observations may be summarized across the user specified time period defined by the **beg** and **end** entries in HHMMSS format. The **step** entry defines the time between intervals in seconds. The **width** entry specifies the summary interval in seconds. It may either be set as an integer number of seconds for a centered time interval or a dictionary with beginning and ending time offsets in seconds. - This example listed above does a 10-minute time summary (width = 600;) every 5 minutes (step = 300;) throughout the day (beg = "000000"; end = 235959";). The first interval will be from 23:55:00 the previous day through 00:04:59 of the current day. The second interval will be from 0:00:00 through 00:09:59. And so on. - The two **width** settings listed above are equivalent. Both define a centered 10-minute time interval. Use the **beg** and **end** entries to define uncentered time intervals. The following example requests observations for one hour prior: .. code-block:: none - - width = { beg = -3600; end = 0; } + width = { beg = -3600; end = 0; } -The summaries will only be calculated for the observations specified in the **grib_code** or **obs_var** entries. The **grib_code** entry is an array of integers while the **obs_var** entries is an array of strings. The supported summaries are **min** (minimum), **max** (maximum), **range, mean, stdev** (standard deviation), **median** and **p##** (percentile, with the desired percentile value specified in place of ##). If multiple summaries are selected in a single run, a string indicating the summary method applied will be appended to the output message type. +The summaries will only be calculated for the observations specified in the **grib_code** or **obs_var** entries. The **grib_code** entry is an array of integers while the **obs_var** entries is an array of strings. The supported summaries are **min** (minimum), **max** (maximum), **range, mean, stdev** (standard deviation), **median** and **p##** (percentile, with the desired percentile value specified in place of ##). If multiple summaries are selected in a single run, a string indicating the summary method applied will be appended to the output message type. The **vld_freq** and **vld_thresh** entries specify the required ratio of valid data for an output time summary value to be computed. This option is only applied when these entries are set to non-zero values. The **vld_freq** entry specifies the expected frequency of observations in seconds. The width of the time window is divided by this frequency to compute the expected number of observations for the time window. The actual number of valid observations is divided by the expected number to compute the ratio of valid data. An output time summary value will only be written if that ratio is greater than or equal to the **vld_thresh** entry. Detailed information about which observations are excluded is provided at debug level 4. - The quality mark for time summaries is always reported by PB2NC as bad data. Time summaries are computed by several MET point pre-processing tools using common library code. While BUFR quality marks are integers, the quality flags for other point data formats (MADIS NetCDF, for example) are stored as strings. MET does not currently contain logic to determine which quality flag strings are better or worse. Note however that any point observation whose quality mark does not meet the **quality_mark_thresh** criteria is not used in the computation of time summaries. .. _pb2nc output: @@ -358,14 +330,14 @@ Each NetCDF file generated by the PB2NC tool contains the dimensions and variabl * - nobs_qty - Number of unique quality control strings (variable) * - obs_var_num - - Number of unique observation variable types (variable) + - Number of unique observation variable types (variable) .. _table_reformat-point_pb2nc_output_vars: .. list-table:: NetCDF variables in pb2nc output :widths: auto :header-rows: 2 - + * - pb2nc NetCDF VARIABLES - - @@ -436,7 +408,6 @@ Each NetCDF file generated by the PB2NC tool contains the dimensions and variabl - obs_var_num, mxstr3 - Lookup table containing a description string for the unique observation variable names in obs_var. - ASCII2NC Tool ============= @@ -518,7 +489,7 @@ ascii2nc Usage Once the ASCII point observations have been formatted as expected, the ASCII file is ready to be processed by the ASCII2NC tool. The usage statement for ASCII2NC tool is shown below: .. code-block:: none - + Usage: ascii2nc ascii_file1 [ascii_file2 ... ascii_filen] netcdf_file @@ -568,9 +539,9 @@ Optional Arguments for ascii2nc An example of the ascii2nc calling sequence is shown below: .. code-block:: none - - ascii2nc sample_ascii_obs.txt \ - sample_ascii_obs.nc + + ascii2nc sample_ascii_obs.txt \ + sample_ascii_obs.nc In this example, the ASCII2NC tool will reformat the input **sample_ascii_obs.txt file** into NetCDF format and write the output to a file named **sample_ascii_obs.nc**. @@ -587,7 +558,7 @@ _____________________ .. code-block:: none - version = "VN.N"; + version = "VN.N"; The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. @@ -595,31 +566,28 @@ _____________________ .. code-block:: none - time_summary = { ... } - + time_summary = { ... } The **time_summary** feature was implemented to allow additional processing of observations with high temporal resolution, such as SURFRAD data every 5 minutes. This option is described in :numref:`pb2nc configuration file`. _____________________ .. code-block:: none - - message_type_map = [ - { key = "FM-12 SYNOP"; val = "ADPSFC"; }, - { key = "FM-13 SHIP"; val = "SFCSHP"; }, - { key = "FM-15 METAR"; val = "ADPSFC"; }, - { key = "FM-18 BUOY"; val = "SFCSHP"; }, - { key = "FM-281 QSCAT"; val = "ASCATW"; }, - { key = "FM-32 PILOT"; val = "ADPUPA"; }, - { key = "FM-35 TEMP"; val = "ADPUPA"; }, - { key = "FM-88 SATOB"; val = "SATWND"; }, - { key = "FM-97 ACARS"; val = "AIRCFT"; } - ]; + message_type_map = [ + { key = "FM-12 SYNOP"; val = "ADPSFC"; }, + { key = "FM-13 SHIP"; val = "SFCSHP"; }, + { key = "FM-15 METAR"; val = "ADPSFC"; }, + { key = "FM-18 BUOY"; val = "SFCSHP"; }, + { key = "FM-281 QSCAT"; val = "ASCATW"; }, + { key = "FM-32 PILOT"; val = "ADPUPA"; }, + { key = "FM-35 TEMP"; val = "ADPUPA"; }, + { key = "FM-88 SATOB"; val = "SATWND"; }, + { key = "FM-97 ACARS"; val = "AIRCFT"; } + ]; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of input strings to output message types. This mapping is currently only applied when converting input little_r report types to output message types. - ascii2nc Output --------------- @@ -627,21 +595,18 @@ The NetCDF output of the ASCII2NC tool is structured in the same way as the outp "obs_vid" variable is replaced with "obs_gc" when the GRIB code is given instead of the variable names. In this case, the global variable "use_var_id" does not exist or set to false (use_var_id = "false" ;). Three variables (obs_var, obs_units, and obs_desc) related with variable names are not added. - MADIS2NC Tool ============= - This section describes how to run the MADIS2NC tool. The MADIS2NC tool is used to reformat `Meteorological Assimilation Data Ingest System (MADIS) `_ point observations into the NetCDF format expected by the MET statistics tools. An optional configuration file controls the processing of the point observations. The MADIS2NC tool supports many of the MADIS data types, as listed in the usage statement below. Support for additional MADIS data types may be added in the future based on user feedback. - madis2nc Usage -------------- The usage statement for the MADIS2NC tool is shown below: .. code-block:: none - + Usage: madis2nc madis_file [madis_file2 ... madis_filen] out_file @@ -658,81 +623,61 @@ The usage statement for the MADIS2NC tool is shown below: [-v level] [-compress level] - madis2nc has required arguments and can also take optional ones. - Required Arguments for madis2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 1. The **madis_file** argument is one or more input MADIS point observation files to be processed. - 2. The **out_file** argument is the NetCDF output file to be written. - 3. The argument **-type str** is a type of MADIS observations (metar, raob, profiler, maritime, mesonet or acarsProfiles). - Optional Arguments for madis2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 4. The **-config file** option specifies the configuration file to generate summaries of the fields in the ASCII files. - 5. The **-qc_dd list** option specifies a comma-separated list of QC flag values to be accepted(Z,C,S,V,X,Q,K,G,B). - 6. The **-lvl_dim list** option specifies a comma-separated list of vertical level dimensions to be processed. - 7. To specify the exact records to be processed, the **-rec_beg n** specifies the index of the first MADIS record to process and **-rec_end n** specifies the index of the last MADIS record to process. Both are zero-based. - 8. The **-mask_grid string** option specifies a named grid or a gridded data file for filtering the point observations spatially. - 9. The **-mask_poly file** option defines a polyline masking file for filtering the point observations spatially. - 10. The **-mask_sid file|list** option is a station ID masking file or a comma-separated list of station ID's for filtering the point observations spatially. See the description of the "sid" entry in :numref:`config_options`. - 11. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. - 12. The **-v level** option indicates the desired level of verbosity. The value of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity will increase the amount of logging. - 13. The **-compress level** option specifies the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression. - An example of the madis2nc calling sequence is shown below: .. code-block:: none - - madis2nc sample_madis_obs.nc \ - sample_madis_obs_met.nc -log madis.log -v 3 + madis2nc sample_madis_obs.nc \ + sample_madis_obs_met.nc -log madis.log -v 3 In this example, the MADIS2NC tool will reformat the input sample_madis_obs.nc file into NetCDF format and write the output to a file named sample_madis_obs_met.nc. Warnings and error messages will be written to the madis.log file, and the verbosity level of logging is three. - madis2nc Configuration File --------------------------- - The default configuration file for the MADIS2NC tool named **Madis2NcConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. - The MADIS2NC configuration file is optional and only necessary when defining time summaries. The contents of the default MADIS2NC configuration file are described below. _____________________ .. code-block:: none - version = "VN.N"; - + version = "VN.N"; The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. @@ -740,12 +685,10 @@ _____________________ .. code-block:: none - time_summary = { ... } - + time_summary = { ... } The **time_summary** dictionary is described in :numref:`pb2nc configuration file`. - madis2nc Output --------------- @@ -753,14 +696,11 @@ The NetCDF output of the MADIS2NC tool is structured in the same way as the outp "obs_vid" variable is replaced with "obs_gc" when the GRIB code is given instead of the variable names. In this case, the global variable "use_var_id" does not exist or set to false (use_var_id = "false" ;). Three variables (obs_var, obs_units, and obs_desc) related with variable names are not added. - LIDAR2NC Tool ============= - The LIDAR2NC tool creates a NetCDF point observation file from a CALIPSO HDF data file. Not all of the data present in the CALIPSO file is reproduced in the output, however. Instead, the output focuses mostly on information about clouds (as opposed to aerosols) as seen by the satellite along its ground track. - lidar2nc Usage -------------- @@ -775,19 +715,16 @@ The usage statement for LIDAR2NC tool is shown below: [-v level] [-compress level] - -Unlike most of the MET tools, lidar2nc does not use a config file. Currently, the options needed to run lidar2nc are not complex enough to require one. +Unlike most of the MET tools, lidar2nc does not use a config file. Currently, the options needed to run lidar2nc are not complex enough to require one. Required Arguments for lidar2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 1. The **lidar_file** argument is the input HDF lidar data file to be processed. Currently, CALIPSO files are supported but support for additional file types will be added in future releases. - 2. The **out_file** argument is the NetCDF output file to be written. - Optional Arguments for lidar2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -802,13 +739,10 @@ lidar2nc Output Each observation type in the lidar2nc output is assigned a GRIB code. These are outlined in :numref:`lidar2nc_grib_code_table`. GRIB codes were assigned to these fields arbitrarily, with GRIB codes in the 600s denoting individual bit fields taken from the feature classification flag field in the CALIPSO file. - We will not give a detailed description of each CALIPSO data product that lidar2nc reads. Users should refer to existing CALIPSO documentation for this information. We will, however, give some explanation of how the cloud layer base and top information is encoded in the lidar2nc NetCDF output file. - **Layer_Base** gives the elevation in meters above ground level of the cloud base for each cloud level at each observation location. Similarly, **Layer_Top** gives the elevation of the top of each cloud layer. Note that if there are multiple cloud layers at a particular location, then there will be more than one base (or top) given for that location. For convenience, **Min_Base** and **Max_Top** give, respectively, the base elevation for the bottom cloud layer, and the top elevation for the top cloud layer. For these data types, there will be only one value per observation location regardless of how many cloud layers there are at that location. - .. _lidar2nc_grib_code_table: .. list-table:: lidar2nc GRIB codes and their meaning, units, and abbreviations @@ -868,21 +802,18 @@ We will not give a detailed description of each CALIPSO data product that lidar2 - NA - Horizontal_Averaging - IODA2NC Tool ============ - This section describes the IODA2NC tool which is used to reformat IODA (Interface for Observation Data Access) point observations from the `Joint Center for Satellite Data Assimilation (JCSDA) `_ into the NetCDF format expected by the MET statistics tools. An optional configuration file controls the processing of the point observations. The IODA2NC tool reads NetCDF point observation files created by the `IODA Converters `_. Support for interfacing with data from IODA may be added in the future based on user feedback. - ioda2nc Usage ------------- The usage statement for the IODA2NC tool is shown below: .. code-block:: none - + Usage: ioda2nc ioda_file netcdf_file @@ -928,12 +859,11 @@ An example of the ioda2nc calling sequence is shown below: .. code-block:: none - ioda2nc \ - ioda.NC001007.2020031012.nc ioda2nc.2020031012.nc \ - -config IODA2NCConfig -v 3 -lg run_ioda2nc.log - -In this example, the IODA2NC tool will reformat the data in the input ioda.NC001007.2020031012.nc file and write the output to a file named ioda2nc.2020031012.nc. The data to be processed is specified by IODA2NCConfig, log messages will be written to the ioda2nc.log file, and the verbosity level is three. + ioda2nc \ + ioda.NC001007.2020031012.nc ioda2nc.2020031012.nc \ + -config IODA2NCConfig -v 3 -lg run_ioda2nc.log +In this example, the IODA2NC tool will reformat the data in the input ioda.NC001007.2020031012.nc file and write the output to a file named ioda2nc.2020031012.nc. The data to be processed is specified by IODA2NCConfig, log messages will be written to the ioda2nc.log file, and the verbosity level is three. ioda2nc Configuration File -------------------------- @@ -946,10 +876,10 @@ _____________________ .. code-block:: none - obs_window = { beg = -5400; end = 5400; } - mask = { grid = ""; poly = ""; } - tmp_dir = "/tmp"; - version = "VN.N"; + obs_window = { beg = -5400; end = 5400; } + mask = { grid = ""; poly = ""; } + tmp_dir = "/tmp"; + version = "VN.N"; The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. @@ -957,15 +887,15 @@ _____________________ .. code-block:: none - message_type = []; - message_type_group_map = []; - message_type_map = []; - station_id = []; - elevation_range = { ... }; - level_range = { ... }; - obs_var = []; - quality_mark_thresh = 0; - time_summary = { ... } + message_type = []; + message_type_group_map = []; + message_type_map = []; + station_id = []; + elevation_range = { ... }; + level_range = { ... }; + obs_var = []; + quality_mark_thresh = 0; + time_summary = { ... } The configuration options listed above are supported by other point observation pre-processing tools and are described in :numref:`pb2nc configuration file`. @@ -973,22 +903,22 @@ _____________________ .. code-block:: none - obs_name_map = []; + obs_name_map = []; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of input IODA variable names to output variable names. The default IODA map, obs_var_map, is appended to this map. _____________________ .. code-block:: none - - metadata_map = [ - { key = "message_type"; val = "msg_type,station_ob"; }, - { key = "station_id"; val = "station_id,report_identifier"; }, - { key = "pressure"; val = "air_pressure,pressure"; }, - { key = "height"; val = "height,height_above_mean_sea_level"; }, - { key = "elevation"; val = "elevation,station_elevation"; }, - { key = "nlocs"; val = "Location"; } - ]; + + metadata_map = [ + { key = "message_type"; val = "msg_type,station_ob"; }, + { key = "station_id"; val = "station_id,report_identifier"; }, + { key = "pressure"; val = "air_pressure,pressure"; }, + { key = "height"; val = "height,height_above_mean_sea_level"; }, + { key = "elevation"; val = "elevation,station_elevation"; }, + { key = "nlocs"; val = "Location"; } + ]; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files. The "nlocs" is for the dimension name of the locations. The following key can be added: "nstring", "latitude" and "longitude". @@ -996,11 +926,11 @@ The "nlocs" is for the dimension name of the locations. The following key can be _____________________ .. code-block:: none - - obs_to_qc_map = [ - { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, - { key = "wind_speed"; val = "eastward_wind,northward_wind"; } - ]; + + obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } + ]; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of QC variable name for IODA data files. @@ -1008,17 +938,15 @@ _____________________ .. code-block:: none - missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; + missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; The **missing_thresh** option is an array of thresholds. Any data values which meet any of these thresholds are interpreted as being bad, or missing, data. - ioda2nc Output -------------- The NetCDF output of the IODA2NC tool is structured in the same way as the output of the PB2NC tool described in :numref:`pb2nc output`. - Point2Grid Tool =============== @@ -1030,15 +958,15 @@ point2grid Usage The usage statement for the Point2Grid tool is shown below: .. code-block:: none - + Usage: point2grid input_filename to_grid output_filename -field string [-config file] - [-qc flags] - [-adp adp_file_name] + [-goes_qc flags] + [-adp adp_filename] [-method type] [-gaussian_dx n] [-gaussian_radius n] @@ -1049,7 +977,6 @@ The usage statement for the Point2Grid tool is shown below: [-v level] [-compress level] - Required Arguments for point2grid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1063,21 +990,20 @@ The MET point observation NetCDF file name as **input_filename** argument is equ 4. The **-field** string argument is a string that defines the data to be regridded. It may be used multiple times. If **-adp** option is given (for AOD data from GOES16/17), the name consists with the variable name from the input data file and the variable name from ADP data file (for example, "AOD_Smoke" or "AOD_Dust": getting AOD variable from the input data and applying smoke or dust variable from ADP data file). - Optional Arguments for point2grid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 5. The **-config** file option is the configuration file to be used. -6. The **-qc** flags option specifies a comma-separated list of quality control (QC) flags, for example "0,1". This should only be applied if grid_mapping is set to "goes_imager_projection" and the QC variable exists. +6. The **-goes_qc** flags option specifies a comma-separated list of quality control (QC) flags, for example "0,1". Only used if grid_mapping is set to "goes_imager_projection" and the QC variable exists. Note that the older **-qc** option name is also supported. -7. The **-adp adp_file_name** option provides an additional Aerosol Detection Product (ADP) information on aerosols, dust, and smoke. This option is ignored if the requested variable is not AOD ("AOD_Dust" or "AOD_Smoke") from GOES16/17. The gridded data is filtered by the presence of dust/smoke. If -qc options are given, it's applied to QC of dust/smoke, too (First filtering with AOD QC values and the second filtering with dust/smoke QC values). +7. The **-adp adp_filename** option provides an additional Aerosol Detection Product (ADP) information on aerosols, dust, and smoke. This option is ignored if the requested variable is not AOD ("AOD_Dust" or "AOD_Smoke") from GOES16/17. The gridded data is filtered by the presence of dust/smoke. If -goes_qc options are given, it's applied to QC of dust/smoke, too (First filtering with AOD QC values and the second filtering with dust/smoke QC values). 8. The **-method type** option specifies the regridding method. The default method is UW_MEAN. -9. The **-gaussian_dx n** option defines the distance interval for Gaussian smoothing. The default is 81.271 km. Ignored if the method is not GAUSSIAN or MAXGAUSS. +9. The **-gaussian_dx n** option specifies the distance interval for Gaussian smoothing. The default is 81.271 km. Only used if the method is GAUSSIAN or MAXGAUSS. -10. The **-gaussian_radius** n option defines the radius of influence for Gaussian interpolation. The default is 120. Ignored if the method is not GAUSSIAN or MAXGAUSS. +10. The **-gaussian_radius** n option specifies the radius of influence for Gaussian interpolation. The default is 120. Only used if the method is GAUSSIAN or MAXGAUSS. 11. The **-prob_cat_thresh string** option sets the threshold to compute the probability of occurrence. The default is set to disabled. This option is relevant when calculating practically perfect forecasts. @@ -1109,26 +1035,26 @@ The grid name or the grid definition can be given with the -field option when th -v 1 .. code-block:: none - - point2grid \ - OR_ABI-L2-AODC-M3_G16_s20181341702215_e20181341704588_c20181341711418.nc \ - G212 \ - regrid_data_plane_GOES-16_AOD_TO_G212.nc \ - -field 'name="AOD"; level="(*,*)";' \ - -qc 0,1,2 - -method MAX -v 1 + + point2grid \ + OR_ABI-L2-AODC-M3_G16_s20181341702215_e20181341704588_c20181341711418.nc \ + G212 \ + regrid_data_plane_GOES-16_AOD_TO_G212.nc \ + -field 'name="AOD"; level="(*,*)";' \ + -goes_qc 0,1,2 + -method MAX -v 1 -When processing GOES-16 data, the **-qc** option may also be used to specify the acceptable quality control flag values. The example above regrids the GOES-16 AOD values to NCEP Grid number 212 (which QC flags are high, medium, and low), writing to the output the maximum AOD value falling inside each grid box. +When processing GOES-16 data, the **-goes_qc** option may also be used to specify the acceptable quality control flag values. The example above regrids the GOES-16 AOD values to NCEP Grid number 212 (which QC flags are high, medium, and low), writing to the output the maximum AOD value falling inside each grid box. Listed below is an example of processing the same set of observations but using Python embedding instead: .. code-block:: none - - point2grid \ - 'PYTHON_NUMPY=MET_BASE/python/examples/read_met_point_obs.py ascii2nc_edr_hourly.20130827.nc' \ - G212 python_gridded_ascii_python.nc -config Point2GridConfig_edr \ - -field 'name="200"; level="*"; valid_time="20130827_205959";' -method MAX -v 1 + + point2grid \ + 'PYTHON_NUMPY=MET_BASE/python/examples/read_met_point_obs.py ascii2nc_edr_hourly.20130827.nc' \ + G212 python_gridded_ascii_python.nc -config Point2GridConfig_edr \ + -field 'name="200"; level="*"; valid_time="20130827_205959";' -method MAX -v 1 Please refer to :numref:`Appendix F, Section %s ` for more details about Python embedding in MET. @@ -1137,25 +1063,18 @@ point2grid Output The point2grid tool will output a gridded NetCDF file containing the following: - 1. Latitude - 2. Longitude - 3. The variable specified in the -field string regridded to the grid defined in the **to_grid** argument. - 4. The count field which represents the number of point observations that were included calculating the value of the variable at that grid cell. - 5. The mask field which is a binary field representing the presence or lack thereof of point observations at that grid cell. A value of "1" indicates that there was at least one point observation within the bounds of that grid cell and a value of "0" indicates the lack of point observations at that grid cell. - 6. The probability field which is the probability of the event defined by the **-prob_cat_thresh** command line option. The output variable name includes the threshold used to define the probability. Ranges from 0 to 1. - 7. The probability mask field which is a binary field that represents whether or not there is probability data at that grid point. Can be either "0" or "1" with "0" meaning the probability value does not exist and a value of "1" meaning that the probability value does exist. For MET observation input and CF complaint NetCDF input with 2D time variable: The latest observation time within the target grid is saved as the observation time. If the "valid_time" is configured at the configuration file, the valid_time from the configuration file is saved into the output file. @@ -1163,7 +1082,6 @@ For MET observation input and CF complaint NetCDF input with 2D time variable: T point2grid Configuration File ----------------------------- - The default configuration file for the point2grid tool named **Point2GridConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. The point2grid configuration file is optional and only necessary when defining the variable name instead of GRIB code or filtering by time. The contents of the default point2grid configuration file are described below. @@ -1172,27 +1090,23 @@ _____________________ .. code-block:: none - version = "VN.N"; - + obs_window = { beg = -5400; end = 5400; } + message_type = []; + obs_quality_inc = []; + obs_quality_exc = []; + version = "VN.N"; The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. _____________________ .. code-block:: none - - valid_time = "YYYYMMDD_HHMMSS"; -This entry is a string to override the obseration time into the output and to filter observation data by time. + valid_time = "YYYYMMDD_HHMMSS"; -.. code-block:: none - - obs_window = { - beg = -5400; - end = 5400; - } +This entry is a string to override the observation time into the output and to filter observation data by time. -The configuration option listed above is common to many MET tools and are described in :numref:`config_options`. +_____________________ .. code-block:: none @@ -1202,21 +1116,14 @@ The configuration option listed above is common to many MET tools and are descri { key = "7"; val = "HGT"; }, // GRIB: Geopotential height { key = "11"; val = "TMP"; }, // GRIB: Temperature { key = "15"; val = "TMAX"; }, // GRIB: Max Temperature - ... - ] - -This entry is an array of dictionaries, each containing a **GRIB code** string and mathcing **variable name** string which define a mapping of GRIB code to the output variable names. - -.. code-block:: none - - var_name_map = [ ... - { key = "lat_vname"; val = "NLAT"; }, - { key = "lon_vname"; val = "NLON"; } + { key = "lat_vname"; val = "NLAT"; }, // NetCDF latitude variable name + { key = "lon_vname"; val = "NLON"; }, // NetCDF longitude varialbe name ... ] -The latitude and longitude variables for NetCDF input can be overridden by the configurations. There are two special keys, **lat_vname** and **lon_vname** which are applied to the NetCDF input, not for GRIB code. +This entry is an array of dictionaries, each containing a **GRIB code** string and matching **variable name** string which define a mapping of GRIB code to the output variable names. +The latitude and longitude variables for NetCDF input can be overridden by the configurations. There are two special keys, **lat_vname** and **lon_vname**, are applied to the NetCDF input, not for a GRIB code. Point NetCDF to ASCII Python Utility ==================================== diff --git a/internal/test_unit/config/Point2GridConfig_WINDS b/internal/test_unit/config/Point2GridConfig_WINDS new file mode 100644 index 0000000000..d39221adbe --- /dev/null +++ b/internal/test_unit/config/Point2GridConfig_WINDS @@ -0,0 +1,2 @@ +obs_quality_inc = [ "2" ]; +obs_quality_exc = [ "1" ]; diff --git a/internal/test_unit/config/Point2GridConfig_valid_time b/internal/test_unit/config/Point2GridConfig_valid_time index 748da58b4f..a2a1a69a0a 100644 --- a/internal/test_unit/config/Point2GridConfig_valid_time +++ b/internal/test_unit/config/Point2GridConfig_valid_time @@ -22,11 +22,12 @@ valid_time = "20201022_173000"; //////////////////////////////////////////////////////////////////////////////// - // -// Observation message type +// Point observation filtering options // -//message_type = []; +message_type = []; +obs_quality_inc = []; +obs_quality_exc = []; //////////////////////////////////////////////////////////////////////////////// @@ -72,10 +73,6 @@ valid_time = "20201022_173000"; //////////////////////////////////////////////////////////////////////////////// -//quality_mark_thresh = 2; - -//////////////////////////////////////////////////////////////////////////////// - tmp_dir = "/tmp"; version = "V12.0.0"; diff --git a/internal/test_unit/xml/unit_point2grid.xml b/internal/test_unit/xml/unit_point2grid.xml index ac94683703..c198834114 100644 --- a/internal/test_unit/xml/unit_point2grid.xml +++ b/internal/test_unit/xml/unit_point2grid.xml @@ -46,6 +46,23 @@ + + &MET_BIN;/point2grid + \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + G212 \ + &OUTPUT_DIR;/point2grid/pb2nc_WINDS.nc \ + -field 'name="UGRD"; level="*";' \ + -field 'name="VGRD"; level="*";' \ + -name UWIND,VWIND \ + -config &CONFIG_DIR;/Point2GridConfig_WINDS \ + -v 3 + + + &OUTPUT_DIR;/point2grid/pb2nc_WINDS.nc + + + &MET_BIN;/point2grid \ @@ -130,7 +147,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_compute.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 0,1 -method MAX \ + -goes_qc 0,1 -method MAX \ -v 1 @@ -167,7 +184,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP.nc \ -field 'name="AOD_Smoke"; level="(*,*)";' \ -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20192662141196_e20192662143569_c20192662144526.nc \ - -qc 0,1 -method MAX \ + -goes_qc 0,1 -method MAX \ -v 1 @@ -185,7 +202,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_grid_map.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 0,1,2 -method MAX \ + -goes_qc 0,1,2 -method MAX \ -v 1 @@ -205,7 +222,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 0,1,2 -method MAX \ + -goes_qc 0,1,2 -method MAX \ -v 1 @@ -224,7 +241,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc \ -field 'name="AOD_Smoke"; level="(*,*)";' \ -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20241100001171_e20241100003544_c20241100006361.nc \ - -qc 0,1 -method MAX \ + -goes_qc 0,1 -method MAX \ -v 1 @@ -319,6 +336,7 @@ + &MET_BIN;/point2grid diff --git a/src/basic/vx_util/crc_array.h b/src/basic/vx_util/crc_array.h index 6a8841afce..0eea7d529c 100644 --- a/src/basic/vx_util/crc_array.h +++ b/src/basic/vx_util/crc_array.h @@ -22,7 +22,6 @@ #include #include "num_array.h" -#include "int_array.h" #include "is_bad_data.h" #include "nint.h" #include "vx_cal.h" diff --git a/src/basic/vx_util/int_array.h b/src/basic/vx_util/int_array.h index c435c4aba7..608f6f2dfa 100644 --- a/src/basic/vx_util/int_array.h +++ b/src/basic/vx_util/int_array.h @@ -20,8 +20,6 @@ #include -#include "num_array.h" - #include "crc_array.h" @@ -34,6 +32,12 @@ typedef CRC_Array IntArray; //////////////////////////////////////////////////////////////////////// +extern ConcatString write_css(const IntArray &); + + +//////////////////////////////////////////////////////////////////////// + + #endif /* __INT_ARRAY_H__ */ diff --git a/src/basic/vx_util/num_array.cc b/src/basic/vx_util/num_array.cc index e2a8c79ff0..cb5046c8a6 100644 --- a/src/basic/vx_util/num_array.cc +++ b/src/basic/vx_util/num_array.cc @@ -19,6 +19,7 @@ #include "num_array.h" +#include "int_array.h" #include "is_bad_data.h" #include "ptile.h" #include "nint.h" @@ -1382,3 +1383,22 @@ ConcatString write_css_hhmmss(const NumArray &na) //////////////////////////////////////////////////////////////////////// + + +ConcatString write_css(const IntArray &ia) + +{ + + ConcatString css; + + for ( int i=0; iobs_cnt = obs_cnt; } - - - //////////////////////////////////////////////////////////////////////// - - - // - // Code for class MetPointDataPython - // - - +// +// Code for class MetPointDataPython +// //////////////////////////////////////////////////////////////////////// MetPointDataPython::MetPointDataPython() { @@ -178,14 +169,12 @@ MetPointDataPython::MetPointDataPython(MetPointDataPython &d) { header_data.assign(*d.get_header_data()); } - //////////////////////////////////////////////////////////////////////// MetPointDataPython::~MetPointDataPython() { clear(); } - //////////////////////////////////////////////////////////////////////// void MetPointDataPython::allocate(int obs_cnt) { @@ -193,63 +182,47 @@ void MetPointDataPython::allocate(int obs_cnt) { obs_data->allocate(); } -//////////////////////////////////////////////////////////////////////// - - - /////////////////////////////////////////////////////////////////////////////// -// struct MetPointObsData - -MetPointObsData::MetPointObsData(): - obs_cnt(0), - obs_ids((int *)0), - obs_hids((int *)0), - obs_qids((int *)0), - obs_lvls((float *)0), - obs_hgts((float *)0), - obs_vals((float *)0), - obs_arr((float *)0), - is_obs_array(false) -{ -} +// +// Code for struct MetPointObsData +// +/////////////////////////////////////////////////////////////////////////////// +MetPointObsData::MetPointObsData() { + clear(); +} /////////////////////////////////////////////////////////////////////////////// void MetPointObsData::allocate() { - if (is_obs_array) obs_arr = new float[obs_cnt*OBS_ARRAY_LEN]; // nobs * 5 + if (is_obs_array) { + obs_arr.resize(obs_cnt*OBS_ARRAY_LEN, bad_data_float); // nobs * 5 + } else { - obs_ids = new int[obs_cnt]; // grib_code or var_id - obs_hids = new int[obs_cnt]; - obs_qids = new int[obs_cnt]; - obs_lvls = new float[obs_cnt]; - obs_hgts = new float[obs_cnt]; - obs_vals = new float[obs_cnt]; + obs_ids.resize(obs_cnt, bad_data_int); // grib_code or var_id + obs_hids.resize(obs_cnt, bad_data_int); + obs_qids.resize(obs_cnt, bad_data_int); + obs_lvls.resize(obs_cnt, bad_data_float); + obs_hgts.resize(obs_cnt, bad_data_float); + obs_vals.resize(obs_cnt, bad_data_float); } } /////////////////////////////////////////////////////////////////////////////// void MetPointObsData::assign(MetPointObsData &o) { + clear(); obs_cnt = o.obs_cnt; is_obs_array = o.is_obs_array; - - clear(); - allocate(); - if (is_obs_array) - for (int idx=0; idx= obs_cnt) { + mlog << Error << "\n" << method_name + << "index value (" << index << ") out of range for " + << obs_cnt << " observations.\n\n"; + exit(1); + } + if(obs_qids[index] < 0 || obs_qids[index] >= qty_names.n()) { + mlog << Error << "\n" << method_name + << "observation quality index (" << obs_qids[index] + << ") out of range for " << qty_names.n() + << " quality strings.\n\n"; + exit(1); + } + + return qty_names[(obs_qids[index])]; +} + +/////////////////////////////////////////////////////////////////////////////// + // struct MetPointHeader MetPointHeader::MetPointHeader() diff --git a/src/libcode/vx_nc_obs/met_point_data.h b/src/libcode/vx_nc_obs/met_point_data.h index 939c63ba35..83ba31e729 100644 --- a/src/libcode/vx_nc_obs/met_point_data.h +++ b/src/libcode/vx_nc_obs/met_point_data.h @@ -18,6 +18,7 @@ #include +#include #include "nc_utils.h" @@ -61,13 +62,13 @@ struct MetPointObsData { int obs_cnt; bool is_obs_array; - int *obs_ids; // grib_code or var_id - int *obs_hids; - int *obs_qids; - float *obs_lvls; - float *obs_hgts; - float *obs_vals; - float *obs_arr; // nobs * 5 + std::vector obs_ids; // grib_code or var_id + std::vector obs_hids; + std::vector obs_qids; + std::vector obs_lvls; + std::vector obs_hgts; + std::vector obs_vals; + std::vector obs_arr; // nobs * 5 StringArray var_names; StringArray qty_names; @@ -78,7 +79,8 @@ struct MetPointObsData { void clear_numbers(); void clear_strings(); bool fill_obs_buf(int buf_size, int offset, float *obs_arr, int *qty_idx_arr); - float get_obs_val(int index); + float get_obs_val(int index) const; + std::string get_obs_qty(int index) const; }; diff --git a/src/libcode/vx_nc_obs/nc_obs_util.cc b/src/libcode/vx_nc_obs/nc_obs_util.cc index 5380923944..d78d281549 100644 --- a/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -87,12 +87,15 @@ bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { clear_numbers(); obs_cnt = obs_vars.obs_cnt; + if (!IS_INVALID_NC(obs_vars.obs_arr_var)) is_obs_array = true; + + // Resize arrays for input data + allocate(); + StringArray missing_vars; StringArray failed_vars; - if (!IS_INVALID_NC(obs_vars.obs_arr_var)) { - is_obs_array = true; - obs_arr = new float[obs_cnt*OBS_ARRAY_LEN]; - if (!get_nc_data(&obs_vars.obs_arr_var, obs_arr)) { + if(is_obs_array) { + if (!get_nc_data(&obs_vars.obs_arr_var, obs_arr.data())) { succeed = false; failed_vars.add(nc_var_obs_arr); } @@ -103,8 +106,7 @@ bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { missing_vars.add(nc_var_obs_hid); } else { - obs_hids = new int[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_hid_var, obs_hids)) { + if (!get_nc_data(&obs_vars.obs_hid_var, obs_hids.data())) { succeed = false; failed_vars.add(nc_var_obs_hid); } @@ -114,8 +116,7 @@ bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { missing_vars.add(nc_var_obs_lvl); } else { - obs_lvls = new float[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_lvl_var, obs_lvls)) { + if (!get_nc_data(&obs_vars.obs_lvl_var, obs_lvls.data())) { succeed = false; failed_vars.add(nc_var_obs_lvl); } @@ -125,8 +126,7 @@ bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { missing_vars.add(nc_var_obs_hgt); } else { - obs_hgts = new float[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_hgt_var, obs_hgts)) { + if (!get_nc_data(&obs_vars.obs_hgt_var, obs_hgts.data())) { succeed = false; failed_vars.add(nc_var_obs_hgt); } @@ -136,28 +136,34 @@ bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { missing_vars.add(nc_var_obs_val); } else { - obs_vals = new float[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_val_var, obs_vals)) { + if (!get_nc_data(&obs_vars.obs_val_var, obs_vals.data())) { succeed = false; failed_vars.add(nc_var_obs_val); } } if (IS_VALID_NC(obs_vars.obs_gc_var)) { - obs_ids = new int[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_gc_var, obs_ids)) { + if (!get_nc_data(&obs_vars.obs_gc_var, obs_ids.data())) { succeed = false; failed_vars.add(nc_var_obs_gc); } } else if (IS_VALID_NC(obs_vars.obs_vid_var)) { - obs_ids = new int[obs_cnt]; - if (!get_nc_data(&obs_vars.obs_vid_var, obs_ids)) { + if (!get_nc_data(&obs_vars.obs_vid_var, obs_ids.data())) { succeed = false; failed_vars.add(nc_var_obs_vid); } } else succeed = false; - + if (IS_INVALID_NC(obs_vars.obs_qty_var)) { + succeed = false; + missing_vars.add(nc_var_obs_qty); + } + else { + if (!get_nc_data(&obs_vars.obs_qty_var, obs_qids.data())) { + succeed = false; + failed_vars.add(nc_var_obs_qty); + } + } } for (int idx=0; idxobs_qids); - set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); - set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); - set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); - set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); - set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); + set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids.data()); + set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids.data()); + set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids.data()); + set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls.data()); + set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts.data()); + set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals.data()); set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); diff --git a/src/libcode/vx_statistics/apply_mask.cc b/src/libcode/vx_statistics/apply_mask.cc index 01c696243c..c299d8d3d0 100644 --- a/src/libcode/vx_statistics/apply_mask.cc +++ b/src/libcode/vx_statistics/apply_mask.cc @@ -60,19 +60,19 @@ Grid parse_vx_grid(const RegridInfo info, const Grid *fgrid, const Grid *ogrid) // Verify on the forecast grid if(info.field == FieldType::Fcst) { - mlog << Debug(2) << "Using the forecast grid as the verification grid\n"; + mlog << Debug(2) << "Using the forecast grid as the verification grid.\n"; vx_grid = *fgrid; } // Verify on the observation grid else if(info.field == FieldType::Obs) { - mlog << Debug(2) << "Using the observation grid as the verification grid\n"; + mlog << Debug(2) << "Using the observation grid as the verification grid.\n"; vx_grid = *ogrid; } // Parse a named grid, grid specification string, // or gridded data file else { - mlog << Debug(2) << "Using named grid as the verification grid. " - << "Name=" << info.name << "\n"; + mlog << Debug(2) << "Using named grid (" << info.name + << ") as the verification grid.\n"; parse_grid_mask(info.name, vx_grid); } } diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index c243b226bc..e159649266 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -22,10 +22,10 @@ // 002 07-06-22 Howard Soh METplus-Internal #19 Rename main to met_main // 003 10-03-23 Prestopnik MET #2227 Remove namespace std and netCDF from header files // 004 05-20-24 Howard Soh MET #2867 Fix -qc ADP bug. +// 004 06-24-24 Halley Gotway MET #2880 Filter obs_quality. // //////////////////////////////////////////////////////////////////////// - #include #include #include @@ -55,7 +55,6 @@ using namespace std; using namespace netCDF; - //////////////////////////////////////////////////////////////////////// static ConcatString program_name; @@ -171,9 +170,8 @@ static void set_gaussian_dx(const StringArray &); static void set_gaussian_radius(const StringArray &); static unixtime compute_unixtime(NcVar *time_var, unixtime var_value); -void clear_cell_mapping(IntArray *cell_mapping); static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, - NcVar var_lat, NcVar var_lon, bool *skip_times); + NcVar var_lat, NcVar var_lon, vector skip_times); static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons); @@ -219,7 +217,7 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, static void save_geostationary_data(const ConcatString geostationary_file, const float *latitudes, const float *longitudes, const GoesImagerData &grid_data); -static void set_qc_flags(const StringArray &); +static void set_goes_qc_flags(const StringArray &); //////////////////////////////////////////////////////////////////////// @@ -274,7 +272,9 @@ static void process_command_line(int argc, char **argv) { cline.add(set_vld_thresh, "-vld_thresh", 1); cline.add(set_name, "-name", 1); cline.add(set_compress, "-compress", 1); - cline.add(set_qc_flags, "-qc", 1); + cline.add(set_goes_qc_flags, "-goes_qc", 1); + // Also support old -qc option + cline.add(set_goes_qc_flags, "-qc", 1); cline.add(set_adp, "-adp", 1); cline.add(set_config, "-config", 1); cline.add(set_prob_cat_thresh, "-prob_cat_thresh", 1); @@ -743,8 +743,8 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI bool empty_input = (nhdr == 0 && nobs == 0); bool use_var_id = met_point_obs->is_using_var_id(); - auto hdr_lats = new float[nhdr]; - auto hdr_lons = new float[nhdr]; + vector hdr_lats(nhdr, bad_data_float); + vector hdr_lons(nhdr, bad_data_float); IntArray var_index_array; IntArray valid_time_array; StringArray qc_tables = met_point_obs->get_qty_data(); @@ -752,8 +752,8 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI StringArray hdr_valid_times = header_data->vld_array; hdr_valid_times.sort(); - met_point_obs->get_lats(hdr_lats); - met_point_obs->get_lons(hdr_lons); + met_point_obs->get_lats(hdr_lats.data()); + met_point_obs->get_lons(hdr_lons.data()); // Check the message types prepare_message_types(header_data->typ_array); @@ -784,7 +784,7 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI int obs_count_zero_from = 0; int obs_count_non_zero_to = 0; int obs_count_non_zero_from = 0; - auto cellMapping = (IntArray *) nullptr; + vector cellMapping; for(int i=0; iobs_qids[idx])) { filtered_by_qc++; continue; } + // Filter by QC inclusion/exclusion lists (obs_quality_inc/exc config option) + if ((conf_info.obs_qty_inc.n() > 0 && + !conf_info.obs_qty_inc.has(obs_data->get_obs_qty(idx))) || + (conf_info.obs_qty_exc.n() > 0 && + conf_info.obs_qty_exc.has(obs_data->get_obs_qty(idx)))) { + filtered_by_qc++; + continue; + } + var_index_array.add(idx); var_count++; if (is_eq(obs_data->obs_vals[idx], (float)0.)) obs_count_zero_from++; @@ -968,14 +977,11 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI } } - if (cellMapping) { - clear_cell_mapping(cellMapping); - delete [] cellMapping; - cellMapping = (IntArray *) nullptr; - } - cellMapping = new IntArray[nx * ny]; - if( get_grid_mapping(to_grid, cellMapping, var_index_array, - obs_data->obs_hids, hdr_lats, hdr_lons) ) { + cellMapping.clear(); + cellMapping.resize(nx * ny); + if( get_grid_mapping(to_grid, cellMapping.data(), var_index_array, + obs_data->obs_hids.data(), + hdr_lats.data(), hdr_lons.data()) ) { int from_index; IntArray cellArray; NumArray dataArray; @@ -1122,30 +1128,35 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI << ", obs_count_non_zero_to: " << obs_count_non_zero_to << "\n"; ConcatString log_msg; - log_msg << "Filtered by time: " << filtered_by_time; + int filtered_count = filtered_by_msg_type + filtered_by_qc + filtered_by_time; + log_msg << "Filtered " << filtered_count << " of " << var_count2 + << " observations by time: " << filtered_by_time; if (0 < requested_valid_time) { log_msg << " [" << unix_to_yyyymmdd_hhmmss(requested_valid_time) << "]"; } - log_msg << ", by msg_type: " << filtered_by_msg_type; + log_msg << ", by message type: " << filtered_by_msg_type; if (0 < filtered_by_msg_type) { - log_msg << " ["; - for(int idx=0; idx 0) log_msg << ","; - log_msg << conf_info.message_type[idx]; - } - log_msg << "]"; + log_msg << " [" << write_css(conf_info.message_type) << "]"; } - log_msg << ", by QC: " << filtered_by_qc; + log_msg << ", and by QC: " << filtered_by_qc; if (0 < filtered_by_qc) { - log_msg << " ["; - for(int idx=0; idx 0) log_msg << ","; - log_msg << qc_flags[idx]; + if (0 < qc_flags.n()) { + log_msg << " [-qc " + << write_css(qc_flags) + << "]"; + } + if (0 < conf_info.obs_qty_inc.n()) { + log_msg << " [obs_quality_inc = " + << write_css(conf_info.obs_qty_inc) + << "]"; + } + if (0 < conf_info.obs_qty_exc.n()) { + log_msg << " [obs_quality_exc = " + << write_css(conf_info.obs_qty_exc) + << "]"; } - log_msg << "]"; } - log_msg << ", out of " << var_count2; - int filtered_count = filtered_by_msg_type + filtered_by_qc + (int)requested_valid_time; + log_msg << "."; if (0 == var_count) { if (0 == filtered_count) { mlog << Warning << "\n" << method_name @@ -1155,24 +1166,17 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI else { mlog << Warning << "\n" << method_name << "No valid data after filtering.\n\t" - << log_msg << ".\n\n"; + << log_msg << "\n\n"; } } else { - mlog << Debug(2) << method_name << "var_count=" << var_count - << ", grid: " << to_count << " out of " << (nx * ny) << " " - << (0 < filtered_count ? log_msg.c_str() : " ") << "\n"; + mlog << Debug(3) << "Using " << var_count << " " + << vinfo->name() << " observations to populate " << to_count + << " of " << to_grid.nxy() << " grid points.\n"; + if (0 < filtered_count ) mlog << log_msg << "\n"; } } // end for i - if (cellMapping) { - clear_cell_mapping(cellMapping); - delete [] cellMapping; cellMapping = (IntArray *) nullptr; - } - - delete [] hdr_lats; - delete [] hdr_lons; - return; } @@ -1256,8 +1260,7 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; unixtime valid_beg_ut, valid_end_ut; - bool *skip_times = nullptr; - double *valid_times = nullptr; + vector skip_times; int filtered_by_time = 0; int time_from_size = 1; clock_t start_clock = clock(); @@ -1348,9 +1351,9 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, if( 1 < get_dim_count(&time_var) ) { double max_time = bad_data_double; time_from_size = get_data_size(&time_var); - skip_times = new bool[time_from_size]; - valid_times = new double[time_from_size]; - if (get_nc_data(&time_var, valid_times)) { + skip_times.resize(time_from_size, false); + vector valid_times(time_from_size, bad_data_double); + if (get_nc_data(&time_var, valid_times.data())) { int sec_per_unit = 0; bool no_leap_year = false; auto ref_ut = (unixtime) 0; @@ -1365,7 +1368,7 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, tmp_time = add_to_unixtime(ref_ut, sec_per_unit, valid_times[i], no_leap_year); skip_times[i] = (valid_beg_ut > tmp_time || tmp_time > valid_end_ut); - if( skip_times[i]) filtered_by_time++; + if( skip_times[i] ) filtered_by_time++; } else skip_times[i] = false; if (max_time < valid_times[i]) max_time = valid_times[i]; @@ -1376,17 +1379,16 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, else valid_time = find_valid_time(time_var); } to_dp.set_size(to_grid.nx(), to_grid.ny()); - IntArray *var_cell_mapping = nullptr; - auto cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; - get_grid_mapping(fr_grid, to_grid, cellMapping, var_lat, var_lon, skip_times); - if( skip_times ) delete [] skip_times; - if( valid_times ) delete [] valid_times; + vector var_cell_mapping; + vector cellMapping(to_grid.nx() * to_grid.ny()); + get_grid_mapping(fr_grid, to_grid, cellMapping.data(), var_lat, var_lon, skip_times); // Loop through the requested fields for(int i=0; iclear(); + var_cell_mapping.clear(); // Populate the VarInfo object using the config string config.read_string(FieldSA[i].c_str()); @@ -1415,8 +1417,8 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, else if (is_nc_unit_longitude(units.c_str())) v_lon = nc_var; } } - var_cell_mapping = new IntArray[to_grid.nx() * to_grid.ny()]; - get_grid_mapping(fr_grid, to_grid, var_cell_mapping, v_lat, v_lon, skip_times); + var_cell_mapping.resize(to_grid.nx() * to_grid.ny()); + get_grid_mapping(fr_grid, to_grid, var_cell_mapping.data(), v_lat, v_lon, skip_times); mlog << Debug(4) << method_name << "Override cell mapping from " << GET_NC_NAME(v_lat) << " and " << GET_NC_NAME(v_lon) << "\n"; } @@ -1427,7 +1429,9 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, to_dp.set_init(valid_time); to_dp.set_valid(valid_time); regrid_nc_variable(nc_in, fr_mtddf, vinfo, fr_dp, to_dp, to_grid, - (nullptr != var_cell_mapping ? var_cell_mapping: cellMapping)); + (var_cell_mapping.size() > 0 ? + var_cell_mapping.data() : + cellMapping.data())); // List range of data values if(mlog.verbosity_level() >= 2) { @@ -1494,19 +1498,10 @@ static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, } } - if (nullptr != var_cell_mapping) { - clear_cell_mapping(var_cell_mapping); - delete [] var_cell_mapping; - var_cell_mapping = nullptr; - } - } // end for i - clear_cell_mapping(cellMapping); - delete [] cellMapping; - cellMapping = (IntArray *) nullptr; if( 0 < filtered_by_time ) { - mlog << Debug(2) << method_name << "Filtered by time: " << filtered_by_time + mlog << Debug(3) << method_name << "Filtered by time: " << filtered_by_time << " out of " << time_from_size << " [" << unix_to_yyyymmdd_hhmmss(valid_beg_ut) << " to " << unix_to_yyyymmdd_hhmmss(valid_end_ut) << "]\n"; @@ -1551,7 +1546,7 @@ static void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, int from_data_size = from_lat_count * from_lon_count; bool is_to_north = !fr_grid.get_swap_to_north(); - auto from_data = new float[from_data_size]; + vector from_data(from_data_size, bad_data_float); for (int xIdx=0; xIdx data(grid.nx()*grid.ny(), bad_data_float); // Store the data int grid_nx = grid.nx(); @@ -1706,15 +1699,12 @@ void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { } // end for x // Write out the data - if(!put_nc_data_with_dims(data_var, &data[0], grid.ny(), grid.nx())) { + if(!put_nc_data_with_dims(data_var, &data.data()[0], grid.ny(), grid.nx())) { mlog << Error << "\nwrite_nc_data() -> " << "error writing data to the output file.\n\n"; exit(1); } - // Clean up - if(data) { delete [] data; data = (float *) nullptr; } - return; } @@ -1723,7 +1713,7 @@ void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { void write_nc_data_int(const DataPlane &dp, const Grid &grid, NcVar *data_var) { // Allocate memory to store data values for each grid point - auto data = new int [grid.nx()*grid.ny()]; + vector data(grid.nx()*grid.ny(), bad_data_int); // Store the data int grid_nx = grid.nx(); @@ -1736,15 +1726,12 @@ void write_nc_data_int(const DataPlane &dp, const Grid &grid, NcVar *data_var) { } // end for x // Write out the data - if(!put_nc_data_with_dims(data_var, &data[0], grid.ny(), grid.nx())) { + if(!put_nc_data_with_dims(data_var, &data.data()[0], grid.ny(), grid.nx())) { mlog << Error << "\nwrite_nc_data_int() -> " << "error writing data to the output file.\n\n"; exit(1); } - // Clean up - if(data) { delete [] data; data = (int *) nullptr; } - return; } @@ -1792,11 +1779,10 @@ static void write_nc_int(const DataPlane &dp, const Grid &grid, return; } -// //////////////////////////////////////////////////////////////////////// +// // GOES related modules // - //////////////////////////////////////////////////////////////////////// static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, @@ -1830,8 +1816,8 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, NcVar time_var = get_nc_var_time(nc_in); unixtime valid_time = find_valid_time(time_var); to_dp.set_size(to_grid.nx(), to_grid.ny()); - auto cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; - get_grid_mapping(fr_grid, to_grid, cellMapping, geostationary_file); + vector cellMapping(to_grid.nx() * to_grid.ny()); + get_grid_mapping(fr_grid, to_grid, cellMapping.data(), geostationary_file); // Loop through the requested fields for(int i=0; i= 2) { @@ -1935,8 +1921,6 @@ static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, } delete nc_adp; nc_adp = nullptr; - clear_cell_mapping(cellMapping); - delete [] cellMapping; cellMapping = (IntArray *) nullptr; mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << get_exe_duration(start_clock) << " seconds\n"; @@ -2000,15 +1984,6 @@ GOES_QC compute_adp_qc_flag(int adp_qc, int shift_bits) { return adp_qc_flag; } - -//////////////////////////////////////////////////////////////////////// - -void clear_cell_mapping(IntArray *cell_mapping) { - if (nullptr != cell_mapping) { - for (int idx=0; idxn(); idx++) cell_mapping[idx].clear(); - } -} - //////////////////////////////////////////////////////////////////////// static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { @@ -2098,7 +2073,7 @@ static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, static void get_grid_mapping_latlon( DataPlane from_dp, DataPlane to_dp, Grid to_grid, IntArray *cellMapping, float *latitudes, float *longitudes, - int from_lat_count, int from_lon_count, bool *skip_times, bool to_north, bool is_2d) { + int from_lat_count, int from_lon_count, vector skip_times, bool to_north, bool is_2d) { double x; double y; double to_ll_lat; @@ -2111,10 +2086,8 @@ static void get_grid_mapping_latlon( int data_size = from_lat_count * from_lon_count; static const char *method_name = "get_grid_mapping_latlon(lats, lons) -> "; - auto to_cell_counts = new int[to_size]; - auto mapping_indices = new int[data_size]; - for (int xIdx=0; xIdx to_cell_counts(to_size, 0); + vector mapping_indices(data_size, bad_data_int); to_grid.xy_to_latlon(0, 0, to_ll_lat, to_ll_lon); mlog << Debug(5) << method_name << " to_grid ll corner: (" << to_ll_lon << ", " << to_ll_lat << ")\n"; @@ -2128,7 +2101,7 @@ static void get_grid_mapping_latlon( int coord_offset = from_dp.two_to_one(xIdx, yIdx, to_north); if (is_2d) { lon_offset = lat_offset = coord_offset; - if( skip_times != 0 && skip_times[coord_offset] ) continue; + if( skip_times.size() > 0 && skip_times[coord_offset] ) continue; } float lat = latitudes[lat_offset]; float lon = longitudes[lon_offset]; @@ -2190,8 +2163,6 @@ static void get_grid_mapping_latlon( } else cellMapping[to_offset].add(xIdx); } - delete [] to_cell_counts; - delete [] mapping_indices; mlog << Debug(3) << method_name << "within grid: " << count_in_grid << " out of " << data_size << " (" << 1.0*count_in_grid/data_size*100 << "%)\n"; @@ -2202,7 +2173,7 @@ static void get_grid_mapping_latlon( //////////////////////////////////////////////////////////////////////// static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, - NcVar var_lat, NcVar var_lon, bool *skip_times) { + NcVar var_lat, NcVar var_lon, vector skip_times) { bool status = false; DataPlane from_dp; DataPlane to_dp; @@ -2246,14 +2217,14 @@ static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray int last_idx = data_size - 1; int lat_count = get_data_size(&var_lat); int lon_count = get_data_size(&var_lon); - auto latitudes = new float[lat_count]; - auto longitudes = new float[lon_count]; - status = get_nc_data(&var_lat, latitudes); - if( status ) status = get_nc_data(&var_lon, longitudes); + vector latitudes(lat_count, bad_data_float); + vector longitudes(lon_count, bad_data_float); + status = get_nc_data(&var_lat, latitudes.data()); + if( status ) status = get_nc_data(&var_lon, longitudes.data()); if( status ) { get_grid_mapping_latlon(from_dp, to_dp, to_grid, cellMapping, - latitudes, longitudes, from_lat_count, - from_lon_count, skip_times, + latitudes.data(), longitudes.data(), + from_lat_count, from_lon_count, skip_times, !fr_grid.get_swap_to_north(), (lon_count==data_size)); if (is_eq(latitudes[0], latitudes[last_idx]) || @@ -2264,8 +2235,6 @@ static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray << longitudes[last_idx] << "\n\n"; } } - if( latitudes ) delete [] latitudes; - if( longitudes ) delete [] longitudes; } // if data_size > 0 mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << get_exe_duration(start_clock) << " seconds\n"; @@ -2376,22 +2345,14 @@ static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray if (data_size > 0) { int lat_count = data_size; int lon_count = data_size; - auto latitudes = (float *)nullptr; - auto longitudes = (float *)nullptr; - auto latitudes_buf = (float *)nullptr; - auto longitudes_buf = (float *)nullptr; - int buff_size = data_size*sizeof(float); + float *latitudes = nullptr; + float *longitudes = nullptr; + vector latitudes_buf(data_size, bad_data_float); + vector longitudes_buf(data_size, bad_data_float); GoesImagerData grid_data; grid_data.reset(); if (has_coord_input) { - latitudes_buf = new float[data_size]; - longitudes_buf = new float[data_size]; - - latitudes = latitudes_buf; - longitudes = longitudes_buf; - memset(latitudes, 0, buff_size); - memset(longitudes, 0, buff_size); if (IS_VALID_NC_P(coord_nc_in)) { NcVar var_lat = get_nc_var(coord_nc_in, var_name_lat); @@ -2399,12 +2360,16 @@ static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray if (IS_VALID_NC(var_lat) && IS_VALID_NC(var_lon)) { lat_count = get_data_size(&var_lat); lon_count = get_data_size(&var_lon); + latitudes = latitudes_buf.data(); + longitudes = longitudes_buf.data(); get_nc_data(&var_lat, latitudes); get_nc_data(&var_lon, longitudes); } } else { FILE *pFile = met_fopen ( cur_coord_name.c_str(), "rb" ); + latitudes = latitudes_buf.data(); + longitudes = longitudes_buf.data(); (void) fread (latitudes,sizeof(latitudes[0]),data_size,pFile); (void) fread (longitudes,sizeof(longitudes[0]),data_size,pFile); fclose (pFile); @@ -2465,24 +2430,23 @@ static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray latitudes, longitudes, grid_data); } } - if (nullptr == latitudes) { + if (latitudes == nullptr) { mlog << Error << "\n" << method_name << "Fail to get latitudes!\n\n"; } - else if (nullptr == longitudes) { + else if (longitudes == nullptr) { mlog << Error << "\n" << method_name << "Fail to get longitudes!\n\n"; } else { check_lat_lon(data_size, latitudes, longitudes); - get_grid_mapping_latlon(from_dp, to_dp, to_grid, cellMapping, latitudes, - longitudes, from_lat_count, from_lon_count, nullptr, + vector skip_times; + get_grid_mapping_latlon(from_dp, to_dp, to_grid, cellMapping, + latitudes, longitudes, + from_lat_count, from_lon_count, skip_times, !fr_grid.get_swap_to_north(), (lon_count==data_size)); } - if (latitudes_buf) delete [] latitudes_buf; - if (longitudes_buf) delete [] longitudes_buf; - grid_data.release(); } // if data_size > 0 @@ -2614,15 +2578,12 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, ConcatString goes_var_sub_name; ConcatString qc_var_name; uchar qc_value; - auto qc_data = new uchar[from_data_size]; - auto adp_data = new uchar[from_data_size]; - auto from_data = new float[from_data_size]; - auto adp_qc_data = new unsigned short[from_data_size]; + vector qc_data(from_data_size, -99); + vector adp_data(from_data_size, 1); + vector from_data(from_data_size, bad_data_float); + vector adp_qc_data(from_data_size, 255); static const char *method_name = "regrid_goes_variable() -> "; - // -99 is arbitrary number as invalid QC value - memset(qc_data, -99, from_data_size*sizeof(uchar)); - adp_qc_high = 3; /* 3 as baseline algorithm, 0 for enterprise algorithm */ adp_qc_medium = 1; /* 1 as baseline algorithm, 1 for enterprise algorithm */ adp_qc_low = 0; /* 0 as baseline algorithm, 2 for enterprise algorithm */ @@ -2637,8 +2598,6 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, auto actual_var_len = actual_var_name.length(); bool is_adp_variable = (0 != actual_var_name.compare(vinfo->name().c_str())); - memset(adp_data, 1, from_data_size*sizeof(uchar)); // Default: 1 = data present - memset(adp_qc_data, 255, from_data_size*sizeof(unsigned short)); if (is_adp_variable && IS_VALID_NC_P(nc_adp)) { is_dust_only = (0 == vinfo->name().comparecase((actual_var_len + 1), vname_dust.length(), vname_dust.c_str())); @@ -2649,13 +2608,13 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, else if (is_smoke_only) var_adp = get_goes_nc_var(nc_adp, vname_smoke); if (IS_VALID_NC(var_adp)) { - get_nc_data(&var_adp, adp_data, true); + get_nc_data(&var_adp, adp_data.data(), true); //ADP Smoke:ancillary_variables: ubyte DQF(y, x) if (get_att_value_string(&var_adp, (string)"ancillary_variables", qc_var_name)) { var_adp_qc = get_nc_var(nc_adp, qc_var_name.c_str()); if (IS_VALID_NC(var_adp_qc)) { - get_nc_data(&var_adp_qc, adp_qc_data); + get_nc_data(&var_adp_qc, adp_qc_data.data()); set_adp_gc_values(var_adp_qc); has_adp_qc_var = true; mlog << Debug(5) << method_name << "found QC var: " << qc_var_name @@ -2677,7 +2636,7 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, if (get_att_value_string(&var_data, (string)"ancillary_variables", qc_var_name)) { var_qc = get_nc_var(nc_in, qc_var_name.c_str()); if (IS_VALID_NC(var_qc)) { - get_nc_data(&var_qc, qc_data); + get_nc_data(&var_qc, qc_data.data()); has_qc_var = true; mlog << Debug(3) << method_name << "found QC var: " << qc_var_name << ".\n"; } @@ -2688,7 +2647,7 @@ static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, } } - get_nc_data(&var_data, from_data); + get_nc_data(&var_data, from_data.data()); fr_dp.set_size(from_lon_count, from_lat_count); for (int xIdx=0; xIdx 15) { - mlog << Warning << "\nPointToGridConfInfo::process_config() -> " - << "the \"" << conf_key_quality_mark_thresh - << "\" entry (" << quality_mark_thresh - << ") should be set between 0 and 15.\n\n"; - } + // Conf: obs_quality_exc + obs_qty_exc = parse_conf_obs_qty_exc(&conf); return; } diff --git a/src/tools/other/point2grid/point2grid_conf_info.h b/src/tools/other/point2grid/point2grid_conf_info.h index 5b12b6166e..725099b89c 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.h +++ b/src/tools/other/point2grid/point2grid_conf_info.h @@ -42,7 +42,8 @@ class PointToGridConfInfo { unixtime valid_time; // valid time int beg_ds; // Time range of observations to be retained, int end_ds; // Defined relative to the PrepBufr center time (seconds) - int quality_mark_thresh; // Quality marks to be retained + StringArray obs_qty_inc; // Quality mark strings to be included + StringArray obs_qty_exc; // Quality mark strings to be excluded ConcatString version; // Config file version PointToGridConfInfo();