diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index dabca74..5dd4fef 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -25,7 +25,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: [macOS, ubuntu] - inlcude: + include: - os: macos-latest PLAT: arm64 INTERFACE64: "" diff --git a/continuous-integration/environment-actions.yml b/continuous-integration/environment-actions.yml index 1762953..22c9e81 100644 --- a/continuous-integration/environment-actions.yml +++ b/continuous-integration/environment-actions.yml @@ -12,5 +12,6 @@ dependencies: - matplotlib - dask - boto3 + - arm-test-data - distributed - xarray==2025.6.1 diff --git a/pyproject.toml b/pyproject.toml index 3775b71..8e9539e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "radclss" -version = "2026.3.3" +version = "2026.2.24" description = "Extracted Radar Columns and In Situ Sensors" readme = "README.md" requires-python = ">=3.9" @@ -19,7 +19,6 @@ dependencies = [ "act-atmos", "matplotlib", "dask", - "boto3", ] [tool.setuptools] diff --git a/src/radclss/config/default_config.py b/src/radclss/config/default_config.py index d4846bf..b3b9ae0 100644 --- a/src/radclss/config/default_config.py +++ b/src/radclss/config/default_config.py @@ -59,6 +59,24 @@ "signal_to_noise_ratio_copolar_h", "signal_to_noise_ratio_copolar_v", ], + "radar_csapr2": [ + "classification_mask", + "censor_mask", + "uncorrected_copol_correlation_coeff", + "uncorrected_differential_phase", + "uncorrected_differential_reflectivity", + "uncorrected_differential_reflectivity_lag_1", + "uncorrected_mean_doppler_velocity_h", + "uncorrected_mean_doppler_velocity_v", + "uncorrected_reflectivity_h", + "uncorrected_reflectivity_v", + "uncorrected_spectral_width_h", + "uncorrected_spectral_width_v", + "signal_to_noise_ratio_copolar_h", + "signal_to_noise_ratio_copolar_v", + "normalized_coherent_power", + "normalized_coherent_power_v", + ], "met": [ "base_time", "time_offset", diff --git a/src/radclss/config/output_config.py b/src/radclss/config/output_config.py index d67d834..427fbda 100644 --- a/src/radclss/config/output_config.py +++ b/src/radclss/config/output_config.py @@ -1,5 +1,5 @@ OUTPUT_SITE = "BNF" -OUTPUT_FACILITY = "S2" +OUTPUT_FACILITY = "M1" OUTPUT_PLATFORM = "radclss" OUTPUT_LEVEL = "c2" OUTPUT_GATE_TIME_ATTRS = { diff --git a/src/radclss/core/radclss_core.py b/src/radclss/core/radclss_core.py index 7fff09a..4ba2802 100644 --- a/src/radclss/core/radclss_core.py +++ b/src/radclss/core/radclss_core.py @@ -55,8 +55,8 @@ def radclss( time_coords : str The instrument to base the time coordinates off of, or an averaging interval in minutes or seconds. For example "radar_csapr2cmac" will use the CSAPR2 times as - the time coordinate for all of the data. "5Min" will average all data to a five - minute resolution using pandas date_range. + the time coordinate for all of the data. NEXRAD is currently not supported as a time coordinate, but can be used as a reference for reindexing. + If the specified time coordinate is not found in the volumes, then an error will be raised. serial : bool, optional Option to denote serial processing. Set to False to use dask cluster for subsetting columns in parallel. Default is True. @@ -90,6 +90,9 @@ def radclss( if discard_var == {}: discard_var = DEFAULT_DISCARD_VAR + if "sonde" not in volumes.keys(): + volumes["sonde"] = None + if verbose: print("=" * 80) print(f"RadCLss Processing for {volumes['date']}") @@ -114,7 +117,10 @@ def radclss( if verbose: print(f"Using {time_coords} as time basis") print(f"Number of {time_coords} files: {len(volumes[time_coords])}") - + else: + raise NotImplementedError( + "Currently, only radar-based time coordinates are supported. Please specify a radar key from the volumes dictionary as the time_coords argument." + ) # Call Subset Points columns = {} if verbose: @@ -241,18 +247,13 @@ def radclss( f" NEXRAD site: {nexrad_site if nexrad_site else 'auto-detect from ARM site'}" ) - if "radar" in time_coords or time_coords.lower() == "nexrad": + if "radar" in time_coords: time_list = sorted( [ str(x["base_time"].dt.strftime("%Y-%m-%dT%H:%M:%S").values[0]) for x in columns[time_coords] ] ) - else: - time_list = [ - x.strftime("%Y-%m-%dT%H:%M:%S") - for x in pd.date_range(min_time, max_time, time_coords) - ] if verbose: print(f" Number of NEXRAD time steps to fetch: {len(time_list)}") @@ -620,6 +621,8 @@ def _get_nexrad_wrapper(time_str): # Find all of the met stations and match to columns vol_keys = list(volumes.keys()) for k in vol_keys: + if k == "sonde": + continue if len(volumes[k]) == 0: if verbose: print(f"No files found for instrument/site: {k}") diff --git a/src/radclss/util/column_utils.py b/src/radclss/util/column_utils.py index 8b2f231..1e96f00 100644 --- a/src/radclss/util/column_utils.py +++ b/src/radclss/util/column_utils.py @@ -132,7 +132,7 @@ def subset_points( input_site_dict, sonde=None, height_bins=np.arange(500, 8500, 250), - rad_key="radar_cmaccsapr2", + rad_key="radar_csapr2", **kwargs, ): """ diff --git a/tests/test_radclss.py b/tests/test_radclss.py index 79c46dd..8aca8e1 100644 --- a/tests/test_radclss.py +++ b/tests/test_radclss.py @@ -21,7 +21,7 @@ def test_radclss_serial(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", + "bnfcsapr2cfrS3.a1", "2025-06-19T12:00:00", "2025-06-19T12:30:00", output=test_data_path, @@ -40,7 +40,7 @@ def test_radclss_serial(): if not os.path.exists(os.path.join(test_data_path, files)): arm_test_data.DATASETS.fetch(files) - rad_path = os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc") + rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) sonde_files = sorted( glob.glob(os.path.join(test_data_path, "*bnfsondewnpnM1.b1*cdf")) @@ -57,7 +57,7 @@ def test_radclss_serial(): ld_S30_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsS30.c1*nc")) volumes = { "date": "20250619", - "radar": radar_files, + "radar_csapr2": radar_files, "sonde": sonde_files, "vd_M1": vd_M1_files, "met_M1": met_M1_files, @@ -80,11 +80,11 @@ def test_radclss_serial(): } my_columns = radclss.core.radclss( - volumes, input_site_dict, serial=True, verbose=False + volumes, input_site_dict, "radar_csapr2", serial=True, verbose=False ) assert isinstance(my_columns, xr.Dataset) - assert "corrected_reflectivity" in my_columns.data_vars - assert my_columns.dims["time"] == 6 + assert "csapr2_reflectivity" in my_columns.data_vars + assert my_columns.dims["time"] == 11 assert my_columns.dims["height"] == 32 assert my_columns.dims["station"] == 6 assert np.array_equal( @@ -94,32 +94,12 @@ def test_radclss_serial(): # Radar and sonde data check for site in my_columns["station"].values: missing_value = ( - my_columns["corrected_reflectivity"] + my_columns["csapr2_reflectivity"] .sel(station=site) .attrs.get("missing_value", None) ) assert not ( - my_columns["corrected_reflectivity"].sel(station=site) == missing_value - ).all() - missing_value = ( - my_columns["attenuation_corrected_differential_reflectivity_lag_1"] - .sel(station=site) - .attrs.get("missing_value", None) - ) - assert not ( - my_columns["attenuation_corrected_differential_reflectivity_lag_1"].sel( - station=site - ) - == missing_value - ).all() - missing_value = ( - my_columns["corrected_specific_diff_phase"] - .sel(station=site) - .attrs.get("missing_value", None) - ) - assert not ( - my_columns["corrected_specific_diff_phase"].sel(station=site) - == missing_value + my_columns["csapr2_reflectivity"].sel(station=site) == missing_value ).all() for site in ["M1", "S4", "S20", "S30", "S40"]: @@ -152,33 +132,33 @@ def test_radclss_serial(): # Met data check for site in ["M1", "S20", "S30", "S40", "S13"]: missing_value = ( - my_columns["temp_mean"].sel(station=site).attrs.get("missing_value", None) + my_columns["temp_mean"].sel(station=site).attrs.get("_FillValue", None) ) assert not (my_columns["temp_mean"].sel(station=site) == missing_value).all() for site in ["S4"]: missing_value = ( - my_columns["temp_mean"].sel(station=site).attrs.get("missing_value", None) + my_columns["temp_mean"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["temp_mean"].sel(station=site) == missing_value).all() # Pluvio data check missing_value = ( - my_columns["accum_nrt"].sel(station="M1").attrs.get("missing_value", None) + my_columns["accum_nrt"].sel(station="M1").attrs.get("_FillValue", None) ) assert not (my_columns["accum_nrt"].sel(station="M1") == missing_value).all() missing_value = ( - my_columns["bucket_nrt"].sel(station="M1").attrs.get("missing_value", None) + my_columns["bucket_nrt"].sel(station="M1").attrs.get("_FillValue", None) ) assert not (my_columns["bucket_nrt"].sel(station="M1") == missing_value).all() for site in ["S20", "S30", "S40", "S13", "S4"]: missing_value = ( - my_columns["accum_nrt"].sel(station=site).attrs.get("missing_value", None) + my_columns["accum_nrt"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["accum_nrt"].sel(station=site) == missing_value).all() missing_value = ( - my_columns["bucket_nrt"].sel(station=site).attrs.get("missing_value", None) + my_columns["bucket_nrt"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["bucket_nrt"].sel(station=site) == missing_value).all() @@ -202,193 +182,6 @@ def test_radclss_serial(): ).all() -def test_radclss_serial_with_nexrad_time(): - """ - Test radclss in serial mode with time_coords set to 'nexrad'. - """ - test_data_path = arm_test_data.DATASETS.abspath - - # Before testing, ensure that ARM credentials are set in environment variables - username = os.getenv("ARM_USERNAME") - token = os.getenv("ARM_PASSWORD") - if not username or not token: - return # Skip test if credentials are not set - - act.discovery.download_arm_data( - username, - token, - "bnfcsapr2cmacS3.c1", - "2025-06-19T12:00:00", - "2025-06-19T12:30:00", - output=test_data_path, - ) - - act.discovery.download_arm_data( - username, - token, - "bnfsondewnpnM1.b1", - "2025-06-18T00:00:00", - "2025-06-20T00:00:00", - output=test_data_path, - ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) - - rad_path = os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc") - radar_files = sorted(glob.glob(rad_path)) - sonde_files = sorted( - glob.glob(os.path.join(test_data_path, "*bnfsondewnpnM1.b1*cdf")) - ) - - vd_M1_files = glob.glob(os.path.join(test_data_path, "*bnfvdisquantsM1.c1*nc")) - met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) - met_S20_files = glob.glob(os.path.join(test_data_path, "*bnfmetS20.b1*")) - met_S30_files = glob.glob(os.path.join(test_data_path, "*bnfmetS30.b1*")) - met_S40_files = glob.glob(os.path.join(test_data_path, "*bnfmetS40.b1*")) - wxt_S13_files = glob.glob(os.path.join(test_data_path, "*bnfmetwxtS13.b1*nc")) - pluvio_M1_files = glob.glob(os.path.join(test_data_path, "*bnfwbpluvio2M1.a1*nc")) - ld_M1_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsM1.c1*nc")) - ld_S30_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsS30.c1*nc")) - volumes = { - "date": "20250619", - "radar": radar_files, - "sonde": sonde_files, - "vd_M1": vd_M1_files, - "met_M1": met_M1_files, - "met_S20": met_S20_files, - "met_S30": met_S30_files, - "met_S40": met_S40_files, - "wxt_S13": wxt_S13_files, - "pluvio_M1": pluvio_M1_files, - "ld_M1": ld_M1_files, - "ld_S30": ld_S30_files, - } - - input_site_dict = { - "M1": (34.34525, -87.33842, 293), - "S4": (34.46451, -87.23598, 197), - "S20": (34.65401, -87.29264, 178), - "S30": (34.38501, -86.92757, 183), - "S40": (34.17932, -87.45349, 236), - "S13": (34.343889, -87.350556, 286), - } - - my_columns = radclss.core.radclss( - volumes, - input_site_dict, - serial=True, - verbose=False, - nexrad=True, - time_coords="nexrad", - ) - assert isinstance(my_columns, xr.Dataset) - assert "corrected_reflectivity" in my_columns.data_vars - assert my_columns.dims["station"] == 6 - assert np.array_equal( - my_columns["station"].values, ["M1", "S4", "S20", "S30", "S40", "S13"] - ) - - # Check that NEXRAD data exists - nexrad_vars = [ - var - for var in my_columns.data_vars - if "nexrad" in var.lower() or "khtx" in var.lower() - ] - assert len(nexrad_vars) > 0, "Expected NEXRAD variables in dataset" - - -def test_radclss_serial_with_5min_time(): - """ - Test radclss in serial mode with time_coords set to '5Min'. - """ - test_data_path = arm_test_data.DATASETS.abspath - - # Before testing, ensure that ARM credentials are set in environment variables - username = os.getenv("ARM_USERNAME") - token = os.getenv("ARM_PASSWORD") - if not username or not token: - return # Skip test if credentials are not set - - act.discovery.download_arm_data( - username, - token, - "bnfcsapr2cmacS3.c1", - "2025-06-19T12:00:00", - "2025-06-19T12:30:00", - output=test_data_path, - ) - - act.discovery.download_arm_data( - username, - token, - "bnfsondewnpnM1.b1", - "2025-06-18T00:00:00", - "2025-06-20T00:00:00", - output=test_data_path, - ) - for files in arm_test_data.DATASETS.registry.keys(): - if "bnf" in files: - if not os.path.exists(os.path.join(test_data_path, files)): - arm_test_data.DATASETS.fetch(files) - - rad_path = os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc") - radar_files = sorted(glob.glob(rad_path)) - sonde_files = sorted( - glob.glob(os.path.join(test_data_path, "*bnfsondewnpnM1.b1*cdf")) - ) - - vd_M1_files = glob.glob(os.path.join(test_data_path, "*bnfvdisquantsM1.c1*nc")) - met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) - met_S20_files = glob.glob(os.path.join(test_data_path, "*bnfmetS20.b1*")) - met_S30_files = glob.glob(os.path.join(test_data_path, "*bnfmetS30.b1*")) - met_S40_files = glob.glob(os.path.join(test_data_path, "*bnfmetS40.b1*")) - wxt_S13_files = glob.glob(os.path.join(test_data_path, "*bnfmetwxtS13.b1*nc")) - pluvio_M1_files = glob.glob(os.path.join(test_data_path, "*bnfwbpluvio2M1.a1*nc")) - ld_M1_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsM1.c1*nc")) - ld_S30_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsS30.c1*nc")) - volumes = { - "date": "20250619", - "radar": radar_files, - "sonde": sonde_files, - "vd_M1": vd_M1_files, - "met_M1": met_M1_files, - "met_S20": met_S20_files, - "met_S30": met_S30_files, - "met_S40": met_S40_files, - "wxt_S13": wxt_S13_files, - "pluvio_M1": pluvio_M1_files, - "ld_M1": ld_M1_files, - "ld_S30": ld_S30_files, - } - - input_site_dict = { - "M1": (34.34525, -87.33842, 293), - "S4": (34.46451, -87.23598, 197), - "S20": (34.65401, -87.29264, 178), - "S30": (34.38501, -86.92757, 183), - "S40": (34.17932, -87.45349, 236), - "S13": (34.343889, -87.350556, 286), - } - - my_columns = radclss.core.radclss( - volumes, input_site_dict, serial=True, verbose=False, time_coords="5Min" - ) - assert isinstance(my_columns, xr.Dataset) - assert "corrected_reflectivity" in my_columns.data_vars - assert my_columns.dims["station"] == 6 - assert np.array_equal( - my_columns["station"].values, ["M1", "S4", "S20", "S30", "S40", "S13"] - ) - - # With 5Min time_coords, we should have a different number of time steps - # The exact number will depend on the time range, but it should be > 1 - assert ( - my_columns.dims["time"] > 1 - ), "Expected multiple time steps with 5Min interval" - - def test_radclss_parallel(): test_data_path = arm_test_data.DATASETS.abspath @@ -401,7 +194,7 @@ def test_radclss_parallel(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", + "bnfcsapr2cfrS3.a1", "2025-06-19T12:00:00", "2025-06-19T12:30:00", output=test_data_path, @@ -420,12 +213,12 @@ def test_radclss_parallel(): if not os.path.exists(os.path.join(test_data_path, files)): arm_test_data.DATASETS.fetch(files) - rad_path = os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc") + rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) sonde_files = sorted( glob.glob(os.path.join(test_data_path, "*bnfsondewnpnM1.b1*cdf")) ) - print(sonde_files) + vd_M1_files = glob.glob(os.path.join(test_data_path, "*bnfvdisquantsM1.c1*nc")) met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) met_S20_files = glob.glob(os.path.join(test_data_path, "*bnfmetS20.b1*")) @@ -437,7 +230,7 @@ def test_radclss_parallel(): ld_S30_files = glob.glob(os.path.join(test_data_path, "*bnfldquantsS30.c1*nc")) volumes = { "date": "20250619", - "radar": radar_files, + "radar_csapr2": radar_files, "sonde": sonde_files, "vd_M1": vd_M1_files, "met_M1": met_M1_files, @@ -460,11 +253,11 @@ def test_radclss_parallel(): } with Client(LocalCluster(n_workers=2, threads_per_worker=1)) as client: # noqa my_columns = radclss.core.radclss( - volumes, input_site_dict, serial=False, verbose=False + volumes, input_site_dict, "radar_csapr2", serial=False, verbose=False ) assert isinstance(my_columns, xr.Dataset) - assert "corrected_reflectivity" in my_columns.data_vars - assert my_columns.dims["time"] == 6 + assert "csapr2_reflectivity" in my_columns.data_vars + assert my_columns.dims["time"] == 11 assert my_columns.dims["height"] == 32 assert my_columns.dims["station"] == 6 assert np.array_equal( @@ -474,32 +267,12 @@ def test_radclss_parallel(): # Radar and sonde data check for site in my_columns["station"].values: missing_value = ( - my_columns["corrected_reflectivity"] - .sel(station=site) - .attrs.get("missing_value", None) - ) - assert not ( - my_columns["corrected_reflectivity"].sel(station=site) == missing_value - ).all() - missing_value = ( - my_columns["attenuation_corrected_differential_reflectivity_lag_1"] - .sel(station=site) - .attrs.get("missing_value", None) - ) - assert not ( - my_columns["attenuation_corrected_differential_reflectivity_lag_1"].sel( - station=site - ) - == missing_value - ).all() - missing_value = ( - my_columns["corrected_specific_diff_phase"] + my_columns["csapr2_reflectivity"] .sel(station=site) .attrs.get("missing_value", None) ) assert not ( - my_columns["corrected_specific_diff_phase"].sel(station=site) - == missing_value + my_columns["csapr2_reflectivity"].sel(station=site) == missing_value ).all() for site in ["M1", "S4", "S20", "S30", "S40"]: @@ -532,33 +305,33 @@ def test_radclss_parallel(): # Met data check for site in ["M1", "S20", "S30", "S40", "S13"]: missing_value = ( - my_columns["temp_mean"].sel(station=site).attrs.get("missing_value", None) + my_columns["temp_mean"].sel(station=site).attrs.get("_FillValue", None) ) assert not (my_columns["temp_mean"].sel(station=site) == missing_value).all() for site in ["S4"]: missing_value = ( - my_columns["temp_mean"].sel(station=site).attrs.get("missing_value", None) + my_columns["temp_mean"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["temp_mean"].sel(station=site) == missing_value).all() # Pluvio data check missing_value = ( - my_columns["accum_nrt"].sel(station="M1").attrs.get("missing_value", None) + my_columns["accum_nrt"].sel(station="M1").attrs.get("_FillValue", None) ) assert not (my_columns["accum_nrt"].sel(station="M1") == missing_value).all() missing_value = ( - my_columns["bucket_nrt"].sel(station="M1").attrs.get("missing_value", None) + my_columns["bucket_nrt"].sel(station="M1").attrs.get("_FillValue", None) ) assert not (my_columns["bucket_nrt"].sel(station="M1") == missing_value).all() for site in ["S20", "S30", "S40", "S13", "S4"]: missing_value = ( - my_columns["accum_nrt"].sel(station=site).attrs.get("missing_value", None) + my_columns["accum_nrt"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["accum_nrt"].sel(station=site) == missing_value).all() missing_value = ( - my_columns["bucket_nrt"].sel(station=site).attrs.get("missing_value", None) + my_columns["bucket_nrt"].sel(station=site).attrs.get("_FillValue", None) ) assert (my_columns["bucket_nrt"].sel(station=site) == missing_value).all() @@ -593,7 +366,7 @@ def test_subset_points(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", + "bnfcsapr2cfrS3.a1", "2025-06-19T12:00:00", "2025-06-19T12:30:00", output=test_data_path, @@ -608,7 +381,7 @@ def test_subset_points(): output=test_data_path, ) - rad_path = os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc") + rad_path = os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc") radar_files = sorted(glob.glob(rad_path)) input_site_dict = { "M1": (34.34525, -87.33842, 293), @@ -616,7 +389,7 @@ def test_subset_points(): } subset_ds = radclss.util.subset_points(radar_files[0], input_site_dict, sonde=None) assert set(subset_ds["station"].values) == {"M1", "S30"} - assert "corrected_reflectivity" in subset_ds.data_vars + assert "reflectivity" in subset_ds.data_vars assert subset_ds.dims["station"] == 2 assert np.array_equal(subset_ds["height"].values, np.arange(500, 8500, 250)) assert "sonde_u_wind" not in subset_ds.data_vars @@ -634,7 +407,7 @@ def test_subset_points(): ) assert set(subset_ds["station"].values) == {"M1", "S30"} assert np.array_equal(subset_ds["height"].values, np.arange(500, 8500, 250)) - assert "corrected_reflectivity" in subset_ds.data_vars + assert "reflectivity" in subset_ds.data_vars assert "sonde_u_wind" in subset_ds.data_vars assert "sonde_v_wind" in subset_ds.data_vars assert "sonde_tdry" in subset_ds.data_vars @@ -661,9 +434,9 @@ def test_radclss_with_kasacr(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", - "2025-06-19T12:00:00", - "2025-06-19T12:30:00", + "bnfcsapr2cfrS3.a1", + "2025-06-01T12:00:00", + "2025-06-01T12:30:00", output=test_data_path, ) @@ -672,8 +445,8 @@ def test_radclss_with_kasacr(): username, token, "bnfkasacrcfrS4.a1", - "2025-06-19T12:00:00", - "2025-06-19T12:30:00", + "2025-06-01T12:00:00", + "2025-06-01T12:30:00", output=test_data_path, ) @@ -682,8 +455,8 @@ def test_radclss_with_kasacr(): username, token, "bnfsondewnpnM1.b1", - "2025-06-18T00:00:00", - "2025-06-20T00:00:00", + "2025-06-01T00:00:00", + "2025-06-02T00:00:00", output=test_data_path, ) @@ -695,7 +468,7 @@ def test_radclss_with_kasacr(): # Gather all the radar files csapr2_files = sorted( - glob.glob(os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc")) + glob.glob(os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc")) ) kasacr_files = sorted( glob.glob(os.path.join(test_data_path, "*bnfkasacrcfrS4.a1*.nc")) @@ -717,7 +490,7 @@ def test_radclss_with_kasacr(): volumes = { "date": "20250619", - "radar_csapr2cmac": csapr2_files, + "radar_csapr2": csapr2_files, "radar_kasacr": kasacr_files, "sonde": sonde_files, "vd_M1": vd_M1_files, @@ -741,7 +514,7 @@ def test_radclss_with_kasacr(): } my_columns = radclss.core.radclss( - volumes, input_site_dict, "radar_csapr2cmac", serial=True, verbose=False + volumes, input_site_dict, "radar_kasacr", serial=True, verbose=False ) # Basic structure checks @@ -753,10 +526,13 @@ def test_radclss_with_kasacr(): # Check that CSAPR2 data exists assert ( - "csapr2cmac_corrected_reflectivity" in my_columns.data_vars - or "corrected_reflectivity" in my_columns.data_vars + "csapr2_reflectivity" in my_columns.data_vars + or "reflectivity" in my_columns.data_vars ) + # Check that we are on KASACR timestamps + assert len(my_columns["time"].values) == 15, "Expected time values in dataset" + # Check that KASACR data exists if files were downloaded if len(kasacr_files) > 0: # Look for KASACR variables (they should have kasacr prefix) @@ -800,7 +576,7 @@ def test_radclss_parallel_with_nexrad(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", + "bnfcsapr2cfrS3.a1", "2025-06-19T12:00:00", "2025-06-19T12:30:00", output=test_data_path, @@ -808,12 +584,12 @@ def test_radclss_parallel_with_nexrad(): # Gather all the radar files csapr2_files = sorted( - glob.glob(os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc")) + glob.glob(os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc")) ) volumes = { "date": "20250619", - "radar_csapr2cmac": csapr2_files, + "radar_csapr2": csapr2_files, } input_site_dict = { @@ -830,7 +606,7 @@ def test_radclss_parallel_with_nexrad(): my_columns = radclss.core.radclss( volumes, input_site_dict, - "radar_csapr2cmac", + "radar_csapr2", serial=False, verbose=False, nexrad=True, @@ -892,7 +668,7 @@ def test_radclss_parallel_with_kasacr(): act.discovery.download_arm_data( username, token, - "bnfcsapr2cmacS3.c1", + "bnfcsapr2cfrS3.a1", "2025-06-19T12:00:00", "2025-06-19T12:30:00", output=test_data_path, @@ -926,7 +702,7 @@ def test_radclss_parallel_with_kasacr(): # Gather all the radar files csapr2_files = sorted( - glob.glob(os.path.join(test_data_path, "*bnfcsapr2cmacS3.c1*.nc")) + glob.glob(os.path.join(test_data_path, "*bnfcsapr2cfrS3.a1*.nc")) ) kasacr_files = sorted( glob.glob(os.path.join(test_data_path, "*bnfkasacrcfrS4.a1*.nc")) @@ -948,7 +724,7 @@ def test_radclss_parallel_with_kasacr(): volumes = { "date": "20250619", - "radar_csapr2cmac": csapr2_files, + "radar_csapr2": csapr2_files, "radar_kasacr": kasacr_files, "sonde": sonde_files, "vd_M1": vd_M1_files, @@ -974,7 +750,7 @@ def test_radclss_parallel_with_kasacr(): # Run radclss in parallel mode with Client(LocalCluster(n_workers=2, threads_per_worker=1)) as client: # noqa my_columns = radclss.core.radclss( - volumes, input_site_dict, "radar_csapr2cmac", serial=False, verbose=False + volumes, input_site_dict, "radar_csapr2", serial=False, verbose=False ) # Basic structure checks @@ -986,8 +762,8 @@ def test_radclss_parallel_with_kasacr(): # Check that CSAPR2 data exists assert ( - "csapr2cmac_corrected_reflectivity" in my_columns.data_vars - or "corrected_reflectivity" in my_columns.data_vars + "csapr2_reflectivity" in my_columns.data_vars + or "reflectivity" in my_columns.data_vars ) # Check that KASACR data exists if files were downloaded @@ -1028,7 +804,6 @@ def test_match_datasets_act(): arm_test_data.DATASETS.fetch(files) met_M1_files = glob.glob(os.path.join(test_data_path, "*bnfmetM1.b1*")) - print(met_M1_files) radclss_ds = xr.open_dataset(radclss_file) radclss_ds = radclss_ds.drop_vars( [var for var in radclss_ds.data_vars if "temp" in var]