refactor sequential_recipe fixture
@@ -207,10 +207,27 @@ def tmp_metadata_target(tmpdir_factory):
@pytest.fixture
def netCDFtoZarr_sequential_recipe(
- daily_xarray_dataset, netcdf_local_paths, tmp_target, tmp_cache, tmp_metadata_target
+ daily_xarray_dataset, netcdf_paths, tmp_target, tmp_cache, tmp_metadata_target
):
- paths, items_per_file = netcdf_local_paths
- file_pattern = pattern_from_file_sequence([str(path) for path in paths], "time", items_per_file)
+ paths, items_per_file = netcdf_paths[:2]
+
+ if len(netcdf_paths) == 2:
+ file_pattern = pattern_from_file_sequence(
+ [str(path) for path in paths], "time", items_per_file
+ )
+ else:
+ _, path_format = netcdf_paths[2:]
+ time_index = list(range(len(paths) // 2))
+
+ def format_function(variable, time):
+ return path_format.format(variable=variable, time=time)
+
+ file_pattern = FilePattern(
+ format_function,
+ ConcatDim("time", time_index, items_per_file),
+ MergeDim("variable", ["foo", "bar"]),
+ )
+
kwargs = dict(
inputs_per_chunk=1,
target=tmp_target,
@@ -222,9 +239,9 @@ def netCDFtoZarr_sequential_recipe(
@pytest.fixture
def netCDFtoZarr_sequential_subset_recipe(
- daily_xarray_dataset, netcdf_local_paths, tmp_target, tmp_cache, tmp_metadata_target
+ daily_xarray_dataset, netcdf_paths, tmp_target, tmp_cache, tmp_metadata_target
):
- paths, items_per_file = netcdf_local_paths
+ paths, items_per_file = netcdf_paths[:2]
if items_per_file != 2:
pytest.skip("This recipe only makes sense with items_per_file == 2.")
file_pattern = pattern_from_file_sequence([str(path) for path in paths], "time", items_per_file)
@@ -238,30 +255,6 @@ def netCDFtoZarr_sequential_subset_recipe(
return recipes.XarrayZarrRecipe, file_pattern, kwargs, daily_xarray_dataset, tmp_target
-@pytest.fixture
-def netCDFtoZarr_sequential_multi_variable_recipe(
- daily_xarray_dataset, netcdf_local_paths_by_variable, tmp_target, tmp_cache, tmp_metadata_target
-):
- paths, items_per_file, fnames_by_variable, path_format = netcdf_local_paths_by_variable
- time_index = list(range(len(paths) // 2))
-
- def format_function(variable, time):
- return path_format.format(variable=variable, time=time)
-
- file_pattern = FilePattern(
- format_function,
- ConcatDim("time", time_index, items_per_file),
- MergeDim("variable", ["foo", "bar"]),
- )
- kwargs = dict(
- inputs_per_chunk=1,
- target=tmp_target,
- input_cache=tmp_cache,
- metadata_cache=tmp_metadata_target,
- )
- return recipes.XarrayZarrRecipe, file_pattern, kwargs, daily_xarray_dataset, tmp_target
-
-
@pytest.fixture(scope="session")
def dask_cluster(request):
cluster = LocalCluster(n_workers=2, threads_per_worker=1, silence_logs=False)
