Added utilities for pickling and unpickling filesystem objects
@@ -8,19 +8,24 @@ file in the root directory of this source tree.
"""
import os
import unittest.mock as mock
+from _thread import RLock
from pathlib import Path
+from typing import List
import pytest
from botocore.credentials import Credentials
+from fs.base import FS
from fs.errors import CreateFailed
+from fs.memoryfs import MemoryFS
from fs.osfs import OSFS
+from fs.tempfs import TempFS
from fs_s3fs import S3FS
from moto import mock_s3
from sentinelhub import SHConfig
from eolearn.core import get_filesystem, load_s3_filesystem
-from eolearn.core.utils.fs import get_aws_credentials, get_full_path, join_path
+from eolearn.core.utils.fs import get_aws_credentials, get_full_path, join_path, pickle_fs, unpickle_fs
def test_get_local_filesystem(tmp_path):
@@ -96,6 +101,29 @@ def test_get_aws_credentials(mocked_copy):
assert config.aws_secret_access_key != default_config.aws_secret_access_key
+@pytest.mark.parametrize(
+ "filesystem, compare_params",
+ [
+ (OSFS("."), ["root_path"]),
+ (TempFS(identifier="test"), ["identifier", "_temp_dir", "_auto_clean"]),
+ (MemoryFS(), []),
+ (
+ S3FS("s3://fake-bucket/", strict=False, acl="public-read"),
+ ["_bucket_name", "dir_path", "strict", "upload_args"],
+ ),
+ ],
+)
+def test_filesystem_serialization(filesystem: FS, compare_params: List[str]):
+ pickled_filesystem = pickle_fs(filesystem)
+ assert isinstance(pickled_filesystem, bytes)
+
+ unpickled_filesystem = unpickle_fs(pickled_filesystem)
+ assert filesystem is not unpickled_filesystem
+ assert isinstance(unpickled_filesystem._lock, RLock)
+ for param in compare_params:
+ assert getattr(filesystem, param) == getattr(unpickled_filesystem, param)
+
+
@pytest.mark.parametrize(
argnames="path_parts, expected_path",
ids=["local", "s3"],
