text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: datasync
using AWS.Compat
using AWS.UUIDs

"""
    cancel_task_execution(task_execution_arn)
    cancel_task_execution(task_execution_arn, params::Dict{String,<:Any})

Cancels execution of a task.  When you cancel a task execution, the transfer of some files
is abruptly interrupted. The contents of files that are transferred to the destination
might be incomplete or inconsistent with the source files. However, if you start a new task
execution on the same task and you allow the task execution to complete, file content on
the destination is complete and consistent. This applies to other unexpected failures that
interrupt a task execution. In all of these cases, DataSync successfully complete the
transfer when you start the next task execution.

# Arguments
- `task_execution_arn`: The Amazon Resource Name (ARN) of the task execution to cancel.

"""
function cancel_task_execution(
    TaskExecutionArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "CancelTaskExecution",
        Dict{String,Any}("TaskExecutionArn" => TaskExecutionArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function cancel_task_execution(
    TaskExecutionArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CancelTaskExecution",
        Dict{String,Any}(
            mergewith(
                _merge, Dict{String,Any}("TaskExecutionArn" => TaskExecutionArn), params
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_agent(activation_key)
    create_agent(activation_key, params::Dict{String,<:Any})

Activates an DataSync agent that you have deployed on your host. The activation process
associates your agent with your account. In the activation process, you specify information
such as the Amazon Web Services Region that you want to activate the agent in. You activate
the agent in the Amazon Web Services Region where your target locations (in Amazon S3 or
Amazon EFS) reside. Your tasks are created in this Amazon Web Services Region. You can
activate the agent in a VPC (virtual private cloud) or provide the agent access to a VPC
endpoint so you can run tasks without going over the public internet. You can use an agent
for more than one location. If a task uses multiple agents, all of them need to have status
AVAILABLE for the task to run. If you use multiple agents for a source location, the status
of all the agents must be AVAILABLE for the task to run.  Agents are automatically updated
by Amazon Web Services on a regular basis, using a mechanism that ensures minimal
interruption to your tasks.

# Arguments
- `activation_key`: Your agent activation key. You can get the activation key either by
  sending an HTTP GET request with redirects that enable you to get the agent IP address
  (port 80). Alternatively, you can get it from the DataSync console. The redirect URL
  returned in the response provides you the activation key for your agent in the query string
  parameter activationKey. It might also include other activation-related parameters;
  however, these are merely defaults. The arguments you pass to this API call determine the
  actual configuration of your agent. For more information, see Activating an Agent in the
  DataSync User Guide.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AgentName"`: The name you configured for your agent. This value is a text reference
  that is used to identify the agent in the console.
- `"SecurityGroupArns"`: The ARNs of the security groups used to protect your data transfer
  task subnets. See SecurityGroupArns.
- `"SubnetArns"`: The Amazon Resource Names (ARNs) of the subnets in which DataSync will
  create elastic network interfaces for each data transfer task. The agent that runs a task
  must be private. When you start a task that is associated with an agent created in a VPC,
  or one that has access to an IP address in a VPC, then the task is also private. In this
  case, DataSync creates four network interfaces for each task in your subnet. For a data
  transfer to work, the agent must be able to route to all these four network interfaces.
- `"Tags"`: The key-value pair that represents the tag that you want to associate with the
  agent. The value can be an empty string. This value helps you manage, filter, and search
  for your agents.  Valid characters for key and value are letters, spaces, and numbers
  representable in UTF-8 format, and the following special characters: + - = . _ : / @.
- `"VpcEndpointId"`: The ID of the VPC (virtual private cloud) endpoint that the agent has
  access to. This is the client-side VPC endpoint, also called a PrivateLink. If you don't
  have a PrivateLink VPC endpoint, see Creating a VPC Endpoint Service Configuration in the
  Amazon VPC User Guide. VPC endpoint ID looks like this: vpce-01234d5aff67890e1.
"""
function create_agent(ActivationKey; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "CreateAgent",
        Dict{String,Any}("ActivationKey" => ActivationKey);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_agent(
    ActivationKey,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateAgent",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("ActivationKey" => ActivationKey), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_efs(ec2_config, efs_filesystem_arn)
    create_location_efs(ec2_config, efs_filesystem_arn, params::Dict{String,<:Any})

Creates an endpoint for an Amazon EFS file system.

# Arguments
- `ec2_config`: The subnet and security group that the Amazon EFS file system uses. The
  security group that you provide needs to be able to communicate with the security group on
  the mount target in the subnet specified. The exact relationship between security group M
  (of the mount target) and security group S (which you provide for DataSync to use at this
  stage) is as follows:     Security group M (which you associate with the mount target) must
  allow inbound access for the Transmission Control Protocol (TCP) on the NFS port (2049)
  from security group S. You can enable inbound connections either by IP address (CIDR range)
  or security group.    Security group S (provided to DataSync to access EFS) should have a
  rule that enables outbound connections to the NFS port on one of the file system’s mount
  targets. You can enable outbound connections either by IP address (CIDR range) or security
  group. For information about security groups and mount targets, see Security Groups for
  Amazon EC2 Instances and Mount Targets in the Amazon EFS User Guide.
- `efs_filesystem_arn`: The Amazon Resource Name (ARN) for the Amazon EFS file system.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Subdirectory"`: A subdirectory in the location’s path. This subdirectory in the EFS
  file system is used to read data from the EFS source location or write data to the EFS
  destination. By default, DataSync uses the root directory.   Subdirectory must be specified
  with forward slashes. For example, /path/to/folder.
- `"Tags"`: The key-value pair that represents a tag that you want to add to the resource.
  The value can be an empty string. This value helps you manage, filter, and search for your
  resources. We recommend that you create a name tag for your location.
"""
function create_location_efs(
    Ec2Config, EfsFilesystemArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "CreateLocationEfs",
        Dict{String,Any}("Ec2Config" => Ec2Config, "EfsFilesystemArn" => EfsFilesystemArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_efs(
    Ec2Config,
    EfsFilesystemArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationEfs",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "Ec2Config" => Ec2Config, "EfsFilesystemArn" => EfsFilesystemArn
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_fsx_lustre(fsx_filesystem_arn, security_group_arns)
    create_location_fsx_lustre(fsx_filesystem_arn, security_group_arns, params::Dict{String,<:Any})

Creates an endpoint for an Amazon FSx for Lustre file system.

# Arguments
- `fsx_filesystem_arn`: The Amazon Resource Name (ARN) for the FSx for Lustre file system.
- `security_group_arns`: The Amazon Resource Names (ARNs) of the security groups that are
  used to configure the FSx for Lustre file system.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Subdirectory"`: A subdirectory in the location's path. This subdirectory in the FSx for
  Lustre file system is used to read data from the FSx for Lustre source location or write
  data to the FSx for Lustre destination.
- `"Tags"`: The key-value pair that represents a tag that you want to add to the resource.
  The value can be an empty string. This value helps you manage, filter, and search for your
  resources. We recommend that you create a name tag for your location.
"""
function create_location_fsx_lustre(
    FsxFilesystemArn, SecurityGroupArns; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "CreateLocationFsxLustre",
        Dict{String,Any}(
            "FsxFilesystemArn" => FsxFilesystemArn, "SecurityGroupArns" => SecurityGroupArns
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_fsx_lustre(
    FsxFilesystemArn,
    SecurityGroupArns,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationFsxLustre",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "FsxFilesystemArn" => FsxFilesystemArn,
                    "SecurityGroupArns" => SecurityGroupArns,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_fsx_windows(fsx_filesystem_arn, password, security_group_arns, user)
    create_location_fsx_windows(fsx_filesystem_arn, password, security_group_arns, user, params::Dict{String,<:Any})

Creates an endpoint for an Amazon FSx for Windows File Server file system.

# Arguments
- `fsx_filesystem_arn`: The Amazon Resource Name (ARN) for the FSx for Windows File Server
  file system.
- `password`: The password of the user who has the permissions to access files and folders
  in the FSx for Windows File Server file system.
- `security_group_arns`: The Amazon Resource Names (ARNs) of the security groups that are
  used to configure the FSx for Windows File Server file system.
- `user`: The user who has the permissions to access files and folders in the FSx for
  Windows File Server file system. For information about choosing a user name that ensures
  sufficient permissions to files, folders, and metadata, see user.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Domain"`: The name of the Windows domain that the FSx for Windows File Server belongs
  to.
- `"Subdirectory"`: A subdirectory in the location's path. This subdirectory in the Amazon
  FSx for Windows File Server file system is used to read data from the Amazon FSx for
  Windows File Server source location or write data to the FSx for Windows File Server
  destination.
- `"Tags"`: The key-value pair that represents a tag that you want to add to the resource.
  The value can be an empty string. This value helps you manage, filter, and search for your
  resources. We recommend that you create a name tag for your location.
"""
function create_location_fsx_windows(
    FsxFilesystemArn,
    Password,
    SecurityGroupArns,
    User;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationFsxWindows",
        Dict{String,Any}(
            "FsxFilesystemArn" => FsxFilesystemArn,
            "Password" => Password,
            "SecurityGroupArns" => SecurityGroupArns,
            "User" => User,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_fsx_windows(
    FsxFilesystemArn,
    Password,
    SecurityGroupArns,
    User,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationFsxWindows",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "FsxFilesystemArn" => FsxFilesystemArn,
                    "Password" => Password,
                    "SecurityGroupArns" => SecurityGroupArns,
                    "User" => User,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_hdfs(agent_arns, authentication_type, name_nodes)
    create_location_hdfs(agent_arns, authentication_type, name_nodes, params::Dict{String,<:Any})

Creates an endpoint for a Hadoop Distributed File System (HDFS).

# Arguments
- `agent_arns`: The Amazon Resource Names (ARNs) of the agents that are used to connect to
  the HDFS cluster.
- `authentication_type`: The type of authentication used to determine the identity of the
  user.
- `name_nodes`: The NameNode that manages the HDFS namespace. The NameNode performs
  operations such as opening, closing, and renaming files and directories. The NameNode
  contains the information to map blocks of data to the DataNodes. You can use only one
  NameNode.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"BlockSize"`: The size of data blocks to write into the HDFS cluster. The block size
  must be a multiple of 512 bytes. The default block size is 128 mebibytes (MiB).
- `"KerberosKeytab"`: The Kerberos key table (keytab) that contains mappings between the
  defined Kerberos principal and the encrypted keys. You can load the keytab from a file by
  providing the file's address. If you're using the CLI, it performs base64 encoding for you.
  Otherwise, provide the base64-encoded text.   If KERBEROS is specified for
  AuthenticationType, this parameter is required.
- `"KerberosKrb5Conf"`: The krb5.conf file that contains the Kerberos configuration
  information. You can load the krb5.conf file by providing the file's address. If you're
  using the CLI, it performs the base64 encoding for you. Otherwise, provide the
  base64-encoded text.   If KERBEROS is specified for AuthenticationType, this parameter is
  required.
- `"KerberosPrincipal"`: The Kerberos principal with access to the files and folders on the
  HDFS cluster.   If KERBEROS is specified for AuthenticationType, this parameter is
  required.
- `"KmsKeyProviderUri"`: The URI of the HDFS cluster's Key Management Server (KMS).
- `"QopConfiguration"`: The Quality of Protection (QOP) configuration specifies the Remote
  Procedure Call (RPC) and data transfer protection settings configured on the Hadoop
  Distributed File System (HDFS) cluster. If QopConfiguration isn't specified, RpcProtection
  and DataTransferProtection default to PRIVACY. If you set RpcProtection or
  DataTransferProtection, the other parameter assumes the same value.
- `"ReplicationFactor"`: The number of DataNodes to replicate the data to when writing to
  the HDFS cluster. By default, data is replicated to three DataNodes.
- `"SimpleUser"`: The user name used to identify the client on the host operating system.
  If SIMPLE is specified for AuthenticationType, this parameter is required.
- `"Subdirectory"`: A subdirectory in the HDFS cluster. This subdirectory is used to read
  data from or write data to the HDFS cluster. If the subdirectory isn't specified, it will
  default to /.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  location. The value can be an empty string. We recommend using tags to name your resources.
"""
function create_location_hdfs(
    AgentArns,
    AuthenticationType,
    NameNodes;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationHdfs",
        Dict{String,Any}(
            "AgentArns" => AgentArns,
            "AuthenticationType" => AuthenticationType,
            "NameNodes" => NameNodes,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_hdfs(
    AgentArns,
    AuthenticationType,
    NameNodes,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationHdfs",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AgentArns" => AgentArns,
                    "AuthenticationType" => AuthenticationType,
                    "NameNodes" => NameNodes,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_nfs(on_prem_config, server_hostname, subdirectory)
    create_location_nfs(on_prem_config, server_hostname, subdirectory, params::Dict{String,<:Any})

Defines a file system on a Network File System (NFS) server that can be read from or
written to.

# Arguments
- `on_prem_config`: Contains a list of Amazon Resource Names (ARNs) of agents that are used
  to connect to an NFS server.  If you are copying data to or from your Snowcone device, see
  NFS Server on Snowcone for more information.
- `server_hostname`: The name of the NFS server. This value is the IP address or Domain
  Name Service (DNS) name of the NFS server. An agent that is installed on-premises uses this
  host name to mount the NFS server in a network.  If you are copying data to or from your
  Snowcone device, see NFS Server on Snowcone for more information.  This name must either be
  DNS-compliant or must be an IP version 4 (IPv4) address.
- `subdirectory`: The subdirectory in the NFS file system that is used to read data from
  the NFS source location or write data to the NFS destination. The NFS path should be a path
  that's exported by the NFS server, or a subdirectory of that path. The path should be such
  that it can be mounted by other NFS clients in your network.  To see all the paths exported
  by your NFS server, run \"showmount -e nfs-server-name\" from an NFS client that has access
  to your server. You can specify any directory that appears in the results, and any
  subdirectory of that directory. Ensure that the NFS export is accessible without Kerberos
  authentication.  To transfer all the data in the folder you specified, DataSync needs to
  have permissions to read all the data. To ensure this, either configure the NFS export with
  no_root_squash, or ensure that the permissions for all of the files that you want DataSync
  allow read access for all users. Doing either enables the agent to read the files. For the
  agent to access directories, you must additionally enable all execute access. If you are
  copying data to or from your Snowcone device, see NFS Server on Snowcone for more
  information. For information about NFS export configuration, see 18.7. The /etc/exports
  Configuration File in the Red Hat Enterprise Linux documentation.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MountOptions"`: The NFS mount options that DataSync can use to mount your NFS share.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  location. The value can be an empty string. We recommend using tags to name your resources.
"""
function create_location_nfs(
    OnPremConfig,
    ServerHostname,
    Subdirectory;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationNfs",
        Dict{String,Any}(
            "OnPremConfig" => OnPremConfig,
            "ServerHostname" => ServerHostname,
            "Subdirectory" => Subdirectory,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_nfs(
    OnPremConfig,
    ServerHostname,
    Subdirectory,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationNfs",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "OnPremConfig" => OnPremConfig,
                    "ServerHostname" => ServerHostname,
                    "Subdirectory" => Subdirectory,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_object_storage(agent_arns, bucket_name, server_hostname)
    create_location_object_storage(agent_arns, bucket_name, server_hostname, params::Dict{String,<:Any})

Creates an endpoint for a self-managed object storage bucket. For more information about
self-managed object storage locations, see Creating a location for object storage.

# Arguments
- `agent_arns`: The Amazon Resource Name (ARN) of the agents associated with the
  self-managed object storage server location.
- `bucket_name`: The bucket on the self-managed object storage server that is used to read
  data from.
- `server_hostname`: The name of the self-managed object storage server. This value is the
  IP address or Domain Name Service (DNS) name of the object storage server. An agent uses
  this host name to mount the object storage server in a network.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AccessKey"`: Optional. The access key is used if credentials are required to access the
  self-managed object storage server. If your object storage requires a user name and
  password to authenticate, use AccessKey and SecretKey to provide the user name and
  password, respectively.
- `"SecretKey"`: Optional. The secret key is used if credentials are required to access the
  self-managed object storage server. If your object storage requires a user name and
  password to authenticate, use AccessKey and SecretKey to provide the user name and
  password, respectively.
- `"ServerPort"`: The port that your self-managed object storage server accepts inbound
  network traffic on. The server port is set by default to TCP 80 (HTTP) or TCP 443 (HTTPS).
  You can specify a custom port if your self-managed object storage server requires one.
- `"ServerProtocol"`: The protocol that the object storage server uses to communicate.
  Valid values are HTTP or HTTPS.
- `"Subdirectory"`: The subdirectory in the self-managed object storage server that is used
  to read data from.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  location. The value can be an empty string. We recommend using tags to name your resources.
"""
function create_location_object_storage(
    AgentArns, BucketName, ServerHostname; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "CreateLocationObjectStorage",
        Dict{String,Any}(
            "AgentArns" => AgentArns,
            "BucketName" => BucketName,
            "ServerHostname" => ServerHostname,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_object_storage(
    AgentArns,
    BucketName,
    ServerHostname,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationObjectStorage",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AgentArns" => AgentArns,
                    "BucketName" => BucketName,
                    "ServerHostname" => ServerHostname,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_s3(s3_bucket_arn, s3_config)
    create_location_s3(s3_bucket_arn, s3_config, params::Dict{String,<:Any})

Creates an endpoint for an Amazon S3 bucket. For more information, see Create an Amazon S3
location in the DataSync User Guide.

# Arguments
- `s3_bucket_arn`: The ARN of the Amazon S3 bucket. If the bucket is on an Amazon Web
  Services Outpost, this must be an access point ARN.
- `s3_config`:

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AgentArns"`: If you are using DataSync on an Amazon Web Services Outpost, specify the
  Amazon Resource Names (ARNs) of the DataSync agents deployed on your Outpost. For more
  information about launching a DataSync agent on an Amazon Web Services Outpost, see Deploy
  your DataSync agent on Outposts.
- `"S3StorageClass"`: The Amazon S3 storage class that you want to store your files in when
  this location is used as a task destination. For buckets in Amazon Web Services Regions,
  the storage class defaults to Standard. For buckets on Outposts, the storage class defaults
  to Amazon Web Services S3 Outposts. For more information about S3 storage classes, see
  Amazon S3 Storage Classes. Some storage classes have behaviors that can affect your S3
  storage cost. For detailed information, see Considerations when working with S3 storage
  classes in DataSync.
- `"Subdirectory"`: A subdirectory in the Amazon S3 bucket. This subdirectory in Amazon S3
  is used to read data from the S3 source location or write data to the S3 destination.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  location. The value can be an empty string. We recommend using tags to name your resources.
"""
function create_location_s3(
    S3BucketArn, S3Config; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "CreateLocationS3",
        Dict{String,Any}("S3BucketArn" => S3BucketArn, "S3Config" => S3Config);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_s3(
    S3BucketArn,
    S3Config,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationS3",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("S3BucketArn" => S3BucketArn, "S3Config" => S3Config),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_location_smb(agent_arns, password, server_hostname, subdirectory, user)
    create_location_smb(agent_arns, password, server_hostname, subdirectory, user, params::Dict{String,<:Any})

Defines a file system on a Server Message Block (SMB) server that can be read from or
written to.

# Arguments
- `agent_arns`: The Amazon Resource Names (ARNs) of agents to use for a Simple Message
  Block (SMB) location.
- `password`: The password of the user who can mount the share, has the permissions to
  access files and folders in the SMB share.
- `server_hostname`: The name of the SMB server. This value is the IP address or Domain
  Name Service (DNS) name of the SMB server. An agent that is installed on-premises uses this
  hostname to mount the SMB server in a network.  This name must either be DNS-compliant or
  must be an IP version 4 (IPv4) address.
- `subdirectory`: The subdirectory in the SMB file system that is used to read data from
  the SMB source location or write data to the SMB destination. The SMB path should be a path
  that's exported by the SMB server, or a subdirectory of that path. The path should be such
  that it can be mounted by other SMB clients in your network.   Subdirectory must be
  specified with forward slashes. For example, /path/to/folder.  To transfer all the data in
  the folder you specified, DataSync needs to have permissions to mount the SMB share, as
  well as to access all the data in that share. To ensure this, either ensure that the
  user/password specified belongs to the user who can mount the share, and who has the
  appropriate permissions for all of the files and directories that you want DataSync to
  access, or use credentials of a member of the Backup Operators group to mount the share.
  Doing either enables the agent to access the data. For the agent to access directories, you
  must additionally enable all execute access.
- `user`: The user who can mount the share, has the permissions to access files and folders
  in the SMB share. For information about choosing a user name that ensures sufficient
  permissions to files, folders, and metadata, see user.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Domain"`: The name of the Windows domain that the SMB server belongs to.
- `"MountOptions"`: The mount options used by DataSync to access the SMB server.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  location. The value can be an empty string. We recommend using tags to name your resources.
"""
function create_location_smb(
    AgentArns,
    Password,
    ServerHostname,
    Subdirectory,
    User;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationSmb",
        Dict{String,Any}(
            "AgentArns" => AgentArns,
            "Password" => Password,
            "ServerHostname" => ServerHostname,
            "Subdirectory" => Subdirectory,
            "User" => User,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_location_smb(
    AgentArns,
    Password,
    ServerHostname,
    Subdirectory,
    User,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateLocationSmb",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AgentArns" => AgentArns,
                    "Password" => Password,
                    "ServerHostname" => ServerHostname,
                    "Subdirectory" => Subdirectory,
                    "User" => User,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_task(destination_location_arn, source_location_arn)
    create_task(destination_location_arn, source_location_arn, params::Dict{String,<:Any})

Creates a task. A task includes a source location and a destination location, and a
configuration that specifies how data is transferred. A task always transfers data from the
source location to the destination location. The configuration specifies options such as
task scheduling, bandwidth limits, etc. A task is the complete definition of a data
transfer. When you create a task that transfers data between Amazon Web Services services
in different Amazon Web Services Regions, one of the two locations that you specify must
reside in the Region where DataSync is being used. The other location must be specified in
a different Region. You can transfer data between commercial Amazon Web Services Regions
except for China, or between Amazon Web Services GovCloud (US) Regions.  When you use
DataSync to copy files or objects between Amazon Web Services Regions, you pay for data
transfer between Regions. This is billed as data transfer OUT from your source Region to
your destination Region. For more information, see Data Transfer pricing.

# Arguments
- `destination_location_arn`: The Amazon Resource Name (ARN) of an Amazon Web Services
  storage resource's location.
- `source_location_arn`: The Amazon Resource Name (ARN) of the source location for the task.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"CloudWatchLogGroupArn"`: The Amazon Resource Name (ARN) of the Amazon CloudWatch log
  group that is used to monitor and log events in the task.
- `"Excludes"`: A list of filter rules that determines which files to exclude from a task.
  The list should contain a single filter string that consists of the patterns to exclude.
  The patterns are delimited by \"|\" (that is, a pipe), for example, \"/folder1|/folder2\".

- `"Includes"`: A list of filter rules that determines which files to include when running
  a task. The pattern contains a single filter string that consists of the patterns to
  include. The patterns are delimited by \"|\" (that is, a pipe), for example,
  \"/folder1|/folder2\".
- `"Name"`: The name of a task. This value is a text reference that is used to identify the
  task in the console.
- `"Options"`: The set of configuration options that control the behavior of a single
  execution of the task that occurs when you call StartTaskExecution. You can configure these
  options to preserve metadata such as user ID (UID) and group ID (GID), file permissions,
  data integrity verification, and so on. For each individual task execution, you can
  override these options by specifying the OverrideOptions before starting the task
  execution. For more information, see the StartTaskExecution operation.
- `"Schedule"`: Specifies a schedule used to periodically transfer files from a source to a
  destination location. The schedule should be specified in UTC time. For more information,
  see Scheduling your task.
- `"Tags"`: The key-value pair that represents the tag that you want to add to the
  resource. The value can be an empty string.
"""
function create_task(
    DestinationLocationArn,
    SourceLocationArn;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateTask",
        Dict{String,Any}(
            "DestinationLocationArn" => DestinationLocationArn,
            "SourceLocationArn" => SourceLocationArn,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_task(
    DestinationLocationArn,
    SourceLocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "CreateTask",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "DestinationLocationArn" => DestinationLocationArn,
                    "SourceLocationArn" => SourceLocationArn,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_agent(agent_arn)
    delete_agent(agent_arn, params::Dict{String,<:Any})

Deletes an agent. To specify which agent to delete, use the Amazon Resource Name (ARN) of
the agent in your request. The operation disassociates the agent from your Amazon Web
Services account. However, it doesn't delete the agent virtual machine (VM) from your
on-premises environment.

# Arguments
- `agent_arn`: The Amazon Resource Name (ARN) of the agent to delete. Use the ListAgents
  operation to return a list of agents for your account and Amazon Web Services Region.

"""
function delete_agent(AgentArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "DeleteAgent",
        Dict{String,Any}("AgentArn" => AgentArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_agent(
    AgentArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DeleteAgent",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("AgentArn" => AgentArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_location(location_arn)
    delete_location(location_arn, params::Dict{String,<:Any})

Deletes the configuration of a location used by DataSync.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the location to delete.

"""
function delete_location(LocationArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "DeleteLocation",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_location(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DeleteLocation",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_task(task_arn)
    delete_task(task_arn, params::Dict{String,<:Any})

Deletes a task.

# Arguments
- `task_arn`: The Amazon Resource Name (ARN) of the task to delete.

"""
function delete_task(TaskArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "DeleteTask",
        Dict{String,Any}("TaskArn" => TaskArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_task(
    TaskArn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DeleteTask",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("TaskArn" => TaskArn), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_agent(agent_arn)
    describe_agent(agent_arn, params::Dict{String,<:Any})

Returns metadata such as the name, the network interfaces, and the status (that is, whether
the agent is running or not) for an agent. To specify which agent to describe, use the
Amazon Resource Name (ARN) of the agent in your request.

# Arguments
- `agent_arn`: The Amazon Resource Name (ARN) of the agent to describe.

"""
function describe_agent(AgentArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "DescribeAgent",
        Dict{String,Any}("AgentArn" => AgentArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_agent(
    AgentArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeAgent",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("AgentArn" => AgentArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_efs(location_arn)
    describe_location_efs(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the path information about an Amazon EFS location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the EFS location to describe.

"""
function describe_location_efs(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationEfs",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_efs(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationEfs",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_fsx_lustre(location_arn)
    describe_location_fsx_lustre(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the path information about an Amazon FSx for Lustre location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the FSx for Lustre location to
  describe.

"""
function describe_location_fsx_lustre(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationFsxLustre",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_fsx_lustre(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationFsxLustre",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_fsx_windows(location_arn)
    describe_location_fsx_windows(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the path information about an Amazon FSx for Windows File Server
location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the FSx for Windows File Server
  location to describe.

"""
function describe_location_fsx_windows(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationFsxWindows",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_fsx_windows(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationFsxWindows",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_hdfs(location_arn)
    describe_location_hdfs(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the authentication information about the Hadoop Distributed File
System (HDFS) location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the HDFS cluster location to describe.

"""
function describe_location_hdfs(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationHdfs",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_hdfs(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationHdfs",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_nfs(location_arn)
    describe_location_nfs(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the path information, about an NFS location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the NFS location to describe.

"""
function describe_location_nfs(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationNfs",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_nfs(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationNfs",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_object_storage(location_arn)
    describe_location_object_storage(location_arn, params::Dict{String,<:Any})

Returns metadata about a self-managed object storage server location. For more information
about self-managed object storage locations, see Creating a location for object storage.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the self-managed object storage server
  location that was described.

"""
function describe_location_object_storage(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationObjectStorage",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_object_storage(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationObjectStorage",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_s3(location_arn)
    describe_location_s3(location_arn, params::Dict{String,<:Any})

Returns metadata, such as bucket name, about an Amazon S3 bucket location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the Amazon S3 bucket location to
  describe.

"""
function describe_location_s3(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationS3",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_s3(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationS3",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_location_smb(location_arn)
    describe_location_smb(location_arn, params::Dict{String,<:Any})

Returns metadata, such as the path and user information about an SMB location.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the SMB location to describe.

"""
function describe_location_smb(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeLocationSmb",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_location_smb(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeLocationSmb",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_task(task_arn)
    describe_task(task_arn, params::Dict{String,<:Any})

Returns metadata about a task.

# Arguments
- `task_arn`: The Amazon Resource Name (ARN) of the task to describe.

"""
function describe_task(TaskArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "DescribeTask",
        Dict{String,Any}("TaskArn" => TaskArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_task(
    TaskArn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeTask",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("TaskArn" => TaskArn), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_task_execution(task_execution_arn)
    describe_task_execution(task_execution_arn, params::Dict{String,<:Any})

Returns detailed metadata about a task that is being executed.

# Arguments
- `task_execution_arn`: The Amazon Resource Name (ARN) of the task that is being executed.

"""
function describe_task_execution(
    TaskExecutionArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "DescribeTaskExecution",
        Dict{String,Any}("TaskExecutionArn" => TaskExecutionArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_task_execution(
    TaskExecutionArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "DescribeTaskExecution",
        Dict{String,Any}(
            mergewith(
                _merge, Dict{String,Any}("TaskExecutionArn" => TaskExecutionArn), params
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_agents()
    list_agents(params::Dict{String,<:Any})

Returns a list of agents owned by an Amazon Web Services account in the Amazon Web Services
Region specified in the request. The returned list is ordered by agent Amazon Resource Name
(ARN). By default, this operation returns a maximum of 100 agents. This operation supports
pagination that enables you to optionally reduce the number of agents returned in a
response. If you have more agents than are returned in a response (that is, the response
returns only a truncated list of your agents), the response contains a marker that you can
specify in your next request to fetch the next page of agents.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The maximum number of agents to list.
- `"NextToken"`: An opaque string that indicates the position at which to begin the next
  list of agents.
"""
function list_agents(; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync("ListAgents"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
end
function list_agents(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "ListAgents", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_locations()
    list_locations(params::Dict{String,<:Any})

Returns a list of source and destination locations. If you have more locations than are
returned in a response (that is, the response returns only a truncated list of your
agents), the response contains a token that you can specify in your next request to fetch
the next page of locations.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Filters"`: You can use API filters to narrow down the list of resources returned by
  ListLocations. For example, to retrieve all tasks on a specific source location, you can
  use ListLocations with filter name LocationType S3 and Operator Equals.
- `"MaxResults"`: The maximum number of locations to return.
- `"NextToken"`: An opaque string that indicates the position at which to begin the next
  list of locations.
"""
function list_locations(; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync("ListLocations"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
end
function list_locations(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "ListLocations", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Returns all the tags associated with a specified resource.

# Arguments
- `resource_arn`: The Amazon Resource Name (ARN) of the resource whose tags to list.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The maximum number of locations to return.
- `"NextToken"`: An opaque string that indicates the position at which to begin the next
  list of locations.
"""
function list_tags_for_resource(
    ResourceArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "ListTagsForResource",
        Dict{String,Any}("ResourceArn" => ResourceArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_tags_for_resource(
    ResourceArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "ListTagsForResource",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("ResourceArn" => ResourceArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_task_executions()
    list_task_executions(params::Dict{String,<:Any})

Returns a list of executed tasks.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The maximum number of executed tasks to list.
- `"NextToken"`: An opaque string that indicates the position at which to begin the next
  list of the executed tasks.
- `"TaskArn"`: The Amazon Resource Name (ARN) of the task whose tasks you want to list.
"""
function list_task_executions(; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "ListTaskExecutions"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_task_executions(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "ListTaskExecutions", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_tasks()
    list_tasks(params::Dict{String,<:Any})

Returns a list of all the tasks.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Filters"`: You can use API filters to narrow down the list of resources returned by
  ListTasks. For example, to retrieve all tasks on a specific source location, you can use
  ListTasks with filter name LocationId and Operator Equals with the ARN for the location.
- `"MaxResults"`: The maximum number of tasks to return.
- `"NextToken"`: An opaque string that indicates the position at which to begin the next
  list of tasks.
"""
function list_tasks(; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync("ListTasks"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET)
end
function list_tasks(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "ListTasks", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    start_task_execution(task_arn)
    start_task_execution(task_arn, params::Dict{String,<:Any})

Starts a specific invocation of a task. A TaskExecution value represents an individual run
of a task. Each task can have at most one TaskExecution at a time.  TaskExecution has the
following transition phases: INITIALIZING | PREPARING | TRANSFERRING | VERIFYING |
SUCCESS/FAILURE.  For detailed information, see the Task Execution section in the
Components and Terminology topic in the DataSync User Guide.

# Arguments
- `task_arn`: The Amazon Resource Name (ARN) of the task to start.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Excludes"`: A list of filter rules that determines which files to exclude from a task.
  The list contains a single filter string that consists of the patterns to exclude. The
  patterns are delimited by \"|\" (that is, a pipe), for example, \"/folder1|/folder2\".
- `"Includes"`: A list of filter rules that determines which files to include when running
  a task. The pattern should contain a single filter string that consists of the patterns to
  include. The patterns are delimited by \"|\" (that is, a pipe), for example,
  \"/folder1|/folder2\".
- `"OverrideOptions"`:
"""
function start_task_execution(TaskArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "StartTaskExecution",
        Dict{String,Any}("TaskArn" => TaskArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function start_task_execution(
    TaskArn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "StartTaskExecution",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("TaskArn" => TaskArn), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Applies a key-value pair to an Amazon Web Services resource.

# Arguments
- `resource_arn`: The Amazon Resource Name (ARN) of the resource to apply the tag to.
- `tags`: The tags to apply.

"""
function tag_resource(ResourceArn, Tags; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "TagResource",
        Dict{String,Any}("ResourceArn" => ResourceArn, "Tags" => Tags);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function tag_resource(
    ResourceArn,
    Tags,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "TagResource",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("ResourceArn" => ResourceArn, "Tags" => Tags),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    untag_resource(keys, resource_arn)
    untag_resource(keys, resource_arn, params::Dict{String,<:Any})

Removes a tag from an Amazon Web Services resource.

# Arguments
- `keys`: The keys in the key-value pair in the tag to remove.
- `resource_arn`: The Amazon Resource Name (ARN) of the resource to remove the tag from.

"""
function untag_resource(
    Keys, ResourceArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "UntagResource",
        Dict{String,Any}("Keys" => Keys, "ResourceArn" => ResourceArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function untag_resource(
    Keys,
    ResourceArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UntagResource",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("Keys" => Keys, "ResourceArn" => ResourceArn),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_agent(agent_arn)
    update_agent(agent_arn, params::Dict{String,<:Any})

Updates the name of an agent.

# Arguments
- `agent_arn`: The Amazon Resource Name (ARN) of the agent to update.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Name"`: The name that you want to use to configure the agent.
"""
function update_agent(AgentArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "UpdateAgent",
        Dict{String,Any}("AgentArn" => AgentArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_agent(
    AgentArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateAgent",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("AgentArn" => AgentArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_location_hdfs(location_arn)
    update_location_hdfs(location_arn, params::Dict{String,<:Any})

Updates some parameters of a previously created location for a Hadoop Distributed File
System cluster.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the source HDFS cluster location.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AgentArns"`: The ARNs of the agents that are used to connect to the HDFS cluster.
- `"AuthenticationType"`: The type of authentication used to determine the identity of the
  user.
- `"BlockSize"`: The size of the data blocks to write into the HDFS cluster.
- `"KerberosKeytab"`: The Kerberos key table (keytab) that contains mappings between the
  defined Kerberos principal and the encrypted keys. You can load the keytab from a file by
  providing the file's address. If you use the AWS CLI, it performs base64 encoding for you.
  Otherwise, provide the base64-encoded text.
- `"KerberosKrb5Conf"`: The krb5.conf file that contains the Kerberos configuration
  information. You can load the krb5.conf file by providing the file's address. If you're
  using the AWS CLI, it performs the base64 encoding for you. Otherwise, provide the
  base64-encoded text.
- `"KerberosPrincipal"`: The Kerberos principal with access to the files and folders on the
  HDFS cluster.
- `"KmsKeyProviderUri"`: The URI of the HDFS cluster's Key Management Server (KMS).
- `"NameNodes"`: The NameNode that manages the HDFS namespace. The NameNode performs
  operations such as opening, closing, and renaming files and directories. The NameNode
  contains the information to map blocks of data to the DataNodes. You can use only one
  NameNode.
- `"QopConfiguration"`: The Quality of Protection (QOP) configuration specifies the Remote
  Procedure Call (RPC) and data transfer privacy settings configured on the Hadoop
  Distributed File System (HDFS) cluster.
- `"ReplicationFactor"`: The number of DataNodes to replicate the data to when writing to
  the HDFS cluster.
- `"SimpleUser"`: The user name used to identify the client on the host operating system.
- `"Subdirectory"`: A subdirectory in the HDFS cluster. This subdirectory is used to read
  data from or write data to the HDFS cluster.
"""
function update_location_hdfs(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "UpdateLocationHdfs",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_location_hdfs(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateLocationHdfs",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_location_nfs(location_arn)
    update_location_nfs(location_arn, params::Dict{String,<:Any})

Updates some of the parameters of a previously created location for Network File System
(NFS) access. For information about creating an NFS location, see Creating a location for
NFS.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the NFS location to update.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MountOptions"`:
- `"OnPremConfig"`:
- `"Subdirectory"`: The subdirectory in the NFS file system that is used to read data from
  the NFS source location or write data to the NFS destination. The NFS path should be a path
  that's exported by the NFS server, or a subdirectory of that path. The path should be such
  that it can be mounted by other NFS clients in your network. To see all the paths exported
  by your NFS server, run \"showmount -e nfs-server-name\" from an NFS client that has access
  to your server. You can specify any directory that appears in the results, and any
  subdirectory of that directory. Ensure that the NFS export is accessible without Kerberos
  authentication.  To transfer all the data in the folder that you specified, DataSync must
  have permissions to read all the data. To ensure this, either configure the NFS export with
  no_root_squash, or ensure that the files you want DataSync to access have permissions that
  allow read access for all users. Doing either option enables the agent to read the files.
  For the agent to access directories, you must additionally enable all execute access. If
  you are copying data to or from your Snowcone device, see NFS Server on Snowcone for more
  information. For information about NFS export configuration, see 18.7. The /etc/exports
  Configuration File in the Red Hat Enterprise Linux documentation.
"""
function update_location_nfs(LocationArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "UpdateLocationNfs",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_location_nfs(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateLocationNfs",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_location_object_storage(location_arn)
    update_location_object_storage(location_arn, params::Dict{String,<:Any})

Updates some of the parameters of a previously created location for self-managed object
storage server access. For information about creating a self-managed object storage
location, see Creating a location for object storage.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the self-managed object storage server
  location to be updated.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AccessKey"`: Optional. The access key is used if credentials are required to access the
  self-managed object storage server. If your object storage requires a user name and
  password to authenticate, use AccessKey and SecretKey to provide the user name and
  password, respectively.
- `"AgentArns"`: The Amazon Resource Name (ARN) of the agents associated with the
  self-managed object storage server location.
- `"SecretKey"`: Optional. The secret key is used if credentials are required to access the
  self-managed object storage server. If your object storage requires a user name and
  password to authenticate, use AccessKey and SecretKey to provide the user name and
  password, respectively.
- `"ServerPort"`: The port that your self-managed object storage server accepts inbound
  network traffic on. The server port is set by default to TCP 80 (HTTP) or TCP 443 (HTTPS).
  You can specify a custom port if your self-managed object storage server requires one.
- `"ServerProtocol"`: The protocol that the object storage server uses to communicate.
  Valid values are HTTP or HTTPS.
- `"Subdirectory"`: The subdirectory in the self-managed object storage server that is used
  to read data from.
"""
function update_location_object_storage(
    LocationArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "UpdateLocationObjectStorage",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_location_object_storage(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateLocationObjectStorage",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_location_smb(location_arn)
    update_location_smb(location_arn, params::Dict{String,<:Any})

Updates some of the parameters of a previously created location for Server Message Block
(SMB) file system access. For information about creating an SMB location, see Creating a
location for SMB.

# Arguments
- `location_arn`: The Amazon Resource Name (ARN) of the SMB location to update.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AgentArns"`: The Amazon Resource Names (ARNs) of agents to use for a Simple Message
  Block (SMB) location.
- `"Domain"`: The name of the Windows domain that the SMB server belongs to.
- `"MountOptions"`:
- `"Password"`: The password of the user who can mount the share has the permissions to
  access files and folders in the SMB share.
- `"Subdirectory"`: The subdirectory in the SMB file system that is used to read data from
  the SMB source location or write data to the SMB destination. The SMB path should be a path
  that's exported by the SMB server, or a subdirectory of that path. The path should be such
  that it can be mounted by other SMB clients in your network.   Subdirectory must be
  specified with forward slashes. For example, /path/to/folder.  To transfer all the data in
  the folder that you specified, DataSync must have permissions to mount the SMB share and to
  access all the data in that share. To ensure this, do either of the following:   Ensure
  that the user/password specified belongs to the user who can mount the share and who has
  the appropriate permissions for all of the files and directories that you want DataSync to
  access.   Use credentials of a member of the Backup Operators group to mount the share.
  Doing either of these options enables the agent to access the data. For the agent to access
  directories, you must also enable all execute access.
- `"User"`: The user who can mount the share has the permissions to access files and
  folders in the SMB share.
"""
function update_location_smb(LocationArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "UpdateLocationSmb",
        Dict{String,Any}("LocationArn" => LocationArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_location_smb(
    LocationArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateLocationSmb",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("LocationArn" => LocationArn), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_task(task_arn)
    update_task(task_arn, params::Dict{String,<:Any})

Updates the metadata associated with a task.

# Arguments
- `task_arn`: The Amazon Resource Name (ARN) of the resource name of the task to update.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"CloudWatchLogGroupArn"`: The Amazon Resource Name (ARN) of the resource name of the
  Amazon CloudWatch log group.
- `"Excludes"`: A list of filter rules that determines which files to exclude from a task.
  The list should contain a single filter string that consists of the patterns to exclude.
  The patterns are delimited by \"|\" (that is, a pipe), for example, \"/folder1|/folder2\".
- `"Includes"`: A list of filter rules that determines which files to include when running
  a task. The pattern contains a single filter string that consists of the patterns to
  include. The patterns are delimited by \"|\" (that is, a pipe), for example,
  \"/folder1|/folder2\".
- `"Name"`: The name of the task to update.
- `"Options"`:
- `"Schedule"`: Specifies a schedule used to periodically transfer files from a source to a
  destination location. You can configure your task to execute hourly, daily, weekly or on
  specific days of the week. You control when in the day or hour you want the task to
  execute. The time you specify is UTC time. For more information, see Scheduling your task.
"""
function update_task(TaskArn; aws_config::AbstractAWSConfig=global_aws_config())
    return datasync(
        "UpdateTask",
        Dict{String,Any}("TaskArn" => TaskArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_task(
    TaskArn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "UpdateTask",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("TaskArn" => TaskArn), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_task_execution(options, task_execution_arn)
    update_task_execution(options, task_execution_arn, params::Dict{String,<:Any})

Updates execution of a task. You can modify bandwidth throttling for a task execution that
is running or queued. For more information, see Adjusting Bandwidth Throttling for a Task
Execution.  The only Option that can be modified by UpdateTaskExecution is  BytesPerSecond
.

# Arguments
- `options`:
- `task_execution_arn`: The Amazon Resource Name (ARN) of the specific task execution that
  is being updated.

"""
function update_task_execution(
    Options, TaskExecutionArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return datasync(
        "UpdateTaskExecution",
        Dict{String,Any}("Options" => Options, "TaskExecutionArn" => TaskExecutionArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_task_execution(
    Options,
    TaskExecutionArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return datasync(
        "UpdateTaskExecution",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "Options" => Options, "TaskExecutionArn" => TaskExecutionArn
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end


text:
# helper function
# parses the name and returns it with or without extension
parse_name(filename, ext::AbstractString, ::Val{false}) = first(rsplit(filename, ext, limit=2))

function parse_name(filename, ext::Regex, ::Val{false})
    idxs = findall(ext, filename)
    return filename[1:first(last(idxs)) - 1]
end

parse_name(filename, ext, ::Val{true}) = filename

# utility function for generating filename
function generate_filename(path, save_location, save_prefix, save_suffix, save_delim, ext)
    # get the filename
    filename = basename(path)

    # splitting name and extension
    modified_name, extension = parse_name_ext(filename, "." * ext)

    # adding prefix and suffix with delimiter
    if !isnothing(save_prefix)
        modified_name = string(save_prefix, save_delim, modified_name)
    end
    if !isnothing(save_suffix)
        modified_name = string(modified_name, save_delim, save_suffix)
    end

    # adding extension to modified_name
    file_path = joinpath(save_location, modified_name * extension)
    return file_path
end

# utility function to return filename and extension separately
# returns extension including "." at the beginning
function parse_name_ext(filename, ext)
    idxs = findall(ext, filename)
    length(idxs) == 0 && return (filename, "")
    breaking_index = first(last(idxs))
    return filename[1:breaking_index - 1], filename[breaking_index:end]
end

"""
    CCDReduction.writefits(file_path, data; header = nothing)
    CCDReduction.writefits(file_path, ccd::CCDData)

Writes `data`/`ccd` in FITS format at `file_path`.

`FITSIO` takes over memory write in by `cfitsio`, which writes in row-major form, whereas when Julia gives that memory, it is assumed as column major.
Therefore all data written by [`FITSIO.write`](http://juliaastro.github.io/FITSIO.jl/latest/api.html#Base.write-Tuple{FITS,Dict{String,V}%20where%20V}) is transposed. This function allows the user to write the data in a consistent way to FITS file by transposing before writing.
"""
function writefits(file_path, data; header = nothing)
    d = ndims(data)
    transposed_data = permutedims(data, d:-1:1)
    FITS(file_path, "w") do fh
        write(fh, transposed_data; header = header)
    end
end

writefits(file_path, ccd::CCDData) = writefits(file_path, ccd.data; header = ccd.hdr)

#---------------------------------------------------------------------------------------
@doc raw"""
    fitscollection(dir;
                   recursive=true,
                   abspath=true,
                   keepext=true,
                   ext=r"fits(\.tar\.gz)?",
                   exclude=nothing,
                   exclude_dir=nothing,
                   exclude_key=("", "HISTORY"))

Walk through `dir` collecting FITS files, scanning their headers, and culminating into a `DataFrame` that can be used with the generators for iterating over many files and processing them. If `recursive` is false, no subdirectories will be walked through.

The table returned will contain the path to the file, the name of the file, and index of the corresponding HDU, and each FITS header column and value. If two FITS files have distinct columns, they will both appear in the table with `missing` in the appropriate rows.

!!! note "Duplicate Keys"
    In certain cases, there are multiple FITS headers with the same key, e.g., `COMMENT`. In these cases, only the first instance of the key-value pair will be stored.

If `abspath` is true, the path in the table will be absolute. If `keepext` is true, the name in the table will include the file extension, given by `ext`. `ext` will be used with `endswith` to filter for fits files compatible with `FITSIO.FITS`. `exclude` is a pattern that can be used with `occursin` to exclude certain filenames. For example, to exclude any files containing "sky",
```julia
fitscollection(...; exclude="sky")
```
to exclude exact filenames, [regex strings](https://docs.julialang.org/en/v1/manual/strings/#Regular-Expressions-1) will prove powerful
```julia
fitscollection(...; exclude=r"^tek001\d")
```
finally, using external tools like [Glob.jl](https://github.com/vtjnash/Glob.jl) allows further customization
```julia
using Glob
fitscollection(...; exclude=fn"tek001*.fits") # same as regex match above
```
Similarly, `exclude_dir` allows excluding entire folders using pattern matching (e.g. skipping a backup folder `exclude_dir="backup"`).
`exclude_key` allows excluding certain entries in the header unit of `ImageHDU` in FITS files (e.g. skipping `"HISTORY"` and `""` `exclude_key = ("HISTORY", "")`).

For more information about the file matching and path deconstruction, see the extended help (`??fitscollection`)
# Extended Help

## Parts of a path

Let's look at some file paths starting from `"/data"`. Here are examples of how they would be parsed

```
 root  dir   base   ext
[----][---][------][---]
/data/test/tek0001.fits

 root    dir     base   ext
[----][-------][------][---]
/data/test/sci/tek0001.fits
```

If `keepext` is `true`, `name=base * ext`, otherwise it is just `base`. If `abspath` is `true`, the path will be `root * dir * base * ext`, otherwise it will be `dir * base * ext`. These options allow flexility in creating a table that can be easily saved and loaded to avoid having to manually filter files. Especially consider how `abspath` can allow keeping tables that will transfer easily between computers or between data sources with common structures.
"""
function fitscollection(basedir::String;
                        recursive = true,
                        abspath = true,
                        keepext = true,
                        ext = r"fits(\.tar\.gz)?"i,
                        exclude = nothing,
                        exclude_dir = nothing,
                        exclude_key = ("", "HISTORY"))
    collection = DataFrame()

    for (root, dirs, files) in walkdir(basedir)
        # recursive searching functionality
        recursive || root == basedir || continue
        # To exclude certain directories
        if exclude_dir !== nothing
            occursin(exclude_dir, root) && continue
        end
        for filename in files
            # accept file if .fits or .fits.tar.gz
            endswith(filename, ext) || continue
            # excluding the files specified by user
            if exclude !== nothing
                occursin(exclude, filename) && continue
            end
            location = joinpath(root, filename)
            fits_data = FITS(location)

            for (index, hdu) in enumerate(fits_data)
                hdu isa ImageHDU || continue
                header_data = read_header(hdu)
                path = abspath ? Base.abspath(location) : location
                name = parse_name(filename, "." * ext, Val(keepext))

                # filtering out excluded columns
                _keys = filter(k -> k ∉ exclude_key, keys(header_data))
                # if there are duplicate keys (usually COMMENT) only use first
                unique_inds = unique(idx -> _keys[idx], eachindex(_keys))
                unique_keys = _keys[unique_inds]
                # create generator for values from the keys
                _values = (header_data[k] for k in unique_keys)

                key_val_itr = zip(Symbol.(unique_keys), _values)

                push!(collection, (path = path, name = name, hdu = index, key_val_itr...); cols = :union)
            end
            close(fits_data)
        end
    end
    return collection
end

"""
    arrays(collection)

Generator for arrays of images of entries in data frame.

Iterates over `collection` using each `path` and `hdu` to load data into an `Array`.

# Examples
```julia
collection = fitscollection("~/data/tekdata")
data = arrays(collection) |> collect
```
This returns all image arrays present in `collection`. This can also be used via a for-loop
```julia
collection = fitscollection("~/data/tekdata")
for arr in arrays(collection)
    @assert arr isa Array
    println(size(arr))
end

# output
(1048, 1068)
(1048, 1068)
...
```
"""
function arrays end

# generator for image arrays specified by data frames (i.e. path of file, hdu etc.)
@resumable function arrays(collection)
    for row in eachrow(collection)
        fh = FITS(row.path)
        @yield getdata(fh[row.hdu])
        close(fh)
    end
end

"""
    filenames(collection)

Generator for filenames of entries in data frame.

Iterates over `collection` using each `path`.

# Examples
```julia
collection = fitscollection("~/data/tekdata")
for path in filenames(collection)
    @assert path isa String
    println(path)
end

# output
"~/data/tekdata/tek001.fits"
"~/data/tekdata/tek002.fits"
...
```
"""
function filenames end

# generator for filenames specified by data frame (i.e. path of file, hdu etc.)
@resumable function filenames(collection)
    for row in eachrow(collection)
        @yield row.path
    end
end

"""
    ccds(collection)

Generator for `CCDData`s of entries in data frame.

Iterates over `collection` using each `path` and `hdu` to load data into a [`CCDData`](@ref).

# Examples
```julia
collection = fitscollection("~/data/tekdata")
for hdu in ccds(collection)
    @assert hdu isa CCDData
end
```
"""
function ccds end

# generator for CCDData specified by data frame (i.e. path of file, hdu etc.)
@resumable function ccds(collection)
    for row in eachrow(collection)
        @yield CCDData(row.path; hdu = row.hdu)
    end
end

"""
    ccds(f,
         collection;
         path = nothing,
         save_prefix = nothing,
         save_suffix = nothing,
         save = any(!isnothing, (save_prefix, path, save_suffix)),
         save_delim = "_",
         ext = r"fits(\\.tar\\.gz)?"i,
         kwargs...)

Iterates over the `CCDData`s of the collection applying function `f` at each step.

The output from `f` can be saved using the appropriate keyword arguments. The `save_prefix` argument will add a prefix to each filename delimited by `save_delim`. `save_suffix` will add a suffix prior to the extension, which can be manually provided via `ext`, similar to [`fitscollection`](@ref). Files will be saved in the directory they are stored unless `path` is given. Finally, `save` will default to `true` if any of the previous arguments are set, but can be manually overridden (useful for testing). Files will be saved using [`CCDReduction.writefits`](@ref).

# Example
```julia
collection = fitscollection("~/data/tekdata")
processed_images = map(ccds(collection)) do img
    trim(img, (:, 1040:1059))
end
```
The above generates `processed_images` which consists of trimmed versions of images present in `collection`.

For saving the `processed_images` simultaneously with the operations performed
```julia
processed_images = map(ccds(collection; path = "~/data/tekdata", save_prefix = "trimmed")) do img
    trim(img, (:, 1040:1059))
end
```
The trimmed images are saved as `trimmed_(original_name)` (FITS files) at `path = "~/data/tekdata"` as specified by the user.
"""
function ccds(f,
              collection;
              path = nothing,
              save_prefix = nothing,
              save_suffix = nothing,
              save = any(!isnothing, (save_prefix, path, save_suffix)),
              save_delim = "_",
              ext = r"fits(\.tar\.gz)?"i,
              kwargs...)
    image_iterator = ccds(collection; kwargs...)
    locations = collection.path

    processed_images = map(zip(locations, image_iterator)) do (location, output)
        processed_image = f(output)
        if save
            # if path is nothing and still the file is being saved, the location of input file is used
            if path isa Nothing
                path = dirname(location)
            end
            save_path = generate_filename(location, path, save_prefix, save_suffix, save_delim, ext)
            writefits(save_path, processed_image)
        end
        processed_image
    end

    return processed_images
end

"""
    filenames(f,
              collection;
              path = nothing,
              save_prefix = nothing,
              save_suffix = nothing,
              save = any(!isnothing, (save_prefix, path, save_suffix)),
              save_delim = "_",
              ext = r"fits(\\.tar\\.gz)?"i,
              kwargs...)

Iterates over the file paths of the collection applying function `f` at each step.

The output from `f` can be saved using the appropriate keyword arguments. The `save_prefix` argument will add a prefix to each filename delimited by `save_delim`. `save_suffix` will add a suffix prior to the extension, which can be manually provided via `ext`, similar to [`fitscollection`](@ref). Files will be saved in the directory they are stored unless `path` is given. Finally, `save` will default to `true` if any of the previous arguments are set, but can be manually overridden (useful for testing). Files will be saved using [`CCDReduction.writefits`](@ref).

# Examples
```julia
collection = fitscollection("~/data/tekdata")
data = map(filenames(collection)) do path
    fh = FITS(path)
    data = getdata(fh[1]) # assuming all 1-hdu are ImageHDUs
    close(fh)
    data
end
```
The above generates `data` which consists of image arrays corresponding to 1st hdu of FITS file paths present in `collection`.
For saving the `data` simultaneously with the operations performed
```julia
data = map(filenames(collection; path = "~/data/tekdata", save_prefix = "retrieved_from_filename")) do img
    fh = FITS(path)
    data = getdata(fh[1]) # assuming all 1-hdu are ImageHDUs
    close(fh)
    data
end
```
The retrieved data is saved as `retrieved_from_filename_(original_name)` (FITS files) at `path = "~/data/tekdata"` as specified by the user.
"""
function filenames(f,
                   collection;
                   path = nothing,
                   save_prefix = nothing,
                   save_suffix = nothing,
                   save = any(!isnothing, (save_prefix, path, save_suffix)),
                   save_delim = "_",
                   ext = r"fits(\.tar\.gz)?"i,
                   kwargs...)
    path_iterator = filenames(collection; kwargs...)
    locations = collection.path

    processed_images = map(zip(locations, path_iterator)) do (location, output)
        processed_image = f(output)
        if save
            # if path is nothing and still the file is being saved, the location of input file is used
            if path isa Nothing
                path = dirname(location)
            end
            save_path = generate_filename(location, path, save_prefix, save_suffix, save_delim, ext)
            writefits(save_path, processed_image)
        end
        processed_image
    end

    return processed_images
end

"""
    arrays(f,
           collection;
           path = nothing,
           save_prefix = nothing,
           save_suffix = nothing,
           save = any(!isnothing, (save_prefix, path, save_suffix)),
           save_delim = "_",
           ext = r"fits(\\.tar\\.gz)?"i,
           kwargs...)

Iterates over the image arrays of the collection applying function `f` at each step.

The output from `f` can be saved using the appropriate keyword arguments. The `save_prefix` argument will add a prefix to each filename delimited by `save_delim`. `save_suffix` will add a suffix prior to the extension, which can be manually provided via `ext`, similar to [`fitscollection`](@ref). Files will be saved in the directory they are stored unless `path` is given. Finally, `save` will default to `true` if any of the previous arguments are set, but can be manually overridden (useful for testing). Files will be saved using [`CCDReduction.writefits`](@ref).

# Examples
```julia
collection = fitscollection("~/data/tekdata")
processed_images = map(arrays(collection)) do arr
    trim(arr, (:, 1040:1059))
end
```
The above generates `processed_images` which consists of trimmed versions of image arrays present in `collection`.
For saving the `processed_images` simultaneously with the operations performed
```julia
processed_images = map(arrays(collection; path = "~/data/tekdata", save_prefix = "trimmed")) do img
    trim(img, (:, 1040:1059))
end
```
The trimmed image arrays are saved as `trimmed_(original_name)` (FITS files) at `path = "~/data/tekdata"` as specified by the user.
"""
function arrays(f,
                collection;
                path = nothing,
                save_prefix = nothing,
                save_suffix = nothing,
                save = any(!isnothing, (save_prefix, path, save_suffix)),
                save_delim = "_",
                ext = r"fits(\.tar\.gz)?"i,
                kwargs...)
    array_iterator = arrays(collection; kwargs...)
    locations = collection.path

    processed_images = map(zip(locations, array_iterator)) do (location, output)
        processed_image = f(output)
        if save
            # if path is nothing and still the file is being saved, the location of input file is used
            if path isa Nothing
                path = dirname(location)
            end
            save_path = generate_filename(location, path, save_prefix, save_suffix, save_delim, ext)
            writefits(save_path, processed_image)
        end
        processed_image
    end

    return processed_images
end

ext: jl