change our default garb_coll level in athena and add an option not to clean
@@ -149,7 +149,7 @@ class AthenaLinker(Linker):
input_table_aliases: Union[str, list] = None,
set_up_basic_logging=True,
output_filepath: str = "splink_warehouse",
- garbage_collection: bool = True,
+ garbage_collection_level: int = 1,
):
"""An athena backend for our main linker class. This funnels our generated SQL
@@ -177,12 +177,16 @@ class AthenaLinker(Linker):
so that Splink sends messages at INFO level to stdout. Defaults to True.
output_filepath (str, optional): Inside of your selected output bucket,
where to write output files to. Defaults to "splink_warehouse".
- garbage_collection (bool): If True, garbage collection will scan your
- specified output database for any previously generated '__splink_df'
- files and delete both the database table and backing s3 data. If False,
- this will only delete the link between this s3 data and the database
- (i.e. your previously created parquet files will still exist on s3
- and can be relinked if desired).
+ garbage_collection_level (int, optional): Garbage collection cleans up both your
+ database and s3 bucket, deleting or unlinking any tables previously
+ generated by splink. 0, 1 and 2 are the accepted levels. Defaults to 1.
+
+ 0 performs no cleaning. Existing tables will not be unlinked or deleted.
+ 1 will unlink any tables with the '__splink_df' prefix within the database you've
+ specified for linking, but leaves the underlying s3 files intact.
+ 2 scans your specified output database for any tables with the '__splink_df'
+ prefix and both unlinks the database table and deletes the backing data on s3.
+
Examples:
>>> # Creating a database in athena and writing to it
>>> import awswrangler as wr
@@ -280,7 +284,7 @@ class AthenaLinker(Linker):
self.output_schema = output_database
self._drop_all_tables_created_by_splink(
- garbage_collection,
+ garbage_collection_level,
homogenised_aliases,
)
@@ -389,7 +393,26 @@ class AthenaLinker(Linker):
self.ctas_query_info.pop(physical_name)
- def _drop_all_tables_created_by_splink(self, garbage_collection, input_tables):
+ def _drop_all_tables_created_by_splink(self, garbage_collection_level=1, input_tables=[]):
+ """A method that runs a cleanup process for the tables created by splink and currently
+ contained in your designated database.
+
+ Historic tables will not be wiped by this process, only those currently contained on the
+ database selected by the user.
+
+ Attributes:
+ garbage_collection_level (int): The amount of cleaning you wish to be performed.
+ 0 performs no cleaning. Existing tables will not be unlinked or deleted.
+ 1 will unlink any tables with the '__splink_df' prefix within the database you've
+ specified for linking, but leaves the underlying s3 files intact.
+ 2 scans your specified output database for any tables with the '__splink_df'
+ prefix and both unlinks the database table and deletes the backing data on s3.
+ input_tables (list): A list of input tables you wish to add to an ignore list. These
+ will not be removed during garbage collection.
+ """
+ # No collection requested
+ if garbage_collection_level == 0:
+ return
# This will only delete tables created within the splink process. These are
# tables containing the specific prefix: "__splink"
@@ -408,7 +431,7 @@ class AthenaLinker(Linker):
table=t["Name"],
boto3_session=self.boto3_session,
)
- if garbage_collection:
+ if garbage_collection_level==2:
path = t["StorageDescriptor"]["Location"]
wr.s3.delete_objects(
path=path,
