fix: ignore an error with terminated master when deleting AWS stacks.
In user has terminated the master instance manually which
caused `det deploy aws down` to fail. We should ignore errors in instance
stopping, because it's going to be deleted anyhow.
@@ -12,6 +12,11 @@ from determined.deploy.aws import constants
# will not fail CI.
NUM_WAITS = 5
+DELETE_MASTER_IGNORE_ERRORS = (
+ "IncorrectInstanceState",
+ "InvalidInstanceID.NotFound",
+)
+
def get_user(boto3_session: boto3.session.Session) -> str:
sts = boto3_session.client("sts")
@@ -21,10 +26,23 @@ def get_user(boto3_session: boto3.session.Session) -> str:
return arn.split("/")[-1]
-def stop_master(master_id: str, boto3_session: boto3.session.Session) -> None:
+def stop_master(master_id: str, boto3_session: boto3.session.Session, delete: bool = False) -> None:
ec2 = boto3_session.client("ec2")
waiter = ec2.get_waiter("instance_stopped")
+ try:
ec2.stop_instances(InstanceIds=[master_id])
+ except ClientError as ex:
+ if delete:
+ error_code = ex.response.get("Error", {}).get("Code")
+ if error_code in DELETE_MASTER_IGNORE_ERRORS:
+ print(
+ f"Failed to stop Master Instance: {error_code}. "
+ "This error is ignored as the instance is going to be deleted."
+ )
+ return
+
+ raise
+
ec2.modify_instance_attribute(
Attribute="disableApiTermination", Value="false", InstanceId=master_id
)
@@ -53,7 +71,7 @@ def delete(stack_name: str, boto3_session: boto3.session.Session) -> None:
if describe_instance_response["Reservations"]:
print("Stopping Master Instance")
- stop_master(master_id, boto3_session)
+ stop_master(master_id, boto3_session, delete=True)
# Second, terminate the agents so nothing can write to the checkpoint bucket. We create agent
# instances outside of cloudformation, so we have to manually terminate them.
