# **************************************************************************** # Copyright (c) Commvault Systems # All Rights Reserved # # File name : AmazonEC2OrphanedResourceUtilityv1.py # # Description : Python script to fetch and delete orphaned, Commvault-created AMIs and # snapshots based on retention # # **************************************************************************** # Description: # Generates a report of the following orphaned AMIs and snapshots: # - CBT AMIs and snapshots that are older than a specified retention period # - Integrity AMIs and snapshots that are not needed anymore # - IntelliSnap snapshots that are not needed anymore # Deletes the orphaned AMIs and snapshots that are created by Commvault. # Parameter: --accessKey # Value: Access key of IAM user to connect to AWS account # Required # Parameter: --secretKey # Value: Secret key of IAM user to connect to AWS account # Required # Parameter: --region # Value: AWS Region to fetch/delete orphaned AMIs and snapshots from (for example, us-east-1) # Required # Parameter: --deleteSnapshots # Value: True or False # Deletes AMIs and snapshots if input specified is True # Required # Parameter: --orphanedSnapshotsReport # Value: .csv # Reads the specified report and deletes resources if DeletionFlag is set to true # Optional # Parameter: --retentionDays # Value: Number of days (default value is 14 days) # Adds CBT AMIs and snapshots older than retentionDays to the orphaned snapshots report # Optional # Parameter: --mode # Value: orphanedCVSnaps or integritySnaps or orphanedCVIntelliSnaps (default value is orphanedCVSnaps) # Modes function as follows: # - orphanedCVSnaps: The script gets/deletes CBT AMIs, snapshots, and _GX_BACKUP_ snapshots # - integritySnaps: The script gets/deletes integrity AMIs and snapshots # - orphanedCVIntelliSnaps: The script gets/deletes IntelliSnap snapshots # Optional # Parameter: --commServeDbHostname # Value: CommServe host name to connect to database # Required if mode is integritySnaps or orphanedCVIntelliSnaps # Parameter: -- commServeDbUserName # Value: DB User name to connect to DB # Required if mode is integritySnaps or orphanedCVIntelliSnaps # Parameter: --commServeDbPassword # Value: Password to connect to DB # Required if mode is integritySnaps or orphanedCVIntelliSnaps # Parameter: --commServeDbPort # Value: Port to connect to database (default value is 1433) # Optional # Parameter: --help # Prints the usage of all input parameters # python AmazonEC2OrphanedResourceUtilityv1.py --help # Output: .csv file with name "AWSOrphanedSnapshotReport_us-east-1_.csv # mode = orphanedCVSnaps: Report contains CBT AMIs, CBT snapshots, and _GX_BACKUP_ snapshots # = integritySnaps: Report contains integrity AMIs and snapshots # = orphanedCVIntelliSnaps: Report contains IntelliSnap snapshots # The above resources are marked for deletion in the report # Sample report: AWSOrphanedSnapshotReport_us-east-1_2023-06-16_21-26-02.csv # ResourceId,DeletionFlag,ResourceType,CreatedFrom,Description,SubclientID,AgeInDays # ami-1234567abcdefghi0,True,CBT AMI,i-01234abcde,,123,20 # snap-1234567abcdefghi0,True,CBT Snap,vol-01234abcde1,,-,20 # snap-1234567abcdefghi0,True,CBT Snap (OLD),vol-01234abcde2,,-,25 # snap-1234567abcdefghi3,True,CV Snap,vol-01234abcde3,,-,30 # Logging: Logs are saved in AmazonEC2OrphanedResourceUtilityv1Log.log # Functionality 1: Only generate orphaned snapshot report # EX: python AmazonEC2OrphanedResourceUtilityv1.py --accessKey --secretKey --region us-east-1 --deleteSnapshots False # This fetches all the Commvault created CBT AMIs and snapshots that are older than default retention period of 14 days in us-east-1 # and generates the report # Functionality 2: Generate orphaned snapshot report and delete AMIs, snapshots # EX: python AmazonEC2OrphanedResourceUtilityv1.py --accessKey --secretKey --region us-east-1 --deleteSnapshots True --retentionDays 30 # This fetches all the Commvault created CBT AMIs and snapshots that are older than 30 days in us-east-1, generates the report and # deletes CBT AMIs, snapshots and _GX_BACKUP_ snapshots # EX: python AmazonEC2OrphanedResourceUtilityv1.py --accessKey --secretKey --region us-east-1 --deleteSnapshots True --mode integritySnaps --commServeDbHostname --commServeDbUserName --commServeDbPassword # This fetches all the Commvault created integrity AMIs and snapshots that are no longer required in us-east-1, generates the report and # deletes Integrity AMIs, snapshots # EX: python AmazonEC2OrphanedResourceUtilityv1.py --accessKey --secretKey --region us-east-1 --deleteSnapshots True --mode orphanedCVIntelliSnaps --commServeDbHostname --commServeDbUserName --commServeDbPassword # This fetches all the Commvault created intelli snapshots that are no longer required in us-east-1, generates the report and # deletes intelli snapshots # Functionality 3: Take report as input and delete the AMIs and snapshots that are marked for deletion. # EX: python AmazonEC2OrphanedResourceUtilityv1.py --accessKey --secretKey --region us-east-1 --deleteSnapshots True --orphanedSnapshotsReport # This reads the report and deletes the AMIs and snapshots for which DeletionFlag is set to True # Prerequisites to run this script: # AWS Permissions: # "ec2:DescribeImages", # "ec2:DeregisterImage", # "ec2:DeleteSnapshot", # "ec2:DescribeSnapshots" # Modules: # Python 3.10+ # Boto3 - AWS SDK for python # pymssql import boto3, argparse, sys, datetime, time, csv, logging, traceback, collections, pymssql, re from datetime import timedelta from dateutil.parser import parse from botocore.config import Config class AmazonEC2OrphanedResourceUtilityv1: def __init__(self, accessKey, secretKey, region, retentionDays): self.accessKey = accessKey self.secretKey = secretKey self.region = region self.retentionDays = int(retentionDays) self.connection = None self.resource = None self.orphanedCbtAmis = set() self.orphanedCbtSnaps = set() self.orphanedGxbackupSnaps = set() self.orphanedIntegrityAmis = set() self.orphanedIntegritySnaps = set() self.orphanedIntelliSnaps = set() self.now = datetime.datetime.now() self.logger = None self.configureLogger() def configureLogger(self): logFile = 'AmazonEC2OrphanedResourceUtilityv1Log.log' logging.basicConfig(filename=logFile, filemode='a', format='%(asctime)s %(funcName)s %(message)s', datefmt='%d-%m-%Y %H:%M:%S', level=logging.INFO ) self.logger = logging.getLogger() #connect to aws account def awsSession(self): try: self.logger.info('Connecting to [%s], user name [%s]', self.region, self.accessKey) self.connection = boto3.Session(aws_access_key_id=self.accessKey, aws_secret_access_key=self.secretKey, region_name=self.region) account_id = boto3.client('sts', aws_access_key_id=self.accessKey, aws_secret_access_key=self.secretKey ).get_caller_identity()['Account'] self.logger.info('Connection successful to account [%s]', account_id) retryConfig = Config( retries = { 'max_attempts': 150, 'mode': 'standard' } ) self.resource = self.connection.resource('ec2', self.region, config=retryConfig) except Exception as ex: self.logger.error('Connection failed. Error: [%s]', traceback.format_exc()) sys.exit() # method to get snapshot list and generate report def getOrphanedSnapshotsAndGenerateReport(self, mode, dbHostName, dbUserName, dbPassword, dbPort): try: orphanedSnapshotsReport = "AWSOrphanedSnapshotReport_"+self.region+'_'+str(datetime.date.today())+'_'+str(time.strftime("%H-%M-%S", time.localtime()))+".csv" with open(orphanedSnapshotsReport, 'w', newline='') as csv_write_obj: csvwriter = csv.writer(csv_write_obj) header = ['ResourceId', 'DeletionFlag', 'ResourceType', 'CreatedFrom', 'Description', 'SubclientID', 'AgeInDays'] csvwriter.writerow(header) if mode == 'orphanedCVSnaps': self.logger.info('Getting CBT AMIs and Snapshots older than [%d] days', self.retentionDays) self.getOrphanedCbtAmis(csvwriter) self.getOrphanedCbtSnaps(csvwriter) intelliSnapsInfo, intelliSnaps = self.getIntelliSnaps() orphanedReplicaCopySnaps = self.getOrphanedReplicaCopySnaps() integritySnapsInfo, integritySnaps = self.getIntegritySnaps() self.getOrphanedGxBackupSnaps(csvwriter, intelliSnaps, orphanedReplicaCopySnaps, integritySnaps) elif mode == 'integritySnaps': self.logger.info('Getting Integrity AMIs and Snapshots') self.getOrphanedIntegritySnaps(csvwriter, dbHostName, dbUserName, dbPassword, dbPort) elif mode == 'orphanedCVIntelliSnaps': self.logger.info('Getting Intelli Snapshots') self.getOrphanedIntelliSnaps(csvwriter, dbHostName, dbUserName, dbPassword, dbPort) self.logger.info('Generated orphaned snapshots report [%s]', orphanedSnapshotsReport) except Exception as ex: self.logger.error('Failed to generate orphaned snapshots report. Error: [%s]', traceback.format_exc()) sys.exit() def getOrphanedCbtAmis(self, csvwriter): self.logger.info('Getting orphaned CBT AMIs') # get AMIs with tag "Name = CV_CBT_Snap" cbtAmis = self.resource.images.filter(Filters=[ { 'Name': 'tag:Name', 'Values': ['CV_CBT_Snap'] } ]) for ami in cbtAmis: creationDate = parse(ami.creation_date) if (self.isResourceOlderThanRetentionPeriod(creationDate)): self.orphanedCbtAmis.add(ami.id) instanceId = '-' subclientId = '-' for tag in ami.tags: if tag['Key'] == '_GX_BACKUP_': instanceId = tag['Value'] elif tag['Key'] == 'CV_Subclient': subclientId = tag['Value'] csvwriter.writerow([ami.id, 'TRUE', 'CBT AMI', instanceId, ami.description, subclientId, self.getAgeInDays(creationDate)]) #write to csv file self.logger.info('Found [%d] orphaned CBT AMIs', len(self.orphanedCbtAmis)) def getOrphanedCbtSnaps(self, csvwriter): self.logger.info('Getting orphaned CBT snapshots') # get snaps with tag "Name = CV_CBT_Snap" cbtTagSnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag:Name', 'Values': ['CV_CBT_Snap'] } ]) for snap in cbtTagSnaps: if (self.isResourceOlderThanRetentionPeriod(snap.start_time)): self.orphanedCbtSnaps.add(snap.id) description = self.getSnapshotDescription(snap) csvwriter.writerow([snap.id, 'TRUE', 'CBT Snap', snap.volume_id, description, '-', self.getAgeInDays(snap.start_time)]) #write to csv file # get snaps with tag key "CV_Retain_Snap" retainTagSnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag-key', 'Values': ['CV_Retain_Snap'] } ]) for snap in retainTagSnaps: if (self.isResourceOlderThanRetentionPeriod(snap.start_time)): self.orphanedCbtSnaps.add(snap.id) description = self.getSnapshotDescription(snap) csvwriter.writerow([snap.id, 'TRUE', 'CBT Snap (OLD)', snap.volume_id, description, '-', self.getAgeInDays(snap.start_time)]) #write to csv file self.logger.info('Found [%d] orphaned CBT snapshot', len(self.orphanedCbtSnaps)) def getIntelliSnaps(self): self.logger.info('Getting Intellisnaps from AWS account') IntelliSnapsInfo = collections.defaultdict(dict) IntelliSnaps = set() # get snaps with tag "Name = SP_*" and tag key "_GX_BACKUP_" intelliSnapsResponse = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag-key', 'Values': ['_GX_BACKUP_'] }, { 'Name': 'tag:Name', 'Values': ['SP_*'] } ]) for snap in intelliSnapsResponse: IntelliSnaps.add(snap.id) IntelliSnapsInfo[snap.id]['volId'] = snap.volume_id IntelliSnapsInfo[snap.id]['description'] = self.getSnapshotDescription(snap) IntelliSnapsInfo[snap.id]['ageInDays'] = self.getAgeInDays(snap.start_time) self.logger.info('Found [%d] Intellisnaps', len(IntelliSnaps)) return IntelliSnapsInfo, IntelliSnaps def getOrphanedIntelliSnaps(self, csvwriter, dbHostName, dbUserName, dbPassword, dbPort): # get intellisnaps that are registered with DB registeredIntelliSnaps = self.getRegisteredIntelliSnaps(dbHostName, dbUserName, dbPassword, dbPort) # get intellisnaps from AWS account intelliSnapsInfo, intelliSnaps = self.getIntelliSnaps() # get orphaned intellisnaps self.orphanedIntelliSnaps = intelliSnaps - registeredIntelliSnaps for snap in self.orphanedIntelliSnaps: csvwriter.writerow([snap, 'TRUE', 'Intelli Snap', intelliSnapsInfo[snap]['volId'], intelliSnapsInfo[snap]['description'], '-', intelliSnapsInfo[snap]['ageInDays']]) #write to csv file self.logger.info('Found [%d] orphaned intellisnapshots', len(self.orphanedIntelliSnaps)) def getRegisteredIntelliSnaps(self, dbHostName, dbUserName, dbPassword, dbPort): try: #Assuming that the current AWS account is protected using single CS. If same account is being protected using multiple CS, we might end up deleting snaps that are registered with other CS. isSuccess = True registeredIntelliSnaps = set() self.logger.info('Getting intelli snapshots from DB') connection = pymssql.connect(dbHostName, dbUserName, dbPassword, "CommServ", dbPort) cursor = connection.cursor(as_dict=True) cursor.execute(""" USE CommServ SET NOCOUNT ON select UniqueIdentifier as 'Snapshot Id' from SMSnap where SnapShotEngineId = 60 --Amazon web services """) for resource in cursor: if resource['Snapshot Id']: registeredIntelliSnaps.add(resource['Snapshot Id']) self.logger.info('Found [%d] intellisnaps that are registered with DB', len(registeredIntelliSnaps)) except Exception as exp: isSuccess = False self.logger.error('Failed to get intellisnaps from DB. Error: [%s]', traceback.format_exc()) finally: connection.close() if isSuccess: return registeredIntelliSnaps else: sys.exit() def getOrphanedReplicaCopySnaps(self): self.logger.info('Getting orphaned replica copy snaps') orphanedReplicaCopySnaps = set() # get snaps with tag "Name = CopyOf_SP_*" and tag key "_GX_BACKUP_" replicaCopySnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag-key', 'Values': ['_GX_BACKUP_'] }, { 'Name': 'tag:Name', 'Values': ['CopyOf_SP_*'] } ]) for snap in replicaCopySnaps: if (self.isResourceOlderThanRetentionPeriod(snap.start_time)): orphanedReplicaCopySnaps.add(snap.id) self.logger.info('Found [%d] orphaned Replica copy snaps', len(orphanedReplicaCopySnaps)) return orphanedReplicaCopySnaps def getIntegritySnaps(self): self.logger.info('Getting Integrity snapshots') integritySnapsInfo = collections.defaultdict(dict) integritySnaps = set() # get snaps with tag "Name = CV_Integrity_Snap" ebsDirectSnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag:Name', 'Values': ['CV_Integrity_Snap'] } ]) integritySnaps, integritySnapsInfo = self.populateIntegritySnapsInfo(ebsDirectSnaps, integritySnaps, integritySnapsInfo) # get snaps with tag "Descrption = Integrity Snapshot Created for replication Job*" hotaddSnapsWithDescriptionTag = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag:Descrption', 'Values': ['Integrity Snapshot Created for replication Job*'] } ]) integritySnaps, integritySnapsInfo = self.populateIntegritySnapsInfo(hotaddSnapsWithDescriptionTag, integritySnaps, integritySnapsInfo) # get snaps with description "Integrity Snapshot Created for replication Job*" (SP34+) hotaddSnapsWithDescription = self.resource.snapshots.filter(Filters=[ { 'Name': 'description', 'Values': ['Integrity Snapshot Created for replication Job*'] } ]) integritySnaps, integritySnapsInfo = self.populateIntegritySnapsInfo(hotaddSnapsWithDescription, integritySnaps, integritySnapsInfo) self.logger.info('Found [%d] Integrity snapshots', len(integritySnaps)) return integritySnapsInfo, integritySnaps def populateIntegritySnapsInfo(self, integritySnapsResponse, integritySnaps, integritySnapsInfo): for snap in integritySnapsResponse: integritySnaps.add(snap.id) integritySnapsInfo[snap.id]['volId'] = snap.volume_id integritySnapsInfo[snap.id]['description'] = self.getSnapshotDescription(snap) integritySnapsInfo[snap.id]['ageInDays'] = self.getAgeInDays(snap.start_time) integritySnapsInfo[snap.id]['amiId'] = self.getAmiIdFromDescription(snap.description) return integritySnaps, integritySnapsInfo def getOrphanedGxBackupSnaps(self, csvwriter, intelliSnaps, orphanedReplicaCopySnaps, integritySnaps): self.logger.info('Getting orphaned _GX_BACKUP_ tag snapshots') gxbackupSnapsInfo = collections.defaultdict(dict) # get snaps with tag "Name = _GX_BACKUP_" gxbackupTagNameSnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag:Name', 'Values': ['_GX_BACKUP_'] } ]) for snap in gxbackupTagNameSnaps: if (self.isResourceOlderThanRetentionPeriod(snap.start_time)): self.orphanedGxbackupSnaps.add(snap.id) gxbackupSnapsInfo[snap.id]['volId'] = snap.volume_id gxbackupSnapsInfo[snap.id]['description'] = self.getSnapshotDescription(snap) gxbackupSnapsInfo[snap.id]['ageInDays'] = self.getAgeInDays(snap.start_time) # get snaps with tag key "_GX_BACKUP_" gxbackupTagKeySnaps = self.resource.snapshots.filter(Filters=[ { 'Name': 'tag-key', 'Values': ['_GX_BACKUP_'] } ]) for snap in gxbackupTagKeySnaps: if (self.isResourceOlderThanRetentionPeriod(snap.start_time)): self.orphanedGxbackupSnaps.add(snap.id) gxbackupSnapsInfo[snap.id]['volId'] = snap.volume_id gxbackupSnapsInfo[snap.id]['description'] = self.getSnapshotDescription(snap) gxbackupSnapsInfo[snap.id]['ageInDays'] = self.getAgeInDays(snap.start_time) # remove cbt, integrity, replica copy and intelli snaps from orphanedGxbackupSnaps self.orphanedGxbackupSnaps = self.orphanedGxbackupSnaps - self.orphanedCbtSnaps - intelliSnaps - orphanedReplicaCopySnaps - integritySnaps for snap in self.orphanedGxbackupSnaps: csvwriter.writerow([snap, 'TRUE', 'CV Snap', gxbackupSnapsInfo[snap]['volId'], gxbackupSnapsInfo[snap]['description'], '-', gxbackupSnapsInfo[snap]['ageInDays']]) #write to csv file self.logger.info('Found [%d] orphaned _GX_BACKUP_ snapshots', len(self.orphanedGxbackupSnaps)) def getOrphanedIntegritySnaps(self, csvwriter, dbHostName, dbUserName, dbPassword, dbPort): self.logger.info('Getting orphaned integrity AMIs and snapshots') integrityAmisInfo = collections.defaultdict(dict) # get integrity snapshots that are in use from DB integritySnapsInUse = self.getIntegritySnapsInUse(dbHostName, dbUserName, dbPassword, dbPort) # get integrity snapshots from AWS account integritySnapsInfo, integritySnaps = self.getIntegritySnaps() # get orphaned integrity snapshots self.orphanedIntegritySnaps = integritySnaps - integritySnapsInUse for snap in self.orphanedIntegritySnaps: csvwriter.writerow([snap, 'TRUE', 'Integrity Snap', integritySnapsInfo[snap]['volId'], integritySnapsInfo[snap]['description'], '-', integritySnapsInfo[snap]['ageInDays']]) #write to csv file amiId = integritySnapsInfo[snap]['amiId'] if amiId: self.orphanedIntegrityAmis.add(amiId) integrityAmisInfo[amiId]['description'] = integritySnapsInfo[snap]['description'] integrityAmisInfo[amiId]['ageInDays'] = integritySnapsInfo[snap]['ageInDays'] self.logger.info('Found [%d] orphaned Integrity snapshots', len(self.orphanedIntegritySnaps)) # save orphaned integrity amis in report for ami in self.orphanedIntegrityAmis: csvwriter.writerow([ami, 'TRUE', 'Integrity AMI', '-', integrityAmisInfo[ami]['description'], '-', integrityAmisInfo[ami]['ageInDays']]) #write to csv file self.logger.info('Found [%d] orphaned Integrity AMIs', len(self.orphanedIntegrityAmis)) def getIntegritySnapsInUse(self, dbHostName, dbUserName, dbPassword, dbPort): try: isSuccess = True integritySnapsInUse = set() self.logger.info('Getting integrity snapshots from DB') connection = pymssql.connect(dbHostName, dbUserName, dbPassword, "CommServ", dbPort) cursor = connection.cursor(as_dict=True) cursor.execute(""" USE CommServ SET NOCOUNT ON SET QUOTED_IDENTIFIER ON CREATE TABLE #vsaInfoTable (repGrpName NVARCHAR(MAX), replicationId INT, sourceVMName NVARCHAR(MAX), destVMName NVARCHAR(MAX), destInstanceType NVARCHAR(MAX), destHypervClientId INT) INSERT INTO #vsaInfoTable select ISNULL(REP.name, T.subTaskName), VR.replicationId, VR.sourceName, VR.destinationName, 0, isnull(st.xmlValue.value('(/TMMsg_JobOption/restoreOptions/virtualServerRstOption/vCenterInstance/@clientId)[1]','INT'),0) from APP_VSAReplication VR WITH (NOLOCK) LEFT JOIN App_ReplicationGroupAssociation REPASSOC WITH (NOLOCK) on VR.taskId = REPASSOC.taskId LEFT JOIN App_ReplicationGroup REP WITH (NOLOCK) on REP.id = REPASSOC.componentNameId INNER JOIN TM_SubTask T WITH (NOLOCK) ON VR.taskId = T.taskId INNER JOIN TM_SubTaskXMLOptions ST WITH (NOLOCK) ON T.subTaskId = ST.subTaskId update VSA set destInstanceType = I.attrVal from #vsaInfoTable VSA INNER JOIN APP_Client c WITH(NOLOCK) on c.id = VSA.destHypervClientId INNER JOIN APP_Application A WITH(NOLOCK) on A.clientId = c.id and A.appTypeId = 106 --CV_APPTYPE_VIRTUAL_SERVER INNER JOIN APP_InstanceProp I WITH(NOLOCK) on I.componentNameId = A.instance and I.attrName = 'Virtual Server Instance Type' and I.modified = 0 and VSA.destHypervClientId > 0 select repGrpName AS 'Replication Group Name', sourceVMName AS 'Source VM Name', destVMName AS 'Destination VM Name', CAST(AP.propertyValue AS XML).value('(/trackingIds/@snapshotId)[1]','NVARCHAR(MAX)') AS 'Snapshot Id' from #vsaInfoTable VSA LEFT JOIN APP_VSAReplicationProp AP WITH(NOLOCK) ON VSA.replicationId = AP.replicationId AND AP.propertyTypeId = 2201 WHERE destInstanceType = 301 --Amazon Web Services """) for resource in cursor: if resource['Snapshot Id']: integritySnapsInUse.add(resource['Snapshot Id']) self.logger.info('Found [%d] Integrity snapshots that are in use', len(integritySnapsInUse)) except Exception as exp: isSuccess = False self.logger.error('Failed to get integrity snapshots from DB. Error: [%s]', traceback.format_exc()) finally: connection.close() if isSuccess: return integritySnapsInUse else: sys.exit() def isResourceOlderThanRetentionPeriod(self, start_time): return (self.now - start_time.replace(tzinfo=None)) > timedelta(days=self.retentionDays) def getAgeInDays(self, start_time): return (self.now - start_time.replace(tzinfo=None)).days def getSnapshotDescription(self, snap): descriptionTag = '' for tag in snap.tags: if tag['Key'] == 'Description' or tag['Key']=='Descrption': descriptionTag = tag['Value'] break gxBackupTag = '' if not descriptionTag: for tag in snap.tags: if tag['Key'] == '_GX_BACKUP_': gxBackupTag = tag['Value'] break description = '-' if descriptionTag: description = descriptionTag elif snap.description: description = snap.description elif gxBackupTag: description = gxBackupTag return description def getAmiIdFromDescription(self, description): amiId = "" if description: match = re.search(r"\bami-.+", description) if match: amiId = match.group() return amiId def populateOrphanedSnapshotListFromReport(self, orphanedSnapshotsReport): try: # read report and populate snaps list self.logger.info('Reading report [%s]', orphanedSnapshotsReport) with open(orphanedSnapshotsReport, 'r') as csv_read_obj: csvReader = csv.DictReader(csv_read_obj) for resource in csvReader: if (resource['DeletionFlag'] == 'TRUE'): if (resource['ResourceType'] == 'CBT AMI'): self.orphanedCbtAmis.add(resource['ResourceId']) elif (resource['ResourceType'] == 'CBT Snap' or resource['ResourceType'] == 'CBT Snap (OLD)'): self.orphanedCbtSnaps.add(resource['ResourceId']) elif (resource['ResourceType'] == 'CV Snap'): self.orphanedGxbackupSnaps.add(resource['ResourceId']) elif (resource['ResourceType'] == 'Integrity AMI'): self.orphanedIntegrityAmis.add(resource['ResourceId']) elif (resource['ResourceType'] == 'Integrity Snap'): self.orphanedIntegritySnaps.add(resource['ResourceId']) elif (resource['ResourceType'] == 'Intelli Snap'): self.orphanedIntelliSnaps.add(resource['ResourceId']) except Exception as ex: self.logger.error('Failed to read report [%s]. Error: [%s]', orphanedSnapshotsReport, traceback.format_exc()) sys.exit() def deleteOrphanedSnapshots(self): if self.orphanedCbtAmis != set(): self.logger.info('Deleting orphaned CBT AMIs') for ami in self.orphanedCbtAmis: self.deleteAMI(ami) if self.orphanedCbtSnaps != set(): self.logger.info('Deleting orphaned CBT snapshots') for snap in self.orphanedCbtSnaps: self.deleteSnapshot(snap) if self.orphanedGxbackupSnaps != set(): self.logger.info('Deleting orphaned _GX_BACKUP_ tag snapshots') for snap in self.orphanedGxbackupSnaps: self.deleteSnapshot(snap) if self.orphanedIntegrityAmis != set(): self.logger.info('Deleting orphaned Integrity AMIs') for ami in self.orphanedIntegrityAmis: self.deleteAMI(ami) if self.orphanedIntegritySnaps != set(): self.logger.info('Deleting orphaned Integrity snapshots') for snap in self.orphanedIntegritySnaps: self.deleteSnapshot(snap) if self.orphanedIntelliSnaps != set(): self.logger.info('Deleting orphaned Intelli snapshots') for snap in self.orphanedIntelliSnaps: self.deleteSnapshot(snap) def deleteAMI(self, _amiId): try: ami = self.resource.Image(_amiId) ami.deregister() except Exception as ex: self.logger.error('Failed to delete AMI [%s]. Error: [%s]', _amiId, traceback.format_exc()) def deleteSnapshot(self, _snap): try: snapshot = self.resource.Snapshot(_snap) snapshot.delete() except Exception as ex: self.logger.error('Failed to delete snapshot [%s]. Error: [%s]', _snap, traceback.format_exc()) def main(): try: parser = argparse.ArgumentParser() parser.add_argument("--accessKey", dest="_accessKey", required=True, help='Access key of IAM user to connect to AWS account [Required]') parser.add_argument("--secretKey", dest="_secretKey", required=True, help='Secret key of IAM user to connect to AWS account [Required]') parser.add_argument("--region", dest="_region", required=True, help='AWS region to get/delete orphaned AMIs and snapshots (ex. us-east-1) [Required]') parser.add_argument("--deleteSnapshots", dest="_deleteSnapshots", required=True, help='True or False - Deletes AMIs and snapshots when set to True [Required]') parser.add_argument("--retentionDays", dest="_retentionDays", default=14, help='No. of days - AMIs and snapshots older than retentionDays will be added to the report (default - 14) [Optional]') parser.add_argument("--orphanedSnapshotsReport", dest="_orphanedSnapshotsReport", default="", help='.csv - Reads the report and deletes the AMIs and snapshots [Optional]') parser.add_argument("--mode", dest="_mode", default="orphanedCVSnaps", help='orphanedCVSnaps or integritySnaps or orphanedCVIntelliSnaps- Fetches CBT or Integrity orphaned AMI/snapshots or orphaned intelli snapshots based on the mode (default - orphanedCVSnaps) [Optional]') parser.add_argument("--commServeDbHostname", dest="_dbHostName", default="", help="Commserver host name to connect to DB [Required if specified mode is integritySnaps/orphanedCVIntelliSnaps]") parser.add_argument("--commServeDbUserName", dest="_dbUserName", default="", help="DB User name to connect to DB [Required if specified mode is integritySnaps/orphanedCVIntelliSnaps]") parser.add_argument("--commServeDbPassword", dest="_dbPassword", default="", help="Password to connect to DB [Required if specified mode is integritySnaps/orphanedCVIntelliSnaps]") parser.add_argument("--commServeDbPort", dest="_dbPort", default="1433", help="Port to connect to DB (default - 1433) [Optional]") args = parser.parse_args() if args._mode == 'integritySnaps' or args._mode == 'orphanedCVIntelliSnaps': if (not args._dbHostName) or (not args._dbUserName) or (not args._dbPassword): print('Inputs --commServeDbHostname, --commServeDbUserName, --commServeDbPassword are required for integritySnaps/orphanedCVIntelliSnaps mode') sys.exit() except argparse.ArgumentError: print('Error in parsing the arguments..Exiting!') sys.exit() orphanedSnapshotsCleanup = AmazonEC2OrphanedResourceUtilityv1(args._accessKey, args._secretKey, args._region, args._retentionDays) #connect to aws account orphanedSnapshotsCleanup.awsSession() if (not args._orphanedSnapshotsReport): orphanedSnapshotsCleanup.getOrphanedSnapshotsAndGenerateReport(args._mode, args._dbHostName, args._dbUserName, args._dbPassword, args._dbPort) else: orphanedSnapshotsCleanup.populateOrphanedSnapshotListFromReport(args._orphanedSnapshotsReport) if args._deleteSnapshots == 'True': orphanedSnapshotsCleanup.deleteOrphanedSnapshots() if __name__ == "__main__": sys.exit(main())