import os import boto3 from botocore.exceptions import ClientError from yapl.Trace import Trace,Level from yapl.Exceptions import MissingArgumentException from yapl.Exceptions import InvalidArgumentException from yapl.Exceptions import AccessDeniedException TR = Trace(__name__) S3ClientMethodRequiredArgs = { 'download_file': ["Bucket", "Key", "Filename"] } class S3Helper(object): """ Various methods that ease the use of the boto3 Python library for working with S3. """ def __init__(self, region=""): """ region - AWS region name """ object.__init__(self) self.s3Resource = boto3.resource('s3') self.region=region if (self.region): self.s3Client = boto3.client('s3', region_name=self.region) else: self.s3Client = boto3.client('s3') #endIf #endDef def _getRequiredArgs(self, method, **kwargs): """ Return a list of required arguments for the given method """ requiredArgs = [] argNames = S3ClientMethodRequiredArgs.get(method) if (argNames): for argName in argNames: argValue = kwargs.get(argName) if (argValue == None): raise MissingArgumentException("The S3 client method: '%s' requires a '%s' argument." % (method,argName)) #endIf requiredArgs.append(argValue) #endFor #endIF return requiredArgs #endDef def bucketExists(self, bucketName): """ Return True if the bucket exists and access is permitted. If bucket does not exist return False If bucket exists but access is forbidden, raise an exception. Picked this up from: https://stackoverflow.com/questions/26871884/how-can-i-easily-determine-if-a-boto-3-s3-bucket-resource-exists """ methodName = "bucketExists" result = False try: self.s3Client.head_bucket(Bucket=bucketName) result = True except ClientError as e: # If a client error is thrown, then check that it was a 404 error. error_code = e.response['Error']['Code'] if (TR.isLoggable(Level.FINEST)): TR.finest(methodName,"Error code: %s" % error_code) #endIf error_code = int(error_code) if (error_code == 404): result = False else: if (error_code == 403): raise AccessDeniedException("Access denied to S3 bucket named: %s" % bucketName) else: raise e #endIf #endIf #endTry return result #endDef def createBucket(self,bucketName,region=None): """ Return an instance of S3 bucket either for a bucket that already exists or for a newly created bucket in the given region. If a region is not specified, the bucket is created in the S3 default region (us-east-1). NOTE: Region is required, either on the method call or to the S3Helper instance. """ methodName = "createBucket" bucket = None if (self.bucketExists(bucketName)): bucket = self.s3Resource.Bucket(bucketName) else: if (self.region is None) or (self.region == 'us-east-1'): response = self.s3Client.create_bucket(Bucket=bucketName) elif (self.region): response = self.s3Client.create_bucket(Bucket=bucketName, CreateBucketConfiguration={'LocationConstraint': self.region}) else: raise MissingArgumentException("The AWS region name for the bucket must be provided either to the S3Helper instance or in the createBucket() arguments.") #endIf if (TR.isLoggable(Level.FINE)): TR.fine(methodName,"Bucket: %s created in region: %s" % (bucketName,response.get('Location'))) #endIf bucket = self.s3Resource.Bucket(bucketName) #endIf return bucket #endDef def put_object(self,**kwargs): """ Very thin wrapper around S3 client put_object() """ self.s3Client.put_object(**kwargs) #endDef def download_file(self, **kwargs): """ Support for downloading a file from an S3 bucket and to a place in the local file system. S3 download_file required arguments: Bucket - S3 bucket name Key - S3 object key Filename - full path to the target file WARNING: (PVS 04 FEB 2019) S3 client download_file() keyword arguments not supported (at this time) kwargs: ExtraArgs, Callback, Config. See S3 client doc for download_file(). Additional kwargs mode - file system mode bits for the copied object NOTES: 1. The S3 client download_file() method only allows documented keyword arguments. It throws an exception if it finds extraneous keyword arguments. 2. The Filename argument needs to include the file name. (The path can be absolute or relative to the current working directory.) 3. The directory structure in the Filename argument must exist. """ requiredArgs = self._getRequiredArgs('download_file',**kwargs) Filename = kwargs.get('Filename') dirName = os.path.dirname(Filename) if (not os.path.exists(dirName)): os.makedirs(dirName) #endIf # TBD: If needed by some use-case, add support for kwargs here self.s3Client.download_file(*requiredArgs) mode = kwargs.get('mode') if (mode): os.chmod(Filename,mode) #endIf #endDef def invokeCommands(self, cmdDocs, start, **kwargs): """ Process command docs to invoke each command in sequence that is of kind s3. Processing of cmdDocs stops as soon as a doc kind that is not s3 is encountered. All cmdDocs that are processed are marked with a status attribute with the value PROCESSED. cmdDocs - a list of 1 or more YAML documents loaded from yaml.load_all() by the caller. start - index where to start processing in the cmdDocs list. NOTE: The method for each command is responsible for pulling out the arguments for the underlying S3 method. The S3 client methods only accept the arguments in the signature. Extraneous keyword arguments cause an exception to be raised. """ if (not cmdDocs): raise MissingArgumentException("A non-empty list of command documents (cmdDocs) must be provided.") #endIf for i in range(start,len(cmdDocs)): doc = cmdDocs[i] kind = doc.get('kind') if (not kind or kind != 's3'): break; # done command = doc.get('command') if (not command): raise InvalidArgumentException("A helm command document: %s, must have a command attribute." % doc) #endIf getattr(self,command)(**doc) doc['status'] = 'PROCESSED' #endFor #endDef #endClass