Skip to content

Instantly share code, notes, and snippets.

@erans

erans/s3clonebucket.py

Last active April 25, 2020 12:27
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save erans/1275085 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save erans/1275085 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. erans revised this gist Jan 15, 2015. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions s3clonebucket.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,5 @@
    import argparse
    import time

    from boto.s3.connection import S3Connection
    from boto.exception import S3ResponseError
  2. erans revised this gist Jan 15, 2015. 1 changed file with 11 additions and 2 deletions.
    13 changes: 11 additions & 2 deletions s3clonebucket.py
    View file
    Original file line number Diff line number Diff line change
    @@ -22,8 +22,17 @@ def run(args):
    if args.verbose:
    print "\tCopying %s" % key.key

    key.copy(destination_bucket.name, key.key)
    counter += 1
    i = 0
    while(True):
    try:
    key.copy(destination_bucket.name, key.key)
    break
    except boto.exception.S3CopyError:
    print 'exception during copying... will try again...'
    time.sleep(2)
    if i > 10:
    raise
    counter += 1

    if args.verbose:
    print "- Copied %d items" % counter
  3. erans created this gist Oct 10, 2011.
    49 changes: 49 additions & 0 deletions s3clonebucket.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,49 @@
    import argparse

    from boto.s3.connection import S3Connection
    from boto.exception import S3ResponseError

    def run(args):
    s3_connection = S3Connection(args.aws_access_key, args.aws_secret_access_key)
    source_bucket = s3_connection.get_bucket(args.source_bucket)
    destination_bucket = None
    try:
    destination_bucket = s3_connection.get_bucket(args.destination_bucket)
    except S3ResponseError, ex:
    if ex.status == 404:
    destination_bucket = s3_connection.create_bucket(args.destination_bucket)

    if args.verbose:
    print "- Copying from %s to %s" % (source_bucket.name, destination_bucket.name)

    counter = 0
    rs = source_bucket.list()
    for key in rs:
    if args.verbose:
    print "\tCopying %s" % key.key

    key.copy(destination_bucket.name, key.key)
    counter += 1

    if args.verbose:
    print "- Copied %d items" % counter
    print "- Done"

    def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--source_bucket', help='Source Bucket')
    parser.add_argument('-d', '--destination_bucket', help='Destination Bucket')
    parser.add_argument('-k', '--aws_access_key', help='AWS Access Key. Default can be taken from environment variable "AWS_ACCESS_KEY_ID"', default=None)
    parser.add_argument('-sk', '--aws_secret_access_key', help='AWS Secret Access Key. Default can be taken from environment variable "AWS_SECRET_ACCESS_KEY"', default=None)
    parser.add_argument('-v', '--verbose', help="Be verbose", action='store_true')

    args = parser.parse_args()

    if args.source_bucket and args.destination_bucket:
    run(args)
    else:
    parser.print_help()


    if __name__ == "__main__":
    main()