|
#!/usr/bin/python
|
|
|
|
import boto
|
|
import math
|
|
import boto.s3.connection
|
|
import hashlib
|
|
|
|
# credentials
|
|
|
|
AWS_ACCESS_KEY_ID = 'KEY_ID_HERE'
|
|
AWS_SECRET_ACCESS_KEY = 'ACCESS_KEY_HERE'
|
|
|
|
# buckets, keys and so on
|
|
|
|
bucket_name = 'my-bucket-name-multipart-uploads-cp-test'
|
|
source_bucket = bucket_name
|
|
destination_bucket = bucket_name
|
|
orig_key_name = 'my-key-name-multipart-uploads-cp-test'
|
|
dest_key_name = 'copy-' + orig_key_name
|
|
|
|
def md5(fname):
|
|
hash = hashlib.md5()
|
|
with open(fname) as f:
|
|
for chunk in iter(lambda: f.read(4096), ""):
|
|
hash.update(chunk)
|
|
return hash.hexdigest()
|
|
|
|
# get connection
|
|
|
|
conn = boto.connect_s3(
|
|
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
|
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
|
is_secure=False,
|
|
calling_format = boto.s3.connection.OrdinaryCallingFormat(),
|
|
host='s3.amazonaws.com', # fake this host in /etc/hosts if you are using radosgw
|
|
port=8000,
|
|
)
|
|
|
|
print "got connection!"
|
|
|
|
# create bucket
|
|
|
|
bucket = conn.create_bucket(source_bucket)
|
|
|
|
print "bucket created!"
|
|
|
|
# get md5 from file
|
|
|
|
md5_before = md5('test.bin')
|
|
|
|
# upload object
|
|
|
|
key = bucket.new_key(orig_key_name)
|
|
key.set_contents_from_filename('test.bin')
|
|
|
|
print "object uploaded!"
|
|
|
|
# lookup object
|
|
|
|
sb = conn.get_bucket(source_bucket)
|
|
ky = sb.lookup(orig_key_name)
|
|
objectSize = ky.size
|
|
|
|
print "found object size of %d" % objectSize
|
|
|
|
# initiate multipart upload
|
|
|
|
b = conn.get_bucket(destination_bucket)
|
|
mp = b.initiate_multipart_upload(dest_key_name, reduced_redundancy=True)
|
|
|
|
print "multipart initiated!"
|
|
|
|
psize = 5 * math.pow(2.0, 20.0) # 2^20 = 1 MiB
|
|
|
|
bytePosition = 0
|
|
i = 1
|
|
while bytePosition < objectSize:
|
|
lastbyte = bytePosition + psize - 1
|
|
if lastbyte > objectSize:
|
|
lastbyte = objectSize - 1
|
|
print "mp.copy_part_from_key part %d (%d %d)" % (i, bytePosition, lastbyte)
|
|
mp.copy_part_from_key(source_bucket, orig_key_name, i, int(bytePosition), int(lastbyte))
|
|
i = i + 1
|
|
bytePosition += psize
|
|
|
|
mp.complete_upload()
|
|
|
|
print "multipart upload completed"
|
|
|
|
# download recent uploaded object
|
|
|
|
b = conn.get_bucket(destination_bucket)
|
|
key = bucket.get_key(dest_key_name)
|
|
key.get_contents_to_filename('test.bin-output')
|
|
|
|
print "recent uploaded object downloaded"
|
|
|
|
# get md5 from file
|
|
|
|
md5_after = md5('test.bin-output')
|
|
|
|
if md5_before == md5_after:
|
|
print "md5 test ok"
|
|
else:
|
|
print "md5 test FAILS"
|
|
|
|
print "done"
|