Ceph

"Hello World!\n world is not enough"*1000000000

"H"*1000000000

$ PYTHONPATH=pybind python /tmp/a.py
Created cluster handle.
Connected to the cluster.

I/O Context and Object Operations
=================================

Creating a context for the 'rbd' pool

Writing object 'bm' with contents 'Bonjour tout le monde!' to pool 'rbd'.
Writing XATTR 'lang' with value 'fr_FR' to object 'bm'

Writing object 'bm1' with contents 'Bonjour tout le monde!' to pool 'rbd'.
Writing XATTR 'lang' with value 'fr_FR' to object 'bm1'

Writing object 'hw' with contents 'Hello World!' to pool 'rbd'.
Traceback (most recent call last):
  File "/tmp/a.py", line 31, in <module>
    ioctx.write("hw", "H"*1000000000)
  File "/home/loic/software/ceph/ceph/src/pybind/rados.py", line 1223, in write
    (self.name, key))
rados.Error: Ioctx.write(rbd): failed to write hw: errno EMSGSIZE

$ ceph --version
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
ceph version 0.85-920-gba02a5e (ba02a5e638ddae3d58e1f5d8bde741da9a6e342d)

import rados
try:
    cluster = rados.Rados(conffile='ceph.conf')
except TypeError as e:
    print 'Argument validation error: ', e
    raise e
print "Created cluster handle." 
try:
    cluster.connect()
except Exception as e:
    print "connection error: ", e
    raise e
finally:
    print "Connected to the cluster." 
    print "\n\nI/O Context and Object Operations" 
    print "=================================" 
    print "\nCreating a context for the 'rbd' pool" 

if not cluster.pool_exists('rbd'):
    raise RuntimeError('No rbd pool exists')
ioctx = cluster.open_ioctx('rbd')
print "\nWriting object 'bm' with contents 'Bonjour tout le monde!' to pool 'rbd'." 
ioctx.write("bm", "Bonjour tout le monde!")
print "Writing XATTR 'lang' with value 'fr_FR' to object 'bm'" 
ioctx.set_xattr("bm", "lang", "fr_FR")
print "\nWriting object 'bm1' with contents 'Bonjour tout le monde!' to pool 'rbd'." 
ioctx.write("bm1", "Bonjour tout le monde!"*10000)
print "Writing XATTR 'lang' with value 'fr_FR' to object 'bm1'" 
ioctx.set_xattr("bm1", "lang", "fr_FR")
print "\nWriting object 'hw' with contents 'Hello World!' to pool 'rbd'." 
ioctx.write("hw", "H"*1000000000)
print "Writing XATTR 'lang' with value 'en_US' to object 'hw'" 
ioctx.set_xattr("hw", "lang", "en_US")
'''
print "\nContents of object 'hw'\n------------------------" 
print ioctx.read("hw")
print "\n\nGetting XATTR 'lang' from object 'hw'" 
print ioctx.get_xattr("hw", "lang")
print "\nContents of object 'bm'\n------------------------" 
print ioctx.read("bm")
print "Getting XATTR 'lang' from object 'bm'" 
print ioctx.get_xattr("bm", "lang")
'''
print "Removing object 'bm'" 
ioctx.remove_object("bm")
print "Removing object 'bm1'" 
ioctx.remove_object("bm1")
print "\nRemoving object 'hw'" 
ioctx.remove_object("hw")

A modified script to debug this issue:- ####################################
import rados
import sys
try:
cluster = rados.Rados(conffile='/etc/ceph/ceph.conf')
except TypeError as e:
print 'Argument validation error: ', e
raise e
print "Created cluster handle."
try:
cluster.connect()
except Exception as e:
print "connection error: ", e
raise e
finally:
print "Connected to the cluster."
print "\n\nI/O Context and Object Operations"
print "================================="
print "\nCreating a context for the 'data' pool"
if not cluster.pool_exists('data'):
raise RuntimeError('No data pool exists')
ioctx = cluster.open_ioctx('data')
s9mb='a'*10000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s9mb)/1024/1024;
try:
print "Writing object s1 to pool 'data'."
ioctx.write("s9mb", s9mb)
ioctx.set_xattr("s9mb", "lang", "en_US")
print 'done'
ioctx.remove_object("s9mb")
except Exception as e :
print 'failed'+str(e)
s90mb='ababababa'*10000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s90mb)/1024/1024;
try:
print "Writing object s90mb to pool 'data'."
ioctx.write("s90mb", s90mb)
ioctx.set_xattr("s90mb", "lang", "en_US")
print 'done'
ioctx.remove_object("s90mb")
except Exception as e :
print 'failed'+str(e)
s100mb='abababababa'*10000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s100mb)/1024/1024;
try:
print "Writing object s100mb to pool 'data'."
ioctx.write("s100mb", s100mb)
ioctx.set_xattr("s100mb", "lang", "en_US")
print 'done'
ioctx.remove_object("s100mb")
except Exception as e :
print 'failed'+str(e)
s1G='abababababa'*100000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s1G)/1024/1024;
try:
print "Writing object 1G to pool 'data'."
ioctx.write("s1G", s1G)
ioctx.set_xattr("s1G", "lang", "en_US")
print 'done'
ioctx.remove_object("s1G")
except Exception as e :
print 'failed'+str(e)
s10G='ababaabababa'*1000000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s10G)/1024/1024;
try:
print "Writing object s10G to pool 'data'."
ioctx.write("s10G", s10G)
ioctx.set_xattr("s10G", "lang", "en_US")
print 'done'
ioctx.remove_object("s10G")
except Exception as e :
print 'failed'+str(e)
s20G='ababaabababaababababababab'*1000000000
print '\n\nsize in mb of str is: '
print sys.getsizeof(s20G)/1024/1024;
try:
print "Writing object s20G to pool 'data'."
ioctx.write("s20G", s20G)
ioctx.set_xattr("s20G", "lang", "en_US")
print 'done'
ioctx.remove_object("s20G")
except Exception as e :
print 'failed'+str(e) ########################
I got following output: #######################
ems@rack6-client-2:~$ python ceph-client.py
Created cluster handle.
Connected to the cluster.
I/O Context and Object Operations =================================
Creating a context for the 'data' pool
size in mb of str is:
9
Writing object s1 to pool 'data'.
done
size in mb of str is:
85
Writing object s90mb to pool 'data'.
done
size in mb of str is:
104
Writing object s100mb to pool 'data'.
failedIoctx.write(data): failed to write s100mb: errno EMSGSIZE
size in mb of str is:
1049
Writing object 1G to pool 'data'.
failedIoctx.write(data): failed to write s1G: errno EMSGSIZE
size in mb of str is:
11444
Writing object s10G to pool 'data'.
common/Throttle.cc: In function 'bool Throttle::get_or_fail(int64_t)' thread 7fb047788700 time 2014-09-26 17:44:49.495822
common/Throttle.cc: 194: FAILED assert(c >= 0)
ceph version 0.84-sd-sprint4 (3215c520e1306f50d0094b5646636c02456c9df4)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7fb04a6f571b]
2: (Throttle::get_or_fail(long)+0x5ae) [0x7fb04a6ebd0e]
3: (Objecter::_throttle_op(Objecter::Op*, int)+0x68) [0x7fb04a666588]
4: (Objecter::_op_submit_with_budget(Objecter::Op*, RWLock::Context&)+0x195) [0x7fb04a677785]
5: (Objecter::op_submit(Objecter::Op*)+0x96) [0x7fb04a6778c6]
6: (librados::IoCtxImpl::operate(object_t const&, ObjectOperation*, long*, int)+0x33d) [0x7fb04a650a0d]
7: (librados::IoCtxImpl::write(object_t const&, ceph::buffer::list&, unsigned long, unsigned long)+0x18d) [0x7fb04a656a2d]
8: (rados_write()+0xc1) [0x7fb04a624ee1]
9: (ffi_call_unix64()+0x4c) [0x7fb04d3a3adc]
10: (ffi_call()+0x1fc) [0x7fb04d3a340c]
11: (_ctypes_callproc()+0x48e) [0x7fb04d5ba5fe]
12: (()+0x15f9e) [0x7fb04d5bbf9e]
13: (PyEval_EvalFrameEx()+0x1f36) [0x52e1e6]
14: (PyEval_EvalFrameEx()+0xc82) [0x52cf32]
15: (PyEval_EvalFrameEx()+0xc82) [0x52cf32]
16: python() [0x56d0aa]
17: python() [0x4d9854]
18: (PyEval_CallObjectWithKeywords()+0x6b) [0x4da20b]
19: python() [0x5872b2]
20: (()+0x8182) [0x7fb04e644182]
21: (clone()+0x6d) [0x7fb04e37130d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
terminate called after throwing an instance of 'ceph::FailedAssertion'
Aborted (core dumped) ######################
Core dumps are attached.

https://github.com/ceph/ceph/pull/2584 should be enough. Unless there is a good reason to write an object with chunks larger than 1GB ? That is : if there is a need to create a 20GB object, is there a reason to create it with rados_write of buffers larger than 1GB ? Or is it reasonable to use smaller buffers (100MB at a time or less) ?