From 16fc3b32e3511d194141ef955b5822c68945ec5d Mon Sep 17 00:00:00 2001
From: Alan Somers <asomers@gmail.com>
Date: Fri, 1 Nov 2013 15:39:15 -0600
Subject: [PATCH] BSD-Linux interoperability

Ceph's networking protocol passes the in-memory representation of struct
sockaddr_storage over the wire, even though that structure is
operating-system and architecture specific.  This commit creates a
portably-defined struct ceph_sockaddr_storage which is identical to struct
sockaddr_storage on Linux amd64, along with appropriate encode and decode
functions.  This allows FreeBSD and Linux machines to coexist on the same
Ceph cluster.

Signed-off-by: Alan Somers <asomers@gmail.com>
---
 configure.ac         |   17 +++++
 src/include/msgr.h   |   15 +++-
 src/msg/msg_types.cc |    3 +
 src/msg/msg_types.h  |   90 ++++++++++++++++-------
 src/test/encoding.cc |  199 ++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 298 insertions(+), 26 deletions(-)

diff --git a/configure.ac b/configure.ac
index c344e7d..53e684f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -123,6 +123,23 @@ if test x"$linux" = x"yes"; then
 fi
 AC_CHECK_FUNCS([syncfs], AC_DEFINE([HAVE_SYS_SYNCFS], [1], [we have syncfs]), [])
 
+# Checks related to struct sockaddr_storage
+AC_CACHE_CHECK([for ss_len field in struct sockaddr_storage],
+		ac_cv_have_ss_len_in_struct_ss, [
+	AC_TRY_COMPILE(
+		[
+#include <sys/types.h>
+#include <sys/socket.h>
+		],
+		[ struct sockaddr_storage s; s.ss_len = 1; ],
+		[ ac_cv_have_ss_len_in_struct_ss="yes" ],
+		[ ac_cv_have_ss_len_in_struct_ss="no" ],
+	)
+])
+if test "x$ac_cv_have_ss_len_in_struct_ss" = "xyes" ; then
+	AC_DEFINE(HAVE_SS_LEN_IN_SOCKADDR_STORAGE, 1, [Does struct sockaddr_storage have ss_len?])
+fi
+
 # Find some crypto library for us to use, while letting user to decide which one to use.
 AC_ARG_WITH([cryptopp],
 	[AS_HELP_STRING([--with-cryptopp], [Use cryptographic functions from cryptopp])],
diff --git a/src/include/msgr.h b/src/include/msgr.h
index 5854682..3621850 100644
--- a/src/include/msgr.h
+++ b/src/include/msgr.h
@@ -58,13 +58,26 @@ struct ceph_entity_name {
 
 extern const char *ceph_entity_type_name(int type);
 
+
+/*
+ * A portable version of sockaddr_storage.  For backwards compatibility, it is
+ * defined binary compatibly with Linux 3.2.0 amd64.
+ * On Linux amd64 this must be castable to a struct sockaddr_storage.  On
+ * other platforms, it must be translated.
+ */
+struct ceph_sockaddr_storage {
+	__be16 ss_family;            /* Big endian */
+	__u8   __ss_padding[126];
+} __attribute__ ((__packed__));
+
+
 /*
  * entity_addr -- network address
  */
 struct ceph_entity_addr {
 	__le32 type;
 	__le32 nonce;  /* unique id for process (e.g. pid) */
-	struct sockaddr_storage in_addr;
+	struct ceph_sockaddr_storage in_addr;
 } __attribute__ ((packed));
 
 struct ceph_entity_inst {
diff --git a/src/msg/msg_types.cc b/src/msg/msg_types.cc
index b02db76..f50b20c 100644
--- a/src/msg/msg_types.cc
+++ b/src/msg/msg_types.cc
@@ -97,6 +97,9 @@ bool entity_addr_t::parse(const char *s, const char **end)
   } else {
     return false;
   }
+#if HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  addr.ss_len = addr_size();
+#endif
 
   if (brackets) {
     if (*p != ']')
diff --git a/src/msg/msg_types.h b/src/msg/msg_types.h
index 40b3f0b..9497001 100644
--- a/src/msg/msg_types.h
+++ b/src/msg/msg_types.h
@@ -17,6 +17,7 @@
 
 #include <netinet/in.h>
 
+#include "acconfig.h"
 #include "include/types.h"
 #include "include/blobhash.h"
 #include "include/encoding.h"
@@ -150,31 +151,72 @@ namespace __gnu_cxx {
 }
 
 
+// Convert a native sockaddr_storage to a portable ceph_sockaddr_storage
+static inline void encode(const sockaddr_storage& src, ceph_sockaddr_storage& dst) {
+  size_t copylen, zerolen;
 
-/*
- * an entity's network address.
- * includes a random value that prevents it from being reused.
- * thus identifies a particular process instance.
- * ipv4 for now.
- */
+  //Copy the entire structure, then deal with the ss_len and ss_family fields.
+  //It's probably no slower than copying everything except those two fields.
+  //In case dst is longer than src, zero out its tail
+  copylen = MIN(sizeof(src), sizeof(dst));
+  zerolen = MAX(0, sizeof(dst) - sizeof(src));
+  memmove(&dst, &src, copylen);
+  bzero((uint8_t*)&dst + copylen, zerolen);
+  dst.ss_family = htons((short)src.ss_family);
+  //Note: when encoding, we can ignore ss_len.  It is always 0 on the wire
+}
+
+// Convert a portable ceph_sockaddr_storage to a native sockaddr_storage 
+// Note: src and dst  may overlap!
+static inline void decode(sockaddr_storage& dst, const ceph_sockaddr_storage& src) {
+  size_t copylen, zerolen;
+
+  copylen = MIN(sizeof(dst), sizeof(struct ceph_sockaddr_storage));
+  zerolen = MAX(0, sizeof(dst) - sizeof(struct ceph_sockaddr_storage));
+  memmove(&dst, &src, copylen);
+  bzero((uint8_t*)&dst + copylen, zerolen);
+  dst.ss_family = ntohs(
+                reinterpret_cast<struct ceph_sockaddr_storage&>(dst).ss_family);
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  // Since ss_len is not sent over the wire, we must calculate it here.
+  // There are dozens of address families, but we only care about those that
+  // may be used as a Ceph network.
+  switch(dst.ss_family) {
+    case AF_INET:
+      dst.ss_len = sizeof(struct sockaddr_in);
+      break;
+    case AF_INET6:
+      dst.ss_len = sizeof(struct sockaddr_in6);
+      break;
+    default:
+      dst.ss_len = sizeof(struct sockaddr_storage);
+      break;
+  }
+#endif
+}
 
 /*
  * encode sockaddr.ss_family in big endian
  */
-static inline void encode(const sockaddr_storage& a, bufferlist& bl) {
-  struct sockaddr_storage ss = a;
-#if !defined(__FreeBSD__)
-  ss.ss_family = htons(ss.ss_family);
-#endif
-  ::encode_raw(ss, bl);
+static inline void encode(const sockaddr_storage& src, bufferlist& bl) {
+  struct ceph_sockaddr_storage imm;
+
+  ::encode(src, imm);
+  ::encode_raw(imm, bl);
 }
-static inline void decode(sockaddr_storage& a, bufferlist::iterator& bl) {
-  ::decode_raw(a, bl);
-#if !defined(__FreeBSD__)
-  a.ss_family = ntohs(a.ss_family);
-#endif
+
+static inline void decode(sockaddr_storage& dst, bufferlist::iterator& bl) {
+  struct ceph_sockaddr_storage imm;
+  ::decode_raw(imm, bl);
+  ::decode(dst, imm);
 }
 
+/*
+ * an entity's network address.
+ * includes a random value that prevents it from being reused.
+ * thus identifies a particular process instance.
+ * ipv4 for now.
+ */
 struct entity_addr_t {
   __u32 type;
   __u32 nonce;
@@ -202,10 +244,7 @@ struct entity_addr_t {
   entity_addr_t(const ceph_entity_addr &o) {
     type = o.type;
     nonce = o.nonce;
-    addr = o.in_addr;
-#if !defined(__FreeBSD__)
-    addr.ss_family = ntohs(addr.ss_family);
-#endif
+    ::decode(addr, o.in_addr);
   }
 
   __u32 get_nonce() const { return nonce; }
@@ -216,6 +255,9 @@ struct entity_addr_t {
   }
   void set_family(int f) {
     addr.ss_family = f;
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+    addr.ss_len = addr_size();
+#endif
   }
   
   sockaddr_storage &ss_addr() {
@@ -276,10 +318,8 @@ struct entity_addr_t {
     ceph_entity_addr a;
     a.type = 0;
     a.nonce = nonce;
-    a.in_addr = addr;
-#if !defined(__FreeBSD__)
-    a.in_addr.ss_family = htons(addr.ss_family);
-#endif
+    ::encode(addr, a.in_addr);
+
     return a;
   }
 
diff --git a/src/test/encoding.cc b/src/test/encoding.cc
index 2662c24..724cf4b 100644
--- a/src/test/encoding.cc
+++ b/src/test/encoding.cc
@@ -1,9 +1,15 @@
 #include "common/config.h"
 #include "include/buffer.h"
 #include "include/encoding.h"
+#include "msg/msg_types.h"
 
 #include "gtest/gtest.h"
 
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
 template < typename T >
 static void test_encode_and_decode(const T& src)
 {
@@ -196,3 +202,196 @@ TEST(EncodingRoundTrip, MultimapConstructorCounter) {
   EXPECT_EQ(my_val_t::get_copy_ctor(), 10);
   EXPECT_EQ(my_val_t::get_assigns(), 0);
 }
+
+inline bool operator==(struct sockaddr_storage l, struct sockaddr_storage r) {
+  return (0 == memcmp(&l, &r, sizeof(l)));
+}
+
+inline bool operator==(struct sockaddr_storage& l, struct sockaddr_storage& r) {
+  return (0 == memcmp(&l, &r, sizeof(l)));
+}
+
+
+TEST(EncodingRoundTrip, struct_sockaddr_in) {
+  struct sockaddr_storage ss;
+  struct sockaddr_in &sin = reinterpret_cast<struct sockaddr_in&>(ss);
+
+  // encode may skip don't care fields.  For testing purposes, we zero them.
+  bzero(&ss, sizeof(ss));
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  sin.sin_len = sizeof(sin);
+#endif
+  sin.sin_family = AF_INET;
+  sin.sin_port = htons(0x9abc);
+  EXPECT_EQ(1, inet_pton(AF_INET, "18.52.86.120", &sin.sin_addr));
+
+  test_encode_and_decode < struct sockaddr_storage >(ss);
+}
+
+TEST(EncodingRoundTrip, struct_sockaddr_in6) {
+  struct sockaddr_storage ss;
+  struct sockaddr_in6 &sin6 = reinterpret_cast<struct sockaddr_in6&>(ss);
+
+  // encode may skip don't care fields.  For testing purposes, we zero them.
+  bzero(&ss, sizeof(ss));
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  sin6.sin6_len = sizeof(sin6);
+#endif
+  sin6.sin6_family = AF_INET6;
+  sin6.sin6_port = htons(0x9abc);
+  sin6.sin6_scope_id = 0xe;    //Global scope
+  sin6.sin6_flowinfo = 0xfedcb;
+  EXPECT_EQ(1, inet_pton(AF_INET6, "2001:1234:5678:9abc:def0:cba9:8765:4321",
+      &sin6.sin6_addr));
+  test_encode_and_decode < struct sockaddr_storage >(ss);
+}
+
+// Test that we can encode a struct ceph_sockaddr_storage and get the same
+// binary layout that Linux amd64 produces when casting a sockaddr_in to
+// sockaddr_storage
+TEST(SockaddrStoragePortability, encode_in) {
+  bufferlist bl;
+  struct sockaddr_storage ss;
+  struct sockaddr_in &sin = reinterpret_cast<struct sockaddr_in&>(ss);
+  uint8_t encoded[128];
+  uint8_t expected[128] = {
+    0,                                              //Upper byte of sin_family
+    AF_INET,                                        //Lower byte of sin_family
+    0x9a, 0xbc,                                     //sin_port in big endian
+    0x12, 0x34, 0x56, 0x78,                         //sin_addr in big endian
+    0, 0, 0, 0,                                     //sin_zero
+    0, 0, 0, 0, 0, 0, 0, 0,                         //__ss_padding[8:16]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[16:32]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[32:64]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[64:80]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[80:96]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[96:112]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[112:128]
+  };
+
+  // encode may skip don't care fields.  For testing purposes, we zero them.
+  bzero(&ss, sizeof(ss));
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  sin.sin_len = sizeof(sin);
+#endif
+  sin.sin_family = AF_INET;
+  sin.sin_port = htons(0x9abc);
+  EXPECT_EQ(1, inet_pton(AF_INET, "18.52.86.120", &sin.sin_addr));
+  encode(ss, bl);
+  EXPECT_EQ(sizeof(expected), bl.length());
+  bl.copy(0, 128, (char*)encoded);
+  EXPECT_EQ(0, memcmp(expected, encoded, 128));
+}
+
+// Test that we can encode a struct ceph_sockaddr_storage and get the same
+// binary layout that Linux amd64 produces when casting a sockaddr_in6 to
+// sockaddr_storage
+TEST(SockaddrStoragePortability, encode_in6) {
+  bufferlist bl;
+  struct sockaddr_storage ss;
+  struct sockaddr_in6 &sin6 = reinterpret_cast<struct sockaddr_in6&>(ss);
+  uint8_t encoded[128];
+  uint8_t expected[128] = {
+    0,                                              //Upper byte of sin6_family
+    AF_INET6,                                       //Lower byte of sin6_family
+    0x9a, 0xbc,                                     //sin6_port in big endian
+    0xcb, 0xed, 0x0f, 0x00,                         //sin6_flowinfo in LE
+    0x20, 0x01, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, //sin6_addr[0:8]
+    0xde, 0xf0, 0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21, //sin6_addr[8:16]
+    0x0e, 0x00, 0x00, 0x00,                         //sin6_scope_id in LE
+    0, 0, 0, 0,                                     //__ss_padding[12:16]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[16:32]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[32:64]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[64:80]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[80:96]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[96:112]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[112:128]
+  };
+
+  // encode may skip don't care fields.  For testing purposes, we zero them.
+  bzero(&ss, sizeof(ss));
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  sin6.sin6_len = sizeof(sin6);
+#endif
+  sin6.sin6_family = AF_INET6;
+  sin6.sin6_port = htons(0x9abc);
+  sin6.sin6_scope_id = 0xe;    //Global scope
+  sin6.sin6_flowinfo = 0xfedcb;
+  EXPECT_EQ(1, inet_pton(AF_INET6, "2001:1234:5678:9abc:def0:cba9:8765:4321",
+      &sin6.sin6_addr));
+
+  encode(ss, bl);
+  EXPECT_EQ(sizeof(expected), bl.length());
+  bl.copy(0, 128, (char*)encoded);
+  EXPECT_EQ(0, memcmp(expected, encoded, 128));
+}
+
+// Test that we can decode a struct ceph_sockaddr_storage using the binary
+// layout of Linux amd64 and get the correct sockaddr_in
+TEST(SockaddrStoragePortability, decode_in) {
+  bufferlist bl;
+  struct sockaddr_storage ss;
+  struct sockaddr_in &sin = reinterpret_cast<struct sockaddr_in&>(ss);
+  struct in_addr expected_addr;
+  uint8_t encoded[128] = {
+    0,                                              //Upper byte of sin_family
+    AF_INET,                                        //Lower byte of sin_family
+    0x9a, 0xbc,                                     //sin_port in big endian
+    0x12, 0x34, 0x56, 0x78,                         //sin_addr in big endian
+    0, 0, 0, 0,                                     //sin_zero
+    0, 0, 0, 0, 0, 0, 0, 0,                         //__ss_padding[8:16]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[16:32]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[32:64]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[64:80]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[80:96]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[96:112]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[112:128]
+  };
+  bl.append((char*)encoded, 128);
+
+  decode(ss, bl);
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  EXPECT_EQ(sizeof(sin), sin.sin_len);
+#endif
+  EXPECT_EQ(AF_INET, sin.sin_family);
+  EXPECT_EQ(0x9abc, ntohs(sin.sin_port));
+  EXPECT_EQ(1, inet_pton(AF_INET, "18.52.86.120", &expected_addr));
+  EXPECT_EQ(expected_addr.s_addr, sin.sin_addr.s_addr);
+}
+
+// Test that we can decode a struct ceph_sockaddr_storage using the binary
+// layout of Linux amd64 and get the correct sockaddr_in6
+TEST(SockaddrStoragePortability, decode_in6) {
+  bufferlist bl;
+  struct sockaddr_storage ss;
+  struct sockaddr_in6 &sin6 = reinterpret_cast<struct sockaddr_in6&>(ss);
+  struct in6_addr expected_addr;
+  uint8_t encoded[128] = {
+    0,                                              //Upper byte of sin6_family
+    AF_INET6,                                       //Lower byte of sin6_family
+    0x9a, 0xbc,                                     //sin6_port in big endian
+    0xcb, 0xed, 0x0f, 0x00,                         //sin6_flowinfo in LE
+    0x20, 0x01, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, //sin6_addr[0:8]
+    0xde, 0xf0, 0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21, //sin6_addr[8:16]
+    0x0e, 0x00, 0x00, 0x00,                         //sin6_scope_id in LE
+    0, 0, 0, 0,                                     //__ss_padding[12:16]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[16:32]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[32:64]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[64:80]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[80:96]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[96:112]
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //__ss_padding[112:128]
+  };
+  bl.append((char*)encoded, 128);
+
+  decode(ss, bl);
+#ifdef HAVE_SS_LEN_IN_SOCKADDR_STORAGE
+  EXPECT_EQ(sizeof(sin6), sin6.sin6_len);
+#endif
+  EXPECT_EQ(AF_INET6, sin6.sin6_family);
+  EXPECT_EQ(0x9abc, ntohs(sin6.sin6_port));
+  EXPECT_EQ(0xeu, sin6.sin6_scope_id);
+  EXPECT_EQ(0xfedcbu, sin6.sin6_flowinfo);
+  EXPECT_EQ(1, inet_pton(AF_INET6, "2001:1234:5678:9abc:def0:cba9:8765:4321", &expected_addr));
+  EXPECT_EQ(0, memcmp(&expected_addr, &sin6.sin6_addr, sizeof(expected_addr)));
+}
-- 
1.7.9.5

