Protocol code to encode/decode various data types

Question

I need to create a protocol for sending data of various types over a socket connection. so I need to serialise the data to a byte stream. I only need signed and unsigned 32 bit integers, 64 bit integers, string and binary. It is only a start but if anyone can code review this I would really appreciate it.

/* custom tlv protocol. byte1=type, bytes2-5=size, remaining=payload
   Following types:
   int32_t  //signed int 32 bit
   uint32_t //unsigned int 32 bit
   uint64_t //unsigned int 64 bit
   string - char* - int8_t*
   byte*  - unsigned char* -anything binary

   Everything uses TLV.  each message must contain type as first byte

   currently a message is one tlv but expand so a message is a linked list of tlvs
*/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>



/* include stdint.h but as this is demo just use these here */
typedef unsigned char        uint8_t;
typedef signed int           int32_t;
typedef unsigned int        uint32_t;

#ifdef WIN32
typedef unsigned __int64   uint64_t;
#else
typedef unsigned int64_t   uint64_t;
#endif

enum data_type {DTYPE_S32 = 0, DTYPE_U32 = 1, DTYPE_U64 = 2, DTYPE_STRING = 3, DTYPE_BINARY = 4 };

struct tlv_msg
{
 data_type datatype;   /* datatypes - 5 types */
 /* payload stored in a union */
 union{
    int32_t        s32val;   /*  signed int 1 */
    uint32_t       u32val;   /*  2 */
    uint64_t       u64val;   /*  3 */
    char*          strval;   /*  4 strings */
    unsigned char* binval;   /*  5 any binary data */
 };

 uint32_t bytelen;  /* no. bytes of union/data part */
};

size_t tlv_encode(data_type type, uint64_t input_length, tlv_msg* inputdata, 
                            unsigned char** outputdata, size_t *output_length);

/* allocation/de-allocation of memory */
size_t alloc_encode(data_type type, unsigned char** outputdata, size_t datasize = 0);
size_t alloc_decode(unsigned char* inputdata, tlv_msg** msg);

void free_encode(unsigned char** outputdata);
void free_decode(tlv_msg** msg);


size_t tlv_decode(unsigned char* inputdata, tlv_msg** msg);

void printbytes(unsigned char* bytes, size_t length);
void printmessage(tlv_msg* msg);

/* testing functions */
void tests32();
void testu32();
void testu64();
void teststring();
void testbinary();


int main() 
{ 
   //test encode/decond of various data values
   tests32();
   testu32();
   testu64();
   teststring();
   testbinary();

   return 0; 
} 



size_t tlv_encode(data_type type, uint64_t input_length, tlv_msg* inputdata, 
                            unsigned char** outputdata, size_t *output_length)
{
    if(!outputdata)
        return 0;  

    //1 byte for type, 4 for length, plus data size
   *output_length = (uint32_t)(1 + sizeof(uint32_t) + input_length);

    unsigned char* p = *outputdata;

   *p++ = (unsigned char)type;

   for (int i = sizeof(uint32_t) - 1; i >= 0; --i)
      *p++ = (unsigned char) ((input_length >> (i * 8)) & 0xFF);

   /* next job is to append actual data on end */
   while(input_length--) {
       switch(inputdata->datatype) {
       case  DTYPE_S32: *p++ = (unsigned char)(inputdata->s32val >> (input_length * 8) & 0xFF); break;
       case  DTYPE_U32: *p++ = (unsigned char)(inputdata->u32val >> (input_length * 8) & 0xFF); break;
       case  DTYPE_U64: *p++ = (unsigned char)(inputdata->u64val >> (input_length * 8) & 0xFF); break;
       case  DTYPE_STRING: *p++ = *inputdata->strval++; break;
       case  DTYPE_BINARY: *p++ = *inputdata->binval++; break;
      }
    }

    return *output_length;
}


size_t tlv_decode(unsigned char* inputdata, tlv_msg** msg) {
    if(!msg)
        alloc_decode(inputdata, msg);

    unsigned char* p = inputdata;

    /* skip first 5 bytes */
    for(int i = 0; i < 5; ++i)
       *p++;


   int length = (*msg)->bytelen - 5;

   while(length--) {

     switch((*msg)->datatype) {
        case DTYPE_S32:
          (*msg)->s32val += *p++ << (length * 8);
        break;
        case DTYPE_U32:
          (*msg)->u32val +=  *p++ << (length * 8);
        break;
        case DTYPE_U64:
          (*msg)->u64val += *p++ << (length * 8);
        break;
        case DTYPE_STRING:
         *(*msg)->strval++ = (char)*p++;
        break;
        case DTYPE_BINARY:
         *(*msg)->binval++ = (unsigned char)*p++;
        break;
        default:
          printf("tlv_decodegeneric error!!! unrecognised datatype %u\n", (*msg)->datatype);
        break;
     }
  }

  /* rewind to beginning of strings */
  switch((*msg)->datatype) {
  case DTYPE_STRING:
      length = (*msg)->bytelen-5;
      while(length--)
        *(*msg)->strval--;

      break;
  case DTYPE_BINARY:
      length = (*msg)->bytelen-5;
      while(length--)
        *(*msg)->binval--;          
      break;

  }


  return 0;
}



/* allocate for new message to be encoded */
size_t alloc_encode(data_type type, unsigned char** outputdata, size_t datasize) {
  size_t allosize = datasize;

  if(allosize == 0) {
    switch(type) {
      case  DTYPE_S32: allosize = 4; break;
      case  DTYPE_U32: allosize = 4; break;
      case  DTYPE_U64: allosize = 8; break;
      default:
         printf("alloc_encode error!!! no datasize for data type %u\n", type);
         return 0;
    }
  }

  allosize += 5;  /* append type and size fields */


  *outputdata = (unsigned char*)malloc(allosize);
  return allosize;
}

size_t alloc_decode(unsigned char* inputdata, tlv_msg** msg) {

   /* check if already malloc'd and if so free and realloc */
   if(*msg)
      printf("alloc_decode already called");

   /* length of data is in bytes 1,2,3,4 */
   size_t sz = 0;

   unsigned char* p = inputdata;

   /*first byte is type */
   unsigned char type = *p++;

   for(int i = 0; i < 4; ++i)
     sz += (unsigned char) (*p++ >> ((3-i) * 8) & 0xFF);

   if(!sz)
       printf("ERROR! zero bytes size found in input data\n");

   *msg = (tlv_msg*)malloc(sizeof(tlv_msg));

   if(*msg) {
       (*msg)->bytelen = sz + 5; /* plus size for header type and length */
       (*msg)->datatype = (data_type)type;

       if(type == DTYPE_STRING)
           (*msg)->strval = (char*)malloc((*msg)->bytelen);
       else if(type == DTYPE_BINARY)
           (*msg)->binval = (unsigned char*)malloc((*msg)->bytelen);


       /* set safe defaults */
       switch(type) {
        case 0: (*msg)->s32val = 0; break; 
        case 1: (*msg)->u32val = 0; break; 
        case 2: (*msg)->u64val = 0; break; 
        case 3: *(*msg)->strval = 0; break; 
        case 4: *(*msg)->binval = 0; break; 
      }
   }

   /* add on 5 for type and length */
   sz += 5;


   return sz;
}

void free_encode(unsigned char** outputdata) {
  free(*outputdata);
}

void free_decode(tlv_msg** msg) {
  switch((*msg)->datatype) {
    case DTYPE_STRING:  free((*msg)->strval); break;
    case DTYPE_BINARY:  free((*msg)->binval); break;
  }
  free(*msg);
}



void printmessage(tlv_msg* msg) {
   switch(msg->datatype) {
   case DTYPE_S32: printf("%i\n", msg->s32val); break;
   case DTYPE_U32: printf("%u\n", msg->u32val); break;
   case DTYPE_U64: printf("%u\n", msg->u64val); break;
   case DTYPE_STRING: printf("%s\n", msg->strval); break;
   case DTYPE_BINARY: printbytes(msg->binval, msg->bytelen-5); break;
   default:  printf("unknown data type\n"); break;
   }
}

void printbytes(unsigned char* bytes, size_t length) {
   for(size_t i = 0; i < length; ++i) {
     char c = 0;
     div_t stResult = div(bytes[i], 16);
     printf("%X%X ", stResult.quot, stResult.rem);
   }
   printf("\n");
}

void teststring() {
  /* test string type encode/decode */

tlv_msg msgstr; 
msgstr.datatype = DTYPE_STRING;  /* ie string type */

char* hellomsg = "Hello World!";

printf("testing string encode/decode, value to encode: %s\n", hellomsg);

size_t szlen = strlen(hellomsg)+1;
msgstr.strval = (char*)malloc(szlen);
strcpy(msgstr.strval, hellomsg);
msgstr.bytelen = szlen;

size_t output_length = 0;
unsigned char* outputmsg = 0;
size_t ret = alloc_encode(DTYPE_STRING, &outputmsg, msgstr.bytelen);
tlv_encode(msgstr.datatype, msgstr.bytelen, &msgstr, &outputmsg, &output_length);

/* print bytes as hex */
printbytes(outputmsg, output_length);


tlv_msg* decode_msg = 0;
size_t bytes = alloc_decode(outputmsg, &decode_msg);
tlv_decode(outputmsg, &decode_msg);
printmessage(decode_msg);

printf("completed testing string encode/decode, decoded value: %s\n", decode_msg->strval);


free_encode(&outputmsg);
free_decode(&decode_msg);


}

void tests32() {
  /* test signed int 32 bit type encode/decode */
  tlv_msg msg; 
  msg.datatype = DTYPE_S32;

  msg.bytelen = 4;
  msg.s32val = -65999;

  printf("testing s32 encode/decode, input value: %d\n", msg.s32val);


  size_t output_length = 0;
  unsigned char* outputmsg = 0;
  size_t ret = alloc_encode(DTYPE_S32, &outputmsg, msg.bytelen);
  tlv_encode(msg.datatype, msg.bytelen, &msg, &outputmsg, &output_length);

  /* print bytes as hex */
  printf("Hex string: ");
  printbytes(outputmsg, output_length);


  tlv_msg* decode_msg = 0;
  size_t bytes = alloc_decode(outputmsg, &decode_msg);
  tlv_decode(outputmsg, &decode_msg);
  printmessage(decode_msg);

  printf("decoded u32 value: %d\n", decode_msg->s32val);

  free_encode(&outputmsg);
  free_decode(&decode_msg);
}

void testu32() {
  /* test unsigned int 32 bit type encode/decode */
   tlv_msg msg; 
   msg.datatype = DTYPE_U32;

   msg.bytelen = 4;
   msg.u32val = 0xFFFFFFFF;  //257;

   printf("testing u32 encode/decode, input value: %u\n", msg.u32val);

   size_t output_length = 0;
   unsigned char* outputmsg = 0;
   size_t ret = alloc_encode(DTYPE_U32, &outputmsg, msg.bytelen);
   tlv_encode(msg.datatype, msg.bytelen, &msg, &outputmsg, &output_length);

   /* print bytes as hex */
   printf("Hex string: ");
   printbytes(outputmsg, output_length);

   tlv_msg* decode_msg = 0;
   size_t bytes = alloc_decode(outputmsg, &decode_msg);
   tlv_decode(outputmsg, &decode_msg);
   printmessage(decode_msg);

   printf("decoded u32 value: %u\n", decode_msg->u32val);

   free_encode(&outputmsg);
   free_decode(&decode_msg);
}

void testu64() {
  /* test unsigned int 64 bit type encode/decode */
   tlv_msg msg; 
   msg.datatype = DTYPE_U64;

   msg.bytelen = 8;
   msg.u64val = 0xFFFF;

   printf("testing u64 encode/decode, input value: %u\n", msg.u64val);


   size_t output_length = 0;
   unsigned char* outputmsg = 0;
   size_t ret = alloc_encode(DTYPE_U64, &outputmsg, msg.bytelen);
   tlv_encode(msg.datatype, msg.bytelen, &msg, &outputmsg, &output_length);

  /* print bytes as hex */
  printf("Hex string: ");
  printbytes(outputmsg, output_length);


  tlv_msg* decode_msg = 0;
  size_t bytes = alloc_decode(outputmsg, &decode_msg);
  tlv_decode(outputmsg, &decode_msg);
  printmessage(decode_msg);

  printf("decoded u64 value: %u\n", decode_msg->u64val);

  free_encode(&outputmsg);
  free_decode(&decode_msg);
}

void testbinary(){
   tlv_msg msgstr; 
   msgstr.datatype = DTYPE_BINARY;  /* ie binary data type */

   unsigned char binmsg[] = { 'a', 'b', 'c', '\0', 'a', 'b', 'c', '\0' };

   printf("testing binary encode/decode, value to encode is abcnullabcnull\n");

   msgstr.binval = (unsigned char*)malloc(sizeof(binmsg));
   memcpy(msgstr.binval, binmsg, sizeof(binmsg));
   msgstr.bytelen = sizeof(binmsg);

   size_t output_length = 0;
   unsigned char* outputmsg = 0;
   size_t ret = alloc_encode(DTYPE_BINARY, &outputmsg, msgstr.bytelen);
   tlv_encode(msgstr.datatype, msgstr.bytelen, &msgstr, &outputmsg, &output_length);

   /* print bytes as hex */
   printbytes(outputmsg, output_length);


   tlv_msg* decode_msg = 0;
   size_t bytes = alloc_decode(outputmsg, &decode_msg);
   tlv_decode(outputmsg, &decode_msg);
   printmessage(decode_msg);

   //not sure a printout is required for binary

   free_encode(&outputmsg);
   free_decode(&decode_msg);
}

look up htonl() and family of functions. – Loki Astari Nov 26 '11 at 22:24 — Loki Astari, Nov 26 '11 at 22:24

Mike Nakis · Accepted Answer · 2015-09-22 10:54:49Z

If I were to see a point in converting data types to a message, that could only be to be able to send such messages across the network without having to perform any further interpretation of their internal structure. However, your scheme produces messages that still require interpretation before they can be sent: the type of the message has to be examined in order to figure out how to transmit its payload, since the message will contain the actual data in the case of fixed-length types, but a pointer to the data in the case of variable-length types.

I think it would make a lot more sense to construct your messages in such a way that they can be sent as binary blocks, without having to know anything more than their length.

A message is supposed to be nothing else but two things: a length, and a body of data. When you are thinking at the message level and writing functions that send and receive messages, you should not be concerned with the internal structure of the data contained within the messages. Therefore, the notion of a "data type" is not pertinent at this level.

At the level where you have just received a block of data of a known length, you can now start interpreting it in order to extract the payload from it. The first byte can contain the data type enum, while the remaining bytes can be the actual value.

In the case of fixed-length data types, assertions can be in place to ensure that the length of the data is equal to the size of the value plus one for the first byte which contained the data type enum.
For variable-length data types, the length of the value is the length of the data minus one for the first byte which contained the data type enum.

An added advantage of doing things this way is that you only have to make one memory allocation per message, not two, one for the message struct and one for the variable-length data contained therein. Fewer allocations equal reduced memory fragmentation, which can be extremely important if your application runs for a long time.

Another consideration you should bear in mind is whether your underlying transport mechanism offers efficient buffering or not. If, by any chance, it does offer efficient buffering, then you might gain performance by reading/writing bytes directly to/from the transport mechanism, instead of allocating and filling message structures.

So, I do not have specific points to address in your code, because in my opinion it needs to be completely re-written.

Next time you write it, please refrain from using hard-coded constants such as 4 where you should have sizeof(int32_t), 8 in some places where you should have sizeof(int64_t), 8 in other places where you should have a manifest constant called "BITS_PER_BYTE" (or BPB for brevity) etc. The reason we use manifest constants is not because their values might change, but because their names document our intentions and clarify the code.

Also, for fixed-length data types, it is a good idea to heed @Loki Astari's advice, and pay due attention to the endianness of your data: always use a specific known endianness for data that is meant to be shared between different machines, and what better choice is there than Network Byte Order.

Thank you for your comments. I am puzzled on your main point which is the use of the type. First you say that a type is not required, then you go on to say that the first byte can contain the data type enum. I will be using this protocol over a network tcp connection and frequently I will get half a message or 3 messages. So I don't see how I can do without a type. I don't want to be converting numbers to strings and vice versa. So I was a little confused. Are you suggesting that each 'message' sent should be a length and payload. and then within payload would be type followed by data? — arcomber, Nov 29 '11 at 16:33
@user619818 Exactly. (I am sorry if I was not clear enough in the text of the answer.) — Mike Nakis, Nov 29 '11 at 16:39

asked	4 years ago
viewed	2898 times
active	1 year ago

current community

your communities

more stack exchange communities

Protocol code to encode/decode various data types

1 Answer 1

Your Answer

Not the answer you're looking for? Browse other questions tagged c or ask your own question.

Hot Network Questions

current community

your communities

more stack exchange communities

Protocol code to encode/decode various data types

1 Answer 1

Your Answer

Sign up or log in

Post as a guest

Not the answer you're looking for? Browse other questions tagged c or ask your own question.

Related

Hot Network Questions