Compiling errors in C while creating a dynamic array inside data structure

1. "why are these errors occurring"

   In your code, branch is an array of bTreeNode pointers, so you must tell compiler the length of branch. try this code:

   #define ORDER 10; // hard-coded
   struct bTreeNode {
       int count;
       int value[ORDER];
       struct bTreeNode *branch[ORDER - 1]; 
       int value[ORDER - 1];
   };
   

   And you can not use any functions in struct bTreeNode definition.

2. " How can I implement this idea properly"

   Flexible array is a good idea, try struct bTreeNode *branch[0], and allocate memory later. Sorry for my ugly english. ^_^

this declaration has a problem

struct bTreeNode *branch[order - 1] = malloc(sizeof(order));

the branch pointer is suppose to point at (order - 1) b-tree nodes, but you are allocating the memory size of order (which is probably an int, so it will return 4 bytes probably) you need to allocate the memory of btreeNode times the order (number of btree nodes * order). try this..

struct bTreeNode *branch = malloc(sizeof(struct bTreeNode)*order);

You are having a flexible array member, that is inside a struct an array without dimensions known at compile time. You can only have one of them, and it should be the last member of the struct:

struct bTreeNode {                      
  int count;                      
  int value[];     // flexible array member
};

^{flexible array members is an advanced feature of C99}

I'm guessing that you want your node to have arbitrary number of values and arbitrary number of sons. You need two flexible struct for that and you should use pointers.

// forward declarations
struct bTreeNode;
struct bTreeNumber;
struct bTreeSons;

struct bTreeNumber {
  unsigned valcount;
  int value[]; // actual dimension is valcount
};

struct bTreeSons {
   unsigned soncount;
   struct bTreeNode* ptrs[]; // actual dimension is soncount
};

struct bTreeNode {
  struct bTreeNumbers* numbers;
  struct bTreeSons* sons; 
};

Here is a function allocating an empty node with a given number of values and another number of sons

struct bTreeNode *make_node (unsigned nbval, unsigned nbsons)
{
   struct bTreeNumber* pnum = 
      malloc(sizeof(bTreeNumber)+nbval*sizeof(int));
   if (!pnum) { perror("malloc bTreeNumber"); exit(EXIT_FAILURE); };
   pnum->valcount = nbval;
   if (nbval>0) memset (pnum->value, 0, nbval*sizeof(int));
   struct bTreeSon* pson = 
      malloc(sizeof(bTreeSon)+nbsons*sizeof(struct bTreeNode*));
   if (!pson) { perror("malloc bTreeSon"); exit(EXIT_FAILURE); };
   pson->soncount = nbsons;
   for (unsigned ix=0; ix<nbsons; ix++) pson->ptrs[i] = NULL;
   struct bTreNode *pnode = malloc(sizeof(struct bTreeNode));
   if (!pnode) {perror("malloc bTreeNode"); exit(EXIT_FAILURE));
   pnode->numbers = pnum;
   pnode->sons = pson;
   return pnode;
 }

Alternatively you could decide that your nodes have a pointer to the numbers, and a flexible array member for sons (or vice versa)

 // alternatively
 struct bTreeNode {
   unsigned nbsons;
   struct bTreeNumber* numbers;
   struct bTreeNode* sons[]; // actual dimension is nbsons
 };

or even more "old-school" you could have the node know the number of sons and number of values, and keep pointers to (heap-allocated) arrays of them:

 // without flexible members
 struct bTreeNode {
    unsigned nbsons;
    unsigned nbvalues;
    int* pvalues; // point to an array of dimension nbvalues 
    struct bTreeNode** psons; // point to an array of dimension nbsons
 };