CC2541 SPI Problem (cannot achive a promissed speed for SPI)

I'm trying to use SPI connection of the CC2541 chip to communicate with a FPGA. For this purpose a UART1 and Alt-2 configuration was used (SLAVE mode). FPGA (MASTER) sends bytes to CC2541. After a packet was read the payload of the packet has to be send back.

My problems are following, by low speed transfer (1.7MHz) CC2541cannot receive data clean enought. By hight speed transfer (4MHz) needs a CC2541 circa 3.6 miliseconds (circa 133 processor cyclen) to react on receive/send interrupt. So i need a delay of 4.5 milices between a read operation driven by FPGA. (The delay is much grater then a send time!)

Can someone explain WHY i cannot send byte-to-byte without any delay?

The used code is:

typedef void (*SPI_CBF)(void);

struct SPISendStructur
{
   uint16 PacketLength;
   uint8 Packet[BUFFER_SIZE]; // das erste byte ist die Länge
   SPI_CBF CBF;
};

// Data structure...
typedef struct SPISendStructur SPISendStruct; 
typedef SPISendStruct *SPISendStructPtr;   

uint8 SPIInit(void) {

    // Configure USART1 for Alternative 2 => Port P1 (PERCFG.U1CFG = 1)
   

    PERCFG = (PERCFG & ~PERCFG_U1CFG) | PERCFG_U1CFG_ALT2; //(PERCFG & ~PERCFG_U0CFG) | PERCFG_U0CFG_ALT1;
    // Give priority to USART 1 over USART 0 then Timer 1 for port 0 pins
    P2DIR &= P2DIR_PRIP0_USART1;
    // Set pins 4, 4, 5 and 7 as peripheral I/O.
    P1SEL = BIT7 | BIT6 | BIT5 | BIT4;
     /***************************************************************************
     * Setup interrupt
     */

    // Clear CPU interrupt flag for USART1 RX (TCON.URX0IF)
    URX1IF = 0;

    // Set individual interrupt enable bit in the peripherals SFR
    // Not any.

    // Enable interrupt from USART1 RX by setting [IEN0.URX1IE=1]
    URX1IE = 1;
    // Enable global interrupts
    EA = 1;
    
    /***************************************************************************
     * Configure SPI
     */
    U1CSR = (U1CSR & ~U1CSR_MODE) | U1CSR_SLAVE;

    
    // Set Baud Rate Parameter:
    // - mantissa value
    // - exponent value

    U1BAUD  =  SPI_BAUD_M; // Set Baud rate Mantisse
    /*************************BAUD RATE FORMULA********************************/
    //
    // BaudRate = OscillatorFreq * ((256 + Baud_m) * 2^Baud_e) / 2^28
    //
    /**************************************************************************/
    // Set:
    // - clock phase to be centered on first edge of SCK period
    // - negative clock polarity (SCK low when idle)
    // - bit order for transfers to MSB first
    U1GCR = (U1GCR & ~(U1GCR_BAUD_E | U1GCR_CPOL | U1GCR_CPHA | U1GCR_ORDER)) | SPI_BAUD_E;
    U1GCR |= U1GCR_ORDER;

   // U1CSR |= RE;                 // Receiver Enable
    
    return 1;
}

#pragma inline                               // clock definition
void setHighSpeedClock() {
   SLEEPCMD |= OSC32K_CALDIS;
    // Change the system clock source to HS XOSC and set the clock speed to 32 MHz.
   CLKCONCMD = (CLKCONCMD & ~(CLKCON_CLKSPD | CLKCON_OSC)) | CLKCON_CLKSPD_32M;
    // Wait until system clock source has changed to HS XOSC (CLKCONSTA.OSC = 0).
   while(CLKCONSTA & CLKCON_OSC);
}

#pragma inline
void configureGPIOPin() {
    P1SEL &= ~BIT3;           // GPIO PIN FROM FPGA!
    P1DIR |= BIT3;            // Set as Output GPIO PIN FROM FPGA!
    P1_3  = 0;
 }

#pragma vector = URX1_VECTOR // Interrupt handler
__interrupt void uart1rx_isr(void) {

  if (PACKET == SEND_DATA) {
      U1DBUF;
      if (byteNumber == SPISndStrc.PacketLength) {
          P1_3 = 0;
          PACKET = IDLE;
      } else {
          U1DBUF         = SPISndStrc.Packet[byteNumber++];
      }
  }
  else {
     receivedByte = U1DBUF;
     dataReceived = TRUE;
  }
}

void PacketReceiveHandler() {
   if ((PACKET == IDLE) && (receivedByte == 0x55)) {   // First byte is received, if this byte is 0x55, start of the packet!
      PACKET = LOAD_LENGTH;                                   // Paket Receiving starts, load the number of bytes to receive
      byteNumber = 0;                                                     // Reset the number of received bytes
   } else if (PACKET == LOAD_LENGTH) {                   // Try to read the packet  // Number of bytes do not loaded jet                         
      SPISndStrc.PacketLength = receivedByte;            // second byte is received, it contains the number of bytes to be received
      PACKET = LOAD_DATA;                                    
   } else if (PACKET == LOAD_DATA) {                       // Load bytes to the buffer
      SPISndStrc.Packet[byteNumber++]  = receivedByte;
      if (byteNumber == SPISndStrc.PacketLength) {  // all bytes are in the buffer
         byteNumber  = 0;                                              // reset the number of bytes
         U1DBUF;                                                          // imidiate read to clear a receive buffer and interrupts
         U1DBUF      = SPISndStrc.Packet[byteNumber++]; // load data to the buffer
         P1_3        = 1;                                                 // set pin to inform the Fpga that chip is ready to send data
         PACKET      = SEND_DATA;                          // change system state to "SEND_DATA" state
      }
   }
}

int main(void) {
   setHighSpeedClock();
   configureGPIOPin();
   SPIInit();
   SPISndStrc.CBF = &SPI_SEND_SUCCES;
   PACKET = IDLE;
   while(1) {
      if (PACKET != SEND_DATA) {
         P1_3 = 0;
         if (dataReceived == TRUE)
         {
            PacketReceiveHandler();
            dataReceived = FALSE;
         }
      }
   }
}