Strange behavior on SGMII of c6678 EVM

Seungsik,

I have discussed this with some of the experts and it appears that the 2.0.3.15 MCSDK didn't fix everything.  There is an internal team working on performance benchmarks and they have made some modifications to the code that produce better and stable results for higher traffic loads.  These fixes will be rolled into a subsequent release, but I will list them.  Sorry for the length, but hopefully this helps you...

Regards,

Travis

1.  The file pbm.c from NDK should increase the number of buffers in the NDK packet pool. This number is too small for line rate testing. We increased it by 6 times.
2.  Need to modify the number of Receive descriptors for the queuing. This can be done by changing the following define in resource_mgr.h, in the pdk platform directory:
Change #define NIMU_NUM_RX_DESC   18u to  #define NIMU_NUM_RX_DESC   128u or more.
3.    Change the pacing value for the accumulator from 3 to 4. For small packet sizes (64 and 128) going to 10 helps but it hurts larger packet sizes. The pacing value can be found in the nimu_internal.h header file:  #define     RX_INT_THRESHOLD            3u    ß-- Change that to 4u.
4.    For tcp change the default window size from 8192 to something higher. This can be done by changing the following in hpdspua.c or use the equivalent call in customer's own code:
    /* TCP Transmit buffer size */
    rc = 8192;
    CfgAddEntry( hCfg, CFGTAG_IP, CFGITEM_IP_SOCKTCPTXBUF,
                 CFG_ADDMODE_UNIQUE, sizeof(uint), (uint8_t *)&rc, 0 );

    /* TCP Receive buffer size (copy mode) */
    rc = 8192;
    CfgAddEntry( hCfg, CFGTAG_IP, CFGITEM_IP_SOCKTCPRXBUF,
                 CFG_ADDMODE_UNIQUE, sizeof(uint), (uint8_t *)&rc, 0 );

    /* TCP Receive limit (non-copy mode) */
    rc = 8192;
    CfgAddEntry( hCfg, CFGTAG_IP, CFGITEM_IP_SOCKTCPRXLIMIT,
                 CFG_ADDMODE_UNIQUE, sizeof(uint), (uint8_t *)&rc, 0 );

This is the NIMU Rx function we have been using for our performance benchmark work, it contains a fix for cleanup on buffer desriptors and a few other enhancements:

void
EmacRxPktISR
(
    NETIF_DEVICE*     ptr_net_device
)
{
    uint32_t            protocol, pktLen;
    uint8_t*            pBuffer;
    Cppi_HostDesc*      pHostDesc;
    PBM_Handle          hPkt;
    Cppi_Desc*          pCppiDesc;
    uint32_t            count, i;
    uint32_t*           ListAddress;
    EMAC_DATA*          ptr_pvt_data;
    PBM_Pkt*            rx_pbm_pkt;
    void*               key;
    void*               accum_list_ptr;


    /* Disable the interrupt */
    coreKey [CSL_chipReadReg (CSL_CHIP_DNUM)] =  Hwi_disableInterrupt(PLATFORM_ETH_INTERRUPT); //Hwi_disable();

    /* Begin Critical Section before accessing shared resources. */
    key = Osal_qmssCsEnter ();

   /* Get the pointer to the private data */
    ptr_pvt_data = (EMAC_DATA *)ptr_net_device->pvt_data;

    if (!gIsPingListUsed){
        accum_list_ptr = (void *)&gHiPriAccumList[0];
    }
    else {
        accum_list_ptr = (void *)&gHiPriAccumList[MAX_HI_PRI_ACCUM_LIST_SIZE];
    }


    /* Invalidate cache if needed --
     *   if accumulator is in DDR then INV L2.
     *   if accumulator is in shared RAM (MSMC) invalidate L1
     */
    if((uint32_t)(gHiPriAccumList) & EMAC_EXTMEM ){
        CACHE_invL2(accum_list_ptr, sizeof(gHiPriAccumList)/2, CACHE_WAIT);
    }

    if ((uint32_t)(gHiPriAccumList) & EMAC_MSMCSRAM ) {
        CACHE_invL1d(accum_list_ptr, sizeof(gHiPriAccumList)/2, CACHE_WAIT);
     }

    i           = MAX_HI_PRI_ACCUM_LIST_SIZE - 1 - (RX_INT_THRESHOLD);
    ListAddress = (uint32_t* )Convert_CoreLocal2GlobalAddr((uint32_t) &gHiPriAccumList[i]);

    /* Process ISR.
     *
     * Get the number of entries in accumulator list.
     * The hardware enqueues data alternatively to Ping/Pong buffer lists in
     * the accumulator. Hence, we need to track which list (Ping/Pong)
     * we serviced the last time and accordingly process the other one
     * this time around.
     */
     if (!gIsPingListUsed)
     {
        /* Serviced Pong list last time. So read off the Ping list now */
        count   =   ListAddress[0];
     }
     else
     {
        /* Serviced Pong list last time. So read off the Ping list now */
        count   =   ListAddress[RX_INT_THRESHOLD + 1];
     }

    /* Nothing to receive, so return... */
    if (count == 0) {
        /* End Critical Section */
        Osal_qmssCsExit (key);
        Hwi_restoreInterrupt(PLATFORM_ETH_INTERRUPT, coreKey [CSL_chipReadReg (CSL_CHIP_DNUM)]);
        /* Clear INTD */
        Qmss_ackInterrupt(PA_ACC_CHANNEL_NUM, 1);
        Qmss_setEoiVector(Qmss_IntdInterruptType_HIGH, PA_ACC_CHANNEL_NUM);
        return ; /* Not enough packets are received */
    }

    /* Process all the Results received
     *
     * Skip the first entry in the list that contains the
     * entry count and proceed processing results.
     */
    for (i = 1; i <= count; i ++)
    {

        gRxCounter ++;

        /* Get the result descriptor.
         *
         * The hardware enqueues data alternatively to Ping/Pong buffer lists in
         * the accumulator. Hence, we need to track which list (Ping/Pong)
         * we serviced the last time and accordingly process the other one
         * this time around.
         */
        if (!gIsPingListUsed)
        {
            /* Serviced Pong list last time. So read off the Ping list now */
            pCppiDesc   =   (Cppi_Desc *) ListAddress [i];
        }
        else
        {
            /* Serviced Ping list last time. So read off the Pong list now
             *
             * Skip over Ping list length to arrive at Pong list start.
             */
            pCppiDesc   =   (Cppi_Desc *) ListAddress [i + RX_INT_THRESHOLD + 1];
        }

        /* Descriptor size appended to the address in the last 4 bits.
         *
         * To get the true descriptor size, always mask off the last
         * 4 bits of the address.
         */
        pCppiDesc = (Ptr) ((uint32_t) pCppiDesc & 0xFFFFFFF0);
        pHostDesc = (Cppi_HostDesc *)pCppiDesc;

        /* Invalidate cache based on where the memory is */
        if((uint32_t)(pHostDesc) & EMAC_EXTMEM ){
            CACHE_invL2((void *) pHostDesc, sizeof(Cppi_HostDesc), CACHE_WAIT);
        }

        if ((uint32_t)(pHostDesc) & EMAC_MSMCSRAM ) {
            CACHE_invL1d((void *)pHostDesc, sizeof(Cppi_HostDesc), CACHE_WAIT);
         }

        if((uint32_t)(pHostDesc->buffPtr) & EMAC_EXTMEM ){
            CACHE_invL2((void *) pHostDesc->buffPtr, pHostDesc->buffLen, CACHE_WAIT);
        }

        if ((uint32_t)(pHostDesc->buffPtr) & EMAC_MSMCSRAM ) {
            CACHE_invL1d((void *)pHostDesc->buffPtr, pHostDesc->buffLen, CACHE_WAIT);
         }

        /*
         * We should not see packets too large but check anyways ...
         * Note that we are subtracting off the FCS the switch added to the frame.
         * If its too large then return it to the free queue.
         */
        if ((pHostDesc->buffLen-4) > (ptr_net_device->mtu + ETHHDR_SIZE)) {
            /* lets try the next one... we should record this as a too large.... */
            gRxDropCounter++;
            pHostDesc->buffLen = pHostDesc->origBufferLen;
            QMSS_QPUSH (gRxFreeQHnd, (Ptr)pHostDesc, pHostDesc->buffLen, SIZE_HOST_DESC, Qmss_Location_TAIL);
            continue;
        }

        /* Allocate the PBM packet for the Max MTU size*/
        if (NULL == (hPkt = PBM_alloc(1514))) {
            /* could not get a free NDK packet, maybe the next time around we can... */
            gRxDropCounter++;
            pHostDesc->buffLen = pHostDesc->origBufferLen;
            QMSS_QPUSH (gRxFreeQHnd, (Ptr)pHostDesc, pHostDesc->buffLen, SIZE_HOST_DESC, Qmss_Location_TAIL);
            continue;
        }

        rx_pbm_pkt = (PBM_Pkt *) hPkt;

        PBM_setDataOffset((PBM_Handle) hPkt, 0);

        /* removing the FCS length the EMAC switch adds  here */
        pktLen    = (pHostDesc->buffLen-4);

        /* Set to minimum packet size */
        if (pktLen < 60) {
            pktLen = 64;
        }

        PBM_setValidLen((PBM_Handle) hPkt, pktLen);

        /* Handle raw frames separately, i.e. check the
        * Ethernet Protocol type in this packet and see
        * if its a well known ether type. If so, this is normal
        * IP stream, enqueue this is in usual Rx queue and let the
        * stack know that a packet has arrived for it. However, if
        * the Ethernet type in the packet is not a well known one,
        * this could be a custom raw Ethernet packet, enqueue it
        * separately in the Raw Rx queue and notify stack. The Raw
        * Ethernet packets when being handed over are given
        * preferential treatment and are serviced before the normal
        * IP traffic. Hence the 2 queues.
        */
        pBuffer =  (uint8_t* )Convert_CoreLocal2GlobalAddr((uint32_t) pHostDesc->buffPtr);

        /* Extract the Ethernet type from the packet. */
        protocol = ( pBuffer[12] << 8) | pBuffer[13] ;
        protocol = (protocol & 0xFFFFu);

        PBM_setIFRx((PBM_Handle) hPkt, (HANDLE) protocol );

        /* Copy the data buffer received to the allocated PBM packet */
        mmCopy((uint8_t* )rx_pbm_pkt->pDataBuffer, (uint8_t* )pBuffer, pktLen) ;

        /* Is it a standard ethernet type? */
        if (protocol != ETHERTYPE_IP && protocol != ETHERTYPE_IPv6 && protocol != ETHERTYPE_VLAN
            && protocol != ETHERTYPE_PPPOECTL && protocol != ETHERTYPE_PPPOEDATA )
        {
            /* This is a raw packet, enqueue in Raw Rx Queue */
            PBMQ_enq( &ptr_pvt_data->pdi.PBMQ_rawrx, (PBM_Handle) hPkt);
        }
        else
        {   /* This is a normal IP packet. Enqueue in Rx Queue */
            PBMQ_enq( &ptr_pvt_data->pdi.PBMQ_rx, (PBM_Handle) hPkt );
        }

       /* Free the packet back to the Rx FDQ */
        pHostDesc->buffLen = pHostDesc->origBufferLen;
        QMSS_QPUSH (gRxFreeQHnd, (Ptr)pHostDesc, pHostDesc->buffLen, SIZE_HOST_DESC, Qmss_Location_TAIL);
    }

    ListAddress = (uint32_t *) Convert_CoreLocal2GlobalAddr((uint32_t) &gHiPriAccumList[0]);

    /* Clear the accumulator list and save whether we used Ping/Pong
     * list information for next time around.
     */
     if (!gIsPingListUsed)
    {
        /* Just processed Ping list */
        gIsPingListUsed  =   1;

        i = sizeof (gHiPriAccumList)/2;

        /* Clear the accumulator list after processing */
        memset ((Void *) &gHiPriAccumList[0], 0, i);

        if(
            ((uint32_t)(&gHiPriAccumList[0]) & EMAC_EXTMEM ) ||
            ((uint32_t)(&gHiPriAccumList[0]) & EMAC_MSMCSRAM )
          )
        {
            /* This needs to be enabled if gHiPriAccumList is in External Memory or MSMCMEM */
            CACHE_wbL2 ((void *)&gHiPriAccumList[0], i, CACHE_WAIT);
        }
    }
    else
    {
        /* Just processed Pong list */
        gIsPingListUsed  =   0;
        i = sizeof (gHiPriAccumList)/2;

        /* Clear the accumulator list after processing */
        memset ((Void *) &gHiPriAccumList[MAX_HI_PRI_ACCUM_LIST_SIZE], 0, i);

        if(
            ((uint32_t)(&gHiPriAccumList[MAX_HI_PRI_ACCUM_LIST_SIZE]) & EMAC_EXTMEM ) ||
            ((uint32_t)(&gHiPriAccumList[MAX_HI_PRI_ACCUM_LIST_SIZE]) & EMAC_MSMCSRAM )
          )
        {
            /* This needs to be enabled if gHiPriAccumList is in External Memory or MSMCMEM */
            CACHE_wbL2((void *) &gHiPriAccumList[MAX_HI_PRI_ACCUM_LIST_SIZE], i, CACHE_WAIT);
        }
    }

    /* Notify NDK stack of pending Rx Ethernet packet */
    STKEVENT_signal( ptr_pvt_data->pdi.hEvent, STKEVENT_ETHERNET, 1 );

    /* End Critical Section */
    Osal_qmssCsExit (key);
    Hwi_restoreInterrupt(PLATFORM_ETH_INTERRUPT, coreKey [CSL_chipReadReg (CSL_CHIP_DNUM)]);

    /* Clear INTD */
    Qmss_ackInterrupt(PA_ACC_CHANNEL_NUM, 1);
    Qmss_setEoiVector(Qmss_IntdInterruptType_HIGH, PA_ACC_CHANNEL_NUM);



    return;
}