TMS320C6655: FPGA push data to C6655 Local L2 SRAM via PCIE error.

Denny%20Yang99373

Part Number: TMS320C6655

Hi all,

Customer use FPGA to push data to local L2 SRAM address 0x800000~0x8FFFFF.

When run about half of a hour, there maybe some error happened.

FPGA send data every 50us, and dsp check the data every time. Some times DSP will find few of the data haven't updated. Most of the data have been updated by FPGA.

But if FPGA push the data to msmc memory, 0xC000000~0xC1FFFFF. it seems everything right.

Attached is the PCIE init code. Please help to this.

thanks!

BR,
Denny

1185.initdsp.c

//--------------------------------  Include -----------------------------------
#include "uapcslave.h"
#include "c66x_common.h"
#include "fpga_cfg.h"
#include "fram.h"
#include "board.h"
#include "ExceptionLog.h"
#include "localdbase.h"

//--------------------------------  Define  -----------------------------------
#define PCIE_PREFETCH_BASE_ADDRESS       0x88000000
#define PCIE_NONFETCH_BASE_ADDRESS       0x60000000
#define PCIE_RC_BAR0_ADDRESS             0x01000000
#define PCIE_HTM0_WR_START              (0x60100000 + (0x0<<2))
#define PCIE_HTM1_WR_START              (0x60100000 + (0x1000<<2))

//----------------------------  Local Variable  -------------------------------

//PCIE
PCIE_Config                  PCIE_cfg;
PCIE_RC_Config               rc_cfg;
PCIE_Remote_CFG_SETUP        remote_cfg_setup;
PCIE_Interrupt_Config        PCIE_int_cfg;

PCIE_Memory_Regions          prefetch_regions; 
PCIE_Memory_Regions          nonfetch_regions;
PCIE_Inbound_Memory_Regions  inbound_memory_regions;
PCIE_Outbound_Memory_Regions outbound_memory_regions;

Uint32 pcie_reg[32];
int pcie_init_fail_flag = 0;


//----------------------------------  Code  -----------------------------------

/*-----------------------------------------------------------------------------
    Function    : void initCache(void)
    Description : ��ʼ��cache
    Input       : ��
    Output      : ��
    Return      : ��
    Others      : 
-----------------------------------------------------------------------------*/
#pragma CODE_SECTION(initCache, ".rtm_code")
void initCache(void)
{
    Uint32 first_MAR, last_MAR, MAR_num;
    
//    CACHE_setL1PSize(CACHE_L1_32KCACHE);
//    CACHE_setL1DSize(CACHE_L1_32KCACHE);
//    CACHE_setL2Size(CACHE_128KCACHE);
    CACHE_setL1PSize(CACHE_L1_0KCACHE);
    CACHE_setL1DSize(CACHE_L1_0KCACHE);
    CACHE_setL2Size(CACHE_0KCACHE);
    
//    first_MAR = 0x80000000 >> 24;
//    last_MAR  = 0x87FFFFFF >> 24;
//    
//    for(MAR_num = first_MAR; MAR_num <= last_MAR; MAR_num++)
//    {
//        CACHE_enableCaching(MAR_num);
//        asm(" NOP");
//    }
//    
//    first_MAR = 0x89000000 >> 24;
//    last_MAR  = 0x9FFFFFFF >> 24;
//    
//    for(MAR_num = first_MAR; MAR_num <= last_MAR; MAR_num++)
//    {
//        CACHE_enableCaching(MAR_num);
////        CSL_FINS (hCache->MAR[MAR_num], CGEM_MAR0_PFX, 1);
//        asm(" NOP");
//    }
    
    CACHE_invAllL1p(CACHE_WAIT);
    CACHE_wbInvAllL1d(CACHE_WAIT);
    CSL_XMC_invalidatePrefetchBuffer();
    /* protect L1 as cache */
    configL1CacheProtection();
    _mfence();
    _mfence();
}

/*-----------------------------------------------------------------------------
    Function    : void initPCIE(void)
    Description : ��ʼ��PCIe
    Input       : ��
    Output      : ��
    Return      : ��
    Others      : 
-----------------------------------------------------------------------------*/
#pragma CODE_SECTION(initPCIE, ".init_code")
void initPCIE(void)
{
    Uint8 pcie_init_fail_cnt=0;
    memset(&prefetch_regions       , 0, sizeof(prefetch_regions       )); 
    memset(&nonfetch_regions       , 0, sizeof(nonfetch_regions       ));
    memset(&inbound_memory_regions , 0, sizeof(inbound_memory_regions ));
    memset(&outbound_memory_regions, 0, sizeof(outbound_memory_regions));
    memset(&PCIE_int_cfg           , 0, sizeof(PCIE_int_cfg           ));
    memset(&rc_cfg                 , 0, sizeof(rc_cfg                 ));
    memset(&remote_cfg_setup       , 0, sizeof(remote_cfg_setup       ));
    memset(&PCIE_cfg               , 0, sizeof(PCIE_cfg               ));
    
    PCIE_cfg.PcieMode= PCIE_RC_MODE;
    PCIE_cfg.loop_mode = PCIE_LOOPBACK_DISABLE;
    PCIE_cfg.address_width = PCIE_ADDRESS_32_BITS;
    PCIE_cfg.serdes_cfg.inputRefClock_MHz = 100;
    PCIE_cfg.serdes_cfg.linkSpeed_GHz = 2.5;
    PCIE_cfg.serdes_cfg.numLanes= 1;
    PCIE_cfg.serdes_cfg.loopBandwidth    = SERDES_PLL_LOOP_BAND_MID;
    PCIE_cfg.serdes_cfg.txInvertPolarity = SERDES_TX_NORMAL_POLARITY; 
    PCIE_cfg.serdes_cfg.rxInvertPolarity = SERDES_RX_NORMAL_POLARITY;
    PCIE_cfg.serdes_cfg.rxEqualizerConfig= SERDES_RX_EQ_ADAPTIVE;
    PCIE_cfg.serdes_cfg.rxCDR            = 1;
    PCIE_cfg.serdes_cfg.rxLos            = SERDES_RX_LOS_DISABLE;
    PCIE_cfg.serdes_cfg.rxAlign          = SERDES_RX_ALIGNMENT_DISABLE;
    
    PCIE_int_cfg.MSI_rx_enable_mask= 0;
    PCIE_int_cfg.Err_rx_enable = FALSE;
    PCIE_int_cfg.PMRST_rx_enable = FALSE;
    PCIE_int_cfg.number_tx_MSI = PCIE_NO_TX_MSI;
    PCIE_cfg.interrupt_cfg= &PCIE_int_cfg;
    
    PCIE_cfg.outbound_memory_regions= &outbound_memory_regions;
    PCIE_cfg.inbound_memory_regions= &inbound_memory_regions;
    outbound_memory_regions.OB_size= PCIE_OB_SIZE_8MB;
    
    rc_cfg.memory_base= PCIE_NONFETCH_BASE_ADDRESS;
    rc_cfg.memory_limit= PCIE_NONFETCH_BASE_ADDRESS+256*1024*1024-1;
    rc_cfg.prefetch_memory_base= PCIE_PREFETCH_BASE_ADDRESS;
    rc_cfg.prefetch_memory_limit= PCIE_PREFETCH_BASE_ADDRESS+256*1024*1024-1;
    rc_cfg.BAR0_address= PCIE_RC_BAR0_ADDRESS;
    
    PCIE_cfg.rc_cfg= &rc_cfg;
    
    remote_cfg_setup.config_type= 0;
    remote_cfg_setup.config_bus= 0;
    remote_cfg_setup.config_device= 0;
    remote_cfg_setup.config_function= 0;
    
    while(pcie_init_fail_cnt<3)
    {
    	if(0==initPCIESS(&PCIE_cfg))
    	{
    	    pcie_init_fail_cnt++;
    	}
    	else
    	{
    	    break;
    	}
    }
    if(pcie_init_fail_cnt>=3)
    {
        pcie_init_fail_flag = 1;
        new_err_code |= BOARD_HD_INIT_ERR;
        return;
    }

    setupPCIERemoteCFG(&remote_cfg_setup);

    setupPCIEAddress(&PCIE_cfg);
    
    delayUs(10);
    
    pcie_reg[0]  = gpPCIE_remote_EP_regs->BAR[0];           //�ض���PCIE�豸�Ĵ�������
    pcie_reg[1]  = gpPCIE_remote_EP_regs->VENDOR_DEVICE_ID;
    pcie_reg[2]  = gpPCIE_remote_EP_regs->STATUS_COMMAND;
    pcie_reg[3]  = gpPCIE_remote_EP_regs->CLASSCODE_REVID;
    pcie_reg[4]  = gpPCIE_remote_EP_regs->BIST_HEADER;
    pcie_reg[5]  = gpPCIE_remote_EP_regs->PCIES_CAP;
    pcie_reg[6]  = gpPCIE_remote_EP_regs->DEVICE_CAP;
    pcie_reg[7]  = gpPCIE_remote_EP_regs->LINK_CAP;
    pcie_reg[8]  = gpPCIE_remote_EP_regs->LINK_STAT_CTRL;
    pcie_reg[9]  = gpPCIE_remote_EP_regs->DEV_CAP2;
    pcie_reg[10] = gpPCIE_remote_EP_regs->DEV_STAT_CTRL2;
    pcie_reg[11] = gpPCIE_app_regs->CMD_STATUS;
    
    //
    *TXS_PCIE_DDR1_L = LL2_PCIE_ADDR_OFFSET;        // 0x10000000
    *TXS_PCIE_DDR1_H = 0x00000000;
    *TXS_PCIE_DDR2_L = LL2_PCIE_ADDR_OFFSET;
    *TXS_PCIE_DDR2_H = 0x00000000;
    *TXS_PCIE_DDR3_L = 0x00000000;
    *TXS_PCIE_DDR3_H = 0x00000000;
    *TXS_PCIE_DDR4_L = 0x00000000;
    *TXS_PCIE_DDR4_H = 0x00000000;
}

//End of File

over 6 years ago

0 Yordan Kovachev over 6 years ago

TI__Guru**** 161600 points

Hi,

I've notified the RTOS team. Their feedback will be posted directly here.

Best Regards,
Yordan

0 lding over 6 years ago in reply to Yordan Kovachev

TI__Guru* 95265 points

Denny,

The code is not TI Processor SDK RTOS PCIE example style and many function implementations are missing. Overall it looks correct, and the issue is L2 vs MSMC, it is not the way how you setup the PCIE.

Some questions:
1) the FPGA push data to DSP. This is FPGA writes? Is any EDMA involved or just like CPU write?
2) How the DSP know the data arrived? Is there any interrupt mechanism? Like FPGA sent a legacy or MSI interrupt to DSP after push?
3) You just saw the data incomplete but not corrupted, correct?
4) How big is the data? If change the FPGA push interval longer or make the data pushed shorter, did it help?
5) Was the PCIE relaxed order enabled or disabled in the setup? "In the RC mode, the user could disable the relaxed ordering feature by clearing the RELAXED field to 0 in DEV_STAT_CTRL register. The RC could also clear
RELAXED field to 0 in TLPCFG register to disable the reordering request for all outgoing TLP"

Regards, Eric

0 Denny%20Yang99373 over 6 years ago in reply to lding

TI__Expert 3805 points

Lding,

1, FPGA send data to C6655 LL2 memory via pcie bus, C6655 only config PCIE module, no additional EDMA.
2,FPGA will send a GPIO interrupt to c6655 after send data to LL2 via pcie. C6655 will check the data in interrupt function. This can keep the order right.
3, The error data keep the old data. Maybe 128 bytes maybe little.
4, FPGA send the data every 50us(640*4 bytes), the send LL2 address and length are 128 byte aligned.
5, No DEV_STAT_CTRL operation.
thanks!

BR,
Denny

0 lding over 6 years ago in reply to Denny%20Yang99373

TI__Guru* 95265 points

Hi,

I originally thought customer used PCIE interrupt to signal the data is arrived. It is possible that PCIE interrupt came before the PCIE write, but as they used GPIO interrupt, this is not the case.

The issue still looks like PCIE write landed slower in L2 than in MSMC. On the DSP side, Is there any mechanism detecting only partial data was written so to halt the FPGA writes again? I want to understand if you check this L2 region later, will you see the correct data (just late arrival before GPIO interrupt) or the data never come in?

Regards, Eric

0 Denny%20Yang99373 over 6 years ago in reply to lding

TI__Expert 3805 points

Hi Eric,

FPGA send data to L2 sequence was keep by GPIO interrupt. We don't know if there "any mechanism detecting only partial data was written so to halt the FPGA writes again" . Maybe PCIE IP can do this?

For your questions. The wrong data happened, the correct one will never come in.

I have let customer to test:

1, The code run on MSMC, L2 just used for PCIE receive. This issue still happened in a hour.

2, They change the data send period from 50us to 200us, this issue still exist. The issue happened after 6 hours testing. It seems it can reduce the issue happen possibility.

BR,
denny

0 lding over 6 years ago in reply to Denny%20Yang99373

TI__Guru* 95265 points

Hi,

The data never came in is puzzling. From the code L1D, L1P, L2 cache are all disabled, it is not cache issue. FPGA writes data to DSP, the inbound translation of DSP determines where the data landed. Between each data block sent from FPGA, did the DSP reconfigure the PCIE inbound translation at run-time? And if the buffer at the DSP is fixed?

In PCIE User Guide, 2.16.2.2 PCI Express Baseline Error Handling and 2.16.2.3 PCI Express Advanced Error Reporting, are they able to enable those errors and to check if any error reported?

Regards, Eric

0 Denny%20Yang99373 over 6 years ago in reply to lding

TI__Expert 3805 points

Hi Eric,

Between each data block sent from FPGA, did the DSP reconfigure the PCIE inbound translation at run-time?

And if the buffer at the DSP is fixed?

Yes

In PCIE User Guide, 2.16.2.2 PCI Express Baseline Error Handling and 2.16.2.3 PCI Express Advanced Error Reporting, are they able to enable those errors and to check if any error reported?

Could you please provide more detailed information? or which register is useful for this issue debug? I can capture it. thanks!

BR,
Denny

Processors

Processors forum

TMS320C6655: FPGA push data to C6655 Local L2 SRAM via PCIE error.