TMS320C6678: PCIe PHY loopback mode

Hi Viney,

I've forwarded this to the PCI experts. Their feedback should be posted here.

BR
Tsvetolin Shulev

Hi,

The Processor SDK RTOS PCIE driver for C66x doesn't support the PCIE PHY loopback, I thought you modified it for this purpose. After PCIE link is up but before data writing test, did you see those error bit set? To make the data test pass, you need do some change of address translation (0x9000_0000 and 0x7000_0000 thing), did you do that?

Regards, Eric

Hi Eric,

You are right, I modified the existing project to support PHY loopback mode.

I noticed that those error bits are being set even before the data is being sent. The LTSM value is 11 as expected and I see "Link is up" message on the console.

I made PCIE_OB_LO_ADDR_RC and PCIE_IB_LO_ADDR_RC the same to get this to work. The data I send is correctly echoed back into the appropriate buffer, but I don't know why those error bits are being set.

Thanks,

Viney

Hi,

Can you upload the modified code for loopback (I guess it just inside pcie_sample.c) and tell me which MCSDK or Processor SDK RTOS release it is, then I can try?

Regards, Eric

Hi Eric,

I'm using Processor SDK RTOS version 3.02.00.05.

Here's the code inside pcie_sample.c :

void main (void)
{
#ifdef _TMS320C6X
TSCL = 1;
#endif
#if !defined(DEVICE_K2K) && !defined(DEVICE_K2H) && !defined(DEVICE_K2E) && !defined(DEVICE_K2L) && \
!defined(SOC_K2K) && !defined(SOC_K2H) && !defined(SOC_K2L) && !defined(SOC_K2E) && !defined(SOC_K2G) && \
!defined(SOC_AM572x) && !defined(SOC_AM571x)
uint16_t lock=0;
#endif
pcieRet_e retVal;
pcieIbTransCfg_t ibCfg;
pcieBarCfg_t barCfg;
Pcie_Handle handle = NULL;
void *pcieBase;
dstBuf_t *pciedstBufBase;
uint32_t i;
#ifdef PCIE_REV1_HW
SemaphoreP_Handle sem = NULL;
#endif

/* Get remote buffer out of cache */
cache_writeback ((void *)&dstBuf, sizeof(dstBuf));
/* Unlock kicker once, and don't relock, because its not multicore safe */
#if !defined(SOC_AM572x) && !defined(SOC_AM571x)
CSL_BootCfgUnlockKicker();
#endif

#ifdef EDMA
EDMA3_DRV_Handle hEdma = NULL;
hEdma = edmaInit(hEdma);
if (hEdma==NULL) PCIE_logPrintf("ERROR: EDMA handle not initialized!\n");
#endif

#ifndef IO_CONSOLE
Console_printf ("IO_CONSOLE not defined. Most output will go to UART\n");
#endif

//PCIE_logPrintf ("**********************************************\n");
//PCIE_logPrintf ("* PCIe Test Start *\n");

printf ("**********************************************\n");
printf ("* PCIe Test Start *\n");

if(PcieModeGbl == pcie_RC_MODE)
//PCIE_logPrintf ("* RC mode *\n");
printf ("* RC mode *\n");
else
PCIE_logPrintf ("* EP mode *\n");

//PCIE_logPrintf ("**********************************************\n\n");
printf ("**********************************************\n\n");

//PCIE_logPrintf ("Version #: 0x%08x; string %s\n\n", (unsigned)Pcie_getVersion(), Pcie_getVersionStr());
printf("Version #: 0x%08x; string %s\n\n", (unsigned)Pcie_getVersion(), Pcie_getVersionStr());

/* Pass device config to LLD */
if ((retVal = Pcie_init (&pcieInitCfg)) != pcie_RET_OK)
{
//PCIE_logPrintf ("LLD device configuration failed\n");
printf ("LLD device configuration failed\n");
exit(1);
}

/* Initialize application buffers */
pcieInitAppBuf();

/* Power up PCIe Module */
if ((retVal = pciePowerCfg()) != pcie_RET_OK) {
//PCIE_logPrintf ("PCIe Power Up failed (%d)\n", (int)retVal);
printf ("PCIe Power Up failed (%d)\n", (int)retVal);
exit(1);
}

//PCIE_logPrintf ("PCIe Power Up.\n");
printf ("PCIe Power Up.\n");

if ((retVal = Pcie_open(0, &handle)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Open failed (%d)\n", (int)retVal);
printf ("Open failed (%d)\n", (int)retVal);
exit(1);
}

/* Configure SERDES*/
if ((retVal = pcieSerdesCfg()) != pcie_RET_OK) {
//PCIE_logPrintf ("PCIe Serdes config failed (%d)\n", (int)retVal);
printf ("PCIe Serdes config failed (%d)\n", (int)retVal);
exit(1);
}

/* Set the PCIe mode*/
if ((retVal = Pcie_setInterfaceMode(handle, PcieModeGbl)) != pcie_RET_OK) {
//PCIE_logPrintf ("Set PCIe Mode failed (%d)\n", (int)retVal);
printf ("Set PCIe Mode failed (%d)\n", (int)retVal);
exit(1);
}

/* Wait until the PCIe SERDES PLL locks */
#if !defined(DEVICE_K2K) && !defined(DEVICE_K2H) && !defined(DEVICE_K2E) && !defined(DEVICE_K2L) && \
!defined(SOC_K2K) && !defined(SOC_K2H) && !defined(SOC_K2L) && !defined(SOC_K2E) && !defined(SOC_K2G) && \
!defined(SOC_AM572x) && !defined(SOC_AM571x)
while (!lock)
{
CSL_BootCfgGetPCIEPLLLock(&lock);
}
#endif /* !DEVICE_K2K && !DEVICE_K2H && !DEVICE_K2E && !DEVICE_K2L && !SOC_K2G && !SOC_AM572x && !SOC_AM571x */


//PCIE_logPrintf ("PLL configured.\n");
printf ("PLL configured.\n");

if(PcieModeGbl == pcie_RC_MODE)
{
/* Configure application registers for Root Complex*/
if ((retVal = pcieCfgRC(handle)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Failed to configure PCIe in RC mode (%d)\n", (int)retVal);
printf ("Failed to configure PCIe in RC mode (%d)\n", (int)retVal);
exit(1);
}

/* Configure Address Translation */

barCfg.location = pcie_LOCATION_LOCAL;
barCfg.mode = pcie_RC_MODE;
barCfg.base = PCIE_IB_LO_ADDR_RC;
barCfg.prefetch = pcie_BAR_NON_PREF;
barCfg.type = pcie_BAR_TYPE32;
barCfg.memSpace = pcie_BAR_MEM_MEM;
barCfg.idx = PCIE_BAR_IDX_RC;

if ((retVal = Pcie_cfgBar(handle, &barCfg)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Failed to configure BAR (%d)\n", (int)retVal);
printf ("Failed to configure BAR (%d)\n", (int)retVal);
exit(1);
}


ibCfg.ibBar = PCIE_BAR_IDX_RC; /* Match BAR that was configured above*/
ibCfg.ibStartAddrLo = PCIE_IB_LO_ADDR_RC;
ibCfg.ibStartAddrHi = PCIE_IB_HI_ADDR_RC;
// ibCfg.ibOffsetAddr = (uint32_t)pcieConvert_CoreLocal2GlobalAddr ((uint32_t)dstBuf.buf);
ibCfg.ibOffsetAddr = (uint32_t)(dstBuf.buf);
ibCfg.region = PCIE_IB_REGION_RC;

if ((retVal = pcieIbTransCfg(handle, &ibCfg)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Failed to configure Inbound Translation (%d)\n", (int)retVal);
printf ("Failed to configure Inbound Translation (%d)\n", (int)retVal);
exit(1);
}
else
{
//PCIE_logPrintf ("Successfully configured Inbound Translation!\n");
printf ("Successfully configured Inbound Translation!\n");
}

//Configure OB_OFFSET_INDEXn register - n refers to the OB translation region
if ((retVal = pcieObTransCfg (handle, PCIE_OB_LO_ADDR_RC, PCIE_OB_HI_ADDR_RC, PCIE_OB_REGION_RC)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Failed to configure Outbound Address Translation (%d)\n", (int)retVal);
printf ("Failed to configure Outbound Address Translation (%d)\n", (int)retVal);
exit(1);
}
else
{
//PCIE_logPrintf ("Successfully configured Outbound Translation!\n");
printf ("Successfully configured Outbound Translation!\n");
}
}


else
{
/* Configure application registers for End Point*/
if ((retVal = pcieCfgEP(handle)) != pcie_RET_OK)
{
//PCIE_logPrintf ("Failed to configure PCIe in EP mode (%d)\n", (int)retVal);
printf ("Failed to configure PCIe in EP mode (%d)\n", (int)retVal);
exit(1);
}

/* Configure Address Translation */

barCfg.location = pcie_LOCATION_LOCAL;
barCfg.mode = pcie_EP_MODE;
barCfg.base = PCIE_IB_LO_ADDR_EP;
barCfg.prefetch = pcie_BAR_NON_PREF;
barCfg.type = pcie_BAR_TYPE32;
barCfg.memSpace = pcie_BAR_MEM_MEM;
barCfg.idx = PCIE_BAR_IDX_EP;

if ((retVal = Pcie_cfgBar(handle, &barCfg)) != pcie_RET_OK)
{
printf ("Failed to configure BAR!\n");
exit(1);
}

ibCfg.ibBar = PCIE_BAR_IDX_EP; /* Match BAR that was configured above*/
ibCfg.ibStartAddrLo = PCIE_IB_LO_ADDR_EP;
ibCfg.ibStartAddrHi = PCIE_IB_HI_ADDR_EP;
ibCfg.ibOffsetAddr = (uint32_t)pcieConvert_CoreLocal2GlobalAddr ((uint32_t)dstBuf.buf); //This is the destination address (DDR/L2/MSMC) for the outbound read
ibCfg.region = PCIE_IB_REGION_EP;

if ((retVal = pcieIbTransCfg(handle, &ibCfg)) != pcie_RET_OK)
{
printf ("Failed to configure Inbound Translation (%d)!\n", (int)retVal);
exit(1);
}
else
{
printf ("Successfully configured Inbound Translation!\n");
}

if ((retVal = pcieObTransCfg (handle, PCIE_OB_LO_ADDR_EP, PCIE_OB_HI_ADDR_EP, PCIE_OB_REGION_EP)) != pcie_RET_OK)
{
printf ("Failed to configure Outbound Address Translation(%d)!\n", (int)retVal);
exit(1);
}
else
{
printf ("Successfully configured Outbound Translation!\n");
}
}

printf ("Starting link training...\n");


//VK 5/15/17
/*
For PHY loopback mode, configure the Serdes config registers to set transmit and receive paths to be in loopback mode.
Enable TX loopback and RX loopback in SerDes Configuration Lane 0
Register (SERDES_CFG0[TX_LOOPBACK]=0x2 and
SERDES_CFG0[RX_LOOPBACK]=0x3).
– Disable loss of signal detection in SerDes Configuration Lane 0 Register
(SERDES_CFG0[RX_LOS]=0).
*/

if ((retVal = enableTxAndRxLoopback(handle)) != pcie_RET_OK)
{
printf ("Failed to enable Tx and Rx loopback! (%d)\n", (int)retVal);
exit(1);
}


/*Enable link training*/
if ((retVal = pcieLtssmCtrl(handle, TRUE)) != pcie_RET_OK)
{
printf ("Failed to Enable Link Training! (%d)\n", (int)retVal);
exit(1);
}

//VK 5/15/17
/* For PHY loopback mode:
we need to skip
Detect state in LTSSM and force link to begin with POLL_ACTIVE state.
– Set 0x2 (POLL_ACTIVE state) in LNK_STATE field of Port Force Link
Register (PL_FORCE_LINK[LINK_STATE]=0x2).
– Set 0x1 in FORCE_LINK field of PL_FORCE_LINK register to force the link
to the state specified by LINK_STATE field
(PL_FORCE_LINK[FORCE_LINK]=0x1).
*/


if ((retVal = skipDetectStateInLtssm(handle)) != pcie_RET_OK)
{
printf ("Failed to skip detect state in LTSSM! (%d)\n", (int)retVal);
exit(1);
}



/* Wait for link to be up */
pcieWaitLinkUp(handle); //Step 9 in section 2.11.1.1

printf ("Link is up.\n");
if ((retVal = pcieCheckLinkParams(handle)) != pcie_RET_OK)
{
printf ("Link width/speed verification FAILed: %d\n", retVal);
/* This exit() can be removed if this example is being used as
* template with non TI card that supports slower or narrower connections
*/
exit(1);
}

if ((retVal = Pcie_getMemSpaceRange (handle, &pcieBase, NULL)) != pcie_RET_OK) {
printf ("getMemSpaceRange failed (%d)\n", (int)retVal);
exit(1);
}

//VK 5/18/17 - read DEV_STAT_CTRL_REG
read_devStatCtrlReg(handle);



#ifdef PCIE_REV1_HW
/* Adjust PCIE base to point at remote target buffer */
pcieBase = (char *)pcieBase +
PCIE_WINDOW_MEM_BASE + /* data area doesn't start at low address */
(((uint32_t)&dstBuf) & 0xfff); /* dstBuf needs to be 4K aligned in addr tran */
#endif
pciedstBufBase = (dstBuf_t *)pcieBase;

if(PcieModeGbl == pcie_RC_MODE)
{
#ifdef PCIE_REV1_HW
sem = PlatformSetupMSIAndINTX (handle);
#endif
/**********************************************************************/
/* Push a single message to the EP then verify that it is echoed back */
/**********************************************************************/

/* Write from RC to EP
*/

for (i=0; i<PCIE_BUFSIZE_APP; i++)
{
pciedstBufBase->buf[i] = srcBuf[i];
}


/*
for (i=0; i<PCIE_BUFSIZE_APP; i++)
{
*((volatile uint32_t *)pcieBase + i) = srcBuf[i];
}
*((volatile uint32_t *)pcieBase + PCIE_BUFSIZE_APP) = PCIE_EXAMPLE_BUF_FULL;
*/

/* Mark that the buffer is full, so EP can process it */
pciedstBufBase->buf[PCIE_BUFSIZE_APP] = PCIE_EXAMPLE_BUF_FULL;

/* Note on cache coherence: Write back is not necessary because pcieBase is in
peripheral address space instead of physical memory*/

/* Data sent to EP.
RC waits for the loopback to be completed and
receive data back from EP */


do
{
unsigned int key;
key = _disable_interrupts();
CSL_XMC_invalidatePrefetchBuffer();
CACHE_invL1d ((void *)dstBuf.buf, PCIE_EXAMPLE_DSTBUF_BYTES, CACHE_FENCE_WAIT);
CACHE_invL2 ((void *)dstBuf.buf, PCIE_EXAMPLE_DSTBUF_BYTES, CACHE_FENCE_WAIT);

/* Reenable Interrupts. */
_restore_interrupts(key);

//cache_invalidate ((void *)dstBuf.buf, PCIE_EXAMPLE_DSTBUF_BYTES);
} while(dstBuf.buf[PCIE_BUFSIZE_APP] != PCIE_EXAMPLE_BUF_FULL);


/* check all the data */
for (i=0; i<PCIE_BUFSIZE_APP; i++)
{
if(dstBuf.buf[i] != srcBuf[i])
{
printf ("Received data = %u\nTransmited data = %u\nIndex = %u.\n\nTest failed.\n",
(unsigned)dstBuf.buf[i], (unsigned)srcBuf[i], (unsigned)i);
exit(1);
}
}

printf ("Root Complex received data.\n");
#ifdef PCIE_REV1_HW
pcieRcWaitInts (handle, sem, pcieBase);
#endif



#ifdef PCIE_EXAMPLE_DMA_RC
if (PcieExampleEdmaRC(pciedstBufBase, 0xbabeface, 100000,
(EDMA3_DRV_Handle)hEdma
)) {
printf ("Failed to pass token \n");
exit(1);
}

printf ("Root Complex DMA received data.\n");
printf ("EDMA Test passed.\n");
printf ("\n=== PCIe results using DMA transfers (optimal)===\n");
printf ("Passed 1 tokens round trip (read+write through PCIe) in %d cycles\n",
(unsigned int)(totalDMATime));
printf ("=== this is not an optimized example ===\n");
#endif
}
else
{
/**********************************************************************/
/* Wait for a single message from the RC then echo it back */
/**********************************************************************/

/* EP waits for the data received from RC */
do {
cache_invalidate ((void *)dstBuf.buf, PCIE_EXAMPLE_DSTBUF_BYTES);
} while(dstBuf.buf[PCIE_BUFSIZE_APP] != PCIE_EXAMPLE_BUF_FULL);

printf ("End Point received data.\n");

/* Loopback to RC what was written in the DST buffer.
Write from EP to RC */
for (i=0; i<PCIE_BUFSIZE_APP; i++)
{
pciedstBufBase->buf[i] = dstBuf.buf[i];
}

/* Mark that the buffer is full, so RC can process it */
pciedstBufBase->buf[PCIE_BUFSIZE_APP] = PCIE_EXAMPLE_BUF_FULL;

/* Note on cache coherence: Write back is not necessary because pcieBase is in
peripheral address space instead of physical memory*/

printf ("End Point sent data to Root Complex, completing the loopback.\n");
#ifdef PCIE_REV1_HW
/* Send msi to EP */
pcieEpSendInts (handle);
#endif
printf ("End of Test.\n");

#ifdef PCIE_EXAMPLE_DMA_EP
if (PcieExampleEdmaEP(pciedstBufBase, 0xbabeface, 100000,
(EDMA3_DRV_Handle)hEdma))
{
printf ("Failed to pass token \n");
exit(1);
}
printf ("End Point sent data to Root Complex, DMA completing the loopback.\n");
printf ("End of DMA Test.\n");
#endif
}

#ifdef EDMA
edmaDeinit(hEdma);
#endif
#ifdef EDMAPKTBENCH
if (PcieEdmaPktBench(&dstBuf.edmaPktBenchBuf.msiTracker[0],
&pciedstBufBase->edmaPktBenchBuf,
PcieModeGbl, sem))
{
printf ("EDMA packet IO benchmark failed to execute correctly\n");
exit(1);
}
#endif
printf ("Test passed.\n");
#ifndef IO_CONSOLE
printf ("Test passed.\n");
#endif

// BIOS_exit(0);

exit (0);

}



void read_devStatCtrlReg(Pcie_Handle handle)
{
pcieRegisters_t getRegs;
pcieDevStatCtrlReg_t devStatCtrl;
uint8_t val;

memset (&getRegs, 0, sizeof(getRegs));
memset (&devStatCtrl, 0, sizeof(devStatCtrl));

getRegs.devStatCtrl = &devStatCtrl;

Pcie_readRegs (handle, pcie_LOCATION_LOCAL, &getRegs);

val = (getRegs.devStatCtrl->corrEr | getRegs.devStatCtrl->nFatalEr << 1 | getRegs.devStatCtrl->fatalEr << 2 | getRegs.devStatCtrl->rqDet << 3);

if (val == 1)
{
printf("Correctable error detected\n");
}

else if (val == 2)
{
printf("Non-fatal error detected\n");
}

else if (val == 4)
{
printf("Fatal error detected\n");
}

else if (val == 10)
{
printf("Unsupported request error detected\n");
}

else
{
printf("No error detected\n");
}
}