TMS320C6678: A performance benchmark on OpenMP throughput using MSMC vs DDR3 buffers. Why OMP parallel run is slower than serial execution??

#include <stdio.h>
#include <c6x.h>
#include <math.h>
#include <omp.h>

#include <xdc/std.h>
#include <xdc/runtime/IHeap.h>
#include <xdc/runtime/System.h>
#include <xdc/runtime/Memory.h>
#include <xdc/runtime/Error.h>
#include <xdc/cfg/global.h>

typedef unsigned long long dt_uint64;
//typedef char buff_datatype;
//typedef short buff_datatype;
//typedef int buff_datatype;
typedef float buff_datatype;
//typedef double buff_datatype;
//typedef long double buff_datatype;


// buffer on MSMC
#pragma DATA_SECTION( msmc_buff, ".msmc_bufSec" )
#pragma DATA_ALIGN( msmc_buff, 128 )
#define BUFLEN  (40*1024)
buff_datatype msmc_buff[BUFLEN];

// buffer on DDR3 allcoated from heap
buff_datatype *ddr_buff;

// external msmc heap
extern const HeapBuf_Handle msmcHeap;
buff_datatype *msmc_heap;

int main(int argc, char* argv[]){
    printf("-= Program Started=- \n");

    TSCH = 0; TSCL = 0;
    dt_uint64 t_start, t_end;

    Error_Block eb;
    Error_init(&eb);
    int blockSize = 256;

    // NOOMP, Buffer on MSMC
    t_start = _itoll(TSCH, TSCL);
    for(int j = 0; j < BUFLEN; j++){
        msmc_buff[j] = 1+(msmc_buff[j]/2*3-50)/2*3-50;
    }
    t_end = _itoll(TSCH, TSCL);
    printf("NOOMP MSMC BUFF PROCESS TIME: %llu \n", t_end - t_start);

    t_start = _itoll(TSCH, TSCL);
    // OMP, Buffer on MSMC
#pragma omp parallel num_threads(8)
    {
#pragma omp for
        for(int j = 0; j < BUFLEN; j++){
//          putchar('M'); putchar('0' + omp_get_thread_num());
            msmc_buff[j] = 1+(msmc_buff[j]/2*3-50)/2*3-50;
        }
    }
      t_end = _itoll(TSCH, TSCL);
    printf("OMP MSMC BUFF PROCESS TIME: %llu \n", t_end - t_start);

    ddr_buff = Memory_alloc(NULL, BUFLEN*sizeof(buff_datatype), 128, &eb);

    if (!ddr_buff){
        printf("Cannot allocate ddr_buff from heap \n");
        exit(1);
    }

    // NO-OMP, Buffer on DDR
    t_start = _itoll(TSCH, TSCL);
        for(int j = 0; j < BUFLEN; j++){
            ddr_buff[j] = 1+(ddr_buff[j]/2*3-50)/2*3-50;
        }
    t_end = _itoll(TSCH, TSCL);
    printf("NOOMP DDR BUFF PROCESS TIME: %llu \n", t_end - t_start);

    // OMP, Buffer on DDR
    t_start = _itoll(TSCH, TSCL);
#pragma omp parallel num_threads(8)
    {
#pragma omp for
        for(int j = 0; j < BUFLEN; j++){
            ddr_buff[j] = 1+(ddr_buff[j]/2*3-50)/2*3-50;
        }
    }
    t_end = _itoll(TSCH, TSCL);
    printf("OMP DDR BUFF PROCESS TIME: %llu \n", t_end - t_start);



    Memory_free(NULL, ddr_buff, BUFLEN*sizeof(buff_datatype));

    printf("-= Program Ended=- \n");
}

/***************************/
/* SECTION MAPPING         */
/***************************/
var program = xdc.useModule('xdc.cfg.Program');

program.sectMap[".args"]        = new Program.SectionSpec();
program.sectMap[".bss"]         = new Program.SectionSpec();
program.sectMap[".cinit"]       = new Program.SectionSpec();
program.sectMap[".cio"]         = new Program.SectionSpec();
program.sectMap[".const"]       = new Program.SectionSpec();
program.sectMap[".data"]        = new Program.SectionSpec();
program.sectMap[".far"]         = new Program.SectionSpec();
program.sectMap[".fardata"]     = new Program.SectionSpec();
program.sectMap[".neardata"]    = new Program.SectionSpec();
program.sectMap[".rodata"]      = new Program.SectionSpec();
program.sectMap[".stack"]       = new Program.SectionSpec();
program.sectMap[".switch"]      = new Program.SectionSpec();
program.sectMap[".sysmem"]      = new Program.SectionSpec();
program.sectMap[".text"]        = new Program.SectionSpec();
    
// Must place these sections in core local memory 
program.sectMap[".args"].loadSegment        = "DDR3";
program.sectMap[".cio"].loadSegment         = "L2SRAM";

// Variables in the following data sections can potentially be 'shared' in
// OpenMP. These sections must be placed in shared memory.
program.sectMap[".bss"].loadSegment         = "DDR3";
program.sectMap[".cinit"].loadSegment       = "DDR3";
program.sectMap[".const"].loadSegment       = "DDR3";
program.sectMap[".data"].loadSegment        = "DDR3";
program.sectMap[".far"].loadSegment         = "DDR3";
program.sectMap[".fardata"].loadSegment     = "DDR3";
program.sectMap[".neardata"].loadSegment    = "DDR3";
program.sectMap[".rodata"].loadSegment      = "DDR3";
program.sectMap[".sysmem"].loadSegment      = "DDR3";

// Code sections shared by cores - place in shared memory to avoid duplication
program.sectMap[".switch"].loadSegment      = program.platform.codeMemory;
program.sectMap[".text"].loadSegment        = "MSMCSRAM";
print(" program.platform.codeMemory = ", program.platform.codeMemory);

// Size the default stack and place it in L2SRAM 
var deviceName = String(Program.cpu.deviceName);
// if  (deviceName.search("DRA7XX") == -1) { program.stack = 0x20000; }
// if  (deviceName.search("DRA7XX") == -1) { program.stack = 256*1024; } // working in serial mode 
// if  (deviceName.search("DRA7XX") == -1) { program.stack = 330*1024; }
if  (deviceName.search("DRA7XX") == -1) { program.stack = 330*1024; }
// if  (deviceName.search("DRA7XX") == -1) { program.stack = 512*1024; } 
else                                    { program.stack = 0x8000;  }
program.sectMap[".stack"].loadSegment       = "L2SRAM"; // works in serial mode 256KB .stack
// program.sectMap[".stack"].loadSegment       = "MSMCSRAM"; // doesnt work in OpenMP

// Since there are no arguments passed to main, set .args size to 0
program.argSize = 0;


/********************************/
/* OPENMP RUNTIME CONFIGURATION */
/********************************/
USE_OPENMP = true;
if (USE_OPENMP) {
	// Include OMP runtime in the build
	var ompSettings = xdc.useModule("ti.runtime.openmp.Settings");
	
	// Set to true if the application uses or has dependencies on BIOS components
	ompSettings.usingRtsc = true;
	
	if (ompSettings.usingRtsc)
	{
	    /* Configure OpenMP for BIOS
	     * - OpenMP.configureCores(masterCoreId, numberofCoresInRuntime)
	     *       Configures the id of the master core and the number of cores
	     *       available to the runtime.
	     */
	
	    var OpenMP = xdc.useModule('ti.runtime.ompbios.OpenMP');
	
	    // Configure the index of the master core and the number of cores available
	    // to the runtime. The cores are contiguous.
	    OpenMP.masterCoreIdx = 0;
	
	    // Setup number of cores based on the device
	    if      (deviceName.search("DRA7XX") != -1) { OpenMP.numCores = 2; }
	    else if (deviceName.search("6670")   != -1) { OpenMP.numCores = 4; }
	    else if (deviceName.search("6657")   != -1) { OpenMP.numCores = 2; }
	    else if (deviceName.search("K2L")    != -1) { OpenMP.numCores = 4; }
	    else                                        { OpenMP.numCores = 8; }
	
	    // Pull in memory ranges described in Platform.xdc to configure the runtime
	    var ddr3       = Program.cpu.memoryMap["DDR3"];
	    var ddr3_nc    = Program.cpu.memoryMap["DDR3_NC"];
	    var msmc       = Program.cpu.memoryMap["MSMCSRAM"];
	    var msmcNcVirt = Program.cpu.memoryMap["OMP_MSMC_NC_VIRT"];
	    var msmcNcPhy  = Program.cpu.memoryMap["OMP_MSMC_NC_PHY"];
	
	    // Initialize the runtime with memory range information
	    if (deviceName.search("DRA7XX") == -1) {
	       OpenMP.msmcBase = msmc.base
	       OpenMP.msmcSize = msmc.len;
	
	       OpenMP.msmcNoCacheVirtualBase  = msmcNcVirt.base;
	       OpenMP.msmcNoCacheVirtualSize  = msmcNcVirt.len;
	
	       OpenMP.msmcNoCachePhysicalBase  = msmcNcPhy.base;
	       
//	       OpenMP.allocateStackFromHeap = true;
//	       OpenMP.allocateStackFromHeapSize = 4*330*1024;
	       
	    }
	    else
	    {
	       OpenMP.allocateStackFromHeap = true;
	       OpenMP.allocateStackFromHeapSize = 0x010000;
	
	       OpenMP.hasMsmc = false;
	       OpenMP.ddrNoCacheBase = ddr3_nc.base;
	       OpenMP.ddrNoCacheSize = ddr3_nc.len;
	    }
	
	    OpenMP.ddrBase          = ddr3.base;
	    OpenMP.ddrSize          = ddr3.len;
	
	    // Configure memory allocation using HeapOMP
	    // HeapOMP handles 
	    // - Memory allocation requests from BIOS components (core local memory)
	    // - Shared memory allocation by utilizing the IPC module to enable 
	    //   multiple cores to allocate memory out of the same heap - used by malloc
	    if (deviceName.search("DRA7XX") == -1) {
	       print("deviceName.search(\"DRA7XX\") == -1")
	       var HeapOMP = xdc.useModule('ti.runtime.ompbios.HeapOMP');
	
	       // Shared Region 0 must be initialized for IPC 
	       var sharedRegionId = 0;
	
	       // Size of the core local heap
	       var localHeapSize  = 0x8000;
	
	       // Size of the heap shared by all the cores
	       // var sharedHeapSize = 0x08000000;
	       var sharedHeapSize = (512-32-22)*1024*1024;
	
	       // Initialize a Shared Region & create a heap in the DDR3 memory region
	       var SharedRegion   = xdc.useModule('ti.sdo.ipc.SharedRegion');
	       SharedRegion.setEntryMeta( sharedRegionId,
	                                  {   base: ddr3.base,
	                                      len:  sharedHeapSize,
	                                      ownerProcId: OpenMP.masterCoreIdx,
	                                      cacheEnable: true,
	                                      createHeap: true,
	                                      isValid: true,
	                                      name: "DDR3_SR0",
	                                  });
	
	       // Configure and setup HeapOMP
	       HeapOMP.configure(sharedRegionId, localHeapSize);
	
	/*       
	       var HeapTrack = xdc.useModule('ti.sysbios.heaps.HeapTrack'); 
			var heapTrackParams = new HeapTrack.Params; 
			heapTrackParams.heap = HeapOMP; 
			var myHeapTracker = HeapTrack.create(heapTrackParams);
			*/
	    }
	    else
	    {
	       OpenMP.useIpcSharedHeap = false;
	       OpenMP.allocateLocalHeapSize = 0x8000
	       OpenMP.allocateSharedHeapSize = 0x00800000
	    }
	
	    
	    var Startup = xdc.useModule('xdc.runtime.Startup');
	    Startup.lastFxns.$add('&__TI_omp_initialize_rtsc_mode');
	}
	else
	{
	    /* Size the heap. It must be placed in shared memory */
	    program.heap = sharedHeapSize;
	}
	

// Use no openmp 
} else {
	var BIOS = xdc.useModule('ti.sysbios.BIOS');	
	BIOS.heapSize = (512-32-22)*1024*1024;
	Program.sectMap["systemHeap"]	= new Program.SectionSpec();
	Program.sectMap["systemHeap"].loadSegment	= "DDR3";
	BIOS.heapSection = "systemHeap";	
}


var BIOS = xdc.useModule('ti.sysbios.BIOS');
print("BIOS.heapSize = ", BIOS.heapSize)
BIOS.cpuFreq.lo = 1000000000;
Program.sectMap[".msmc_bufSec"]    = "MSMCSRAM";