TMS320C6655: FFT problem

I am trying to use DSPLIB to do a 2D FFT on an image. I'm starting with an image, and doing FFT, then IFFT to get back the same exact result. I must be doing something wrong as I am not getting the desired result.

I used the example from here, and adapted it to work with the C6655 DSPLIB:

#include <math.h>
#include <ti/dsplib/dsplib.h>

/* Number of samples for which FFT needs to be calculated */
#define N 512

/* Input NxN grayscale 8-bpp image */
char image[N][N]= {
0x32,0x31,0x3B // ... see TI example link for full image array
};

float colreal[N][N]={0};
float colimag[N][N]= {0};

/* Align the tables that we have to use */
#pragma DATA_ALIGN(8)
float   x_ref [2*N];

#pragma DATA_ALIGN(8)
float   x_32x32 [2*N];
#pragma DATA_ALIGN(8)
float   y_32x32 [2*N];
#pragma DATA_ALIGN(8)
float   w_32x32 [2*N];

unsigned char brev[64] = {
    0x0, 0x20, 0x10, 0x30, 0x8, 0x28, 0x18, 0x38,
    0x4, 0x24, 0x14, 0x34, 0xc, 0x2c, 0x1c, 0x3c,
    0x2, 0x22, 0x12, 0x32, 0xa, 0x2a, 0x1a, 0x3a,
    0x6, 0x26, 0x16, 0x36, 0xe, 0x2e, 0x1e, 0x3e,
    0x1, 0x21, 0x11, 0x31, 0x9, 0x29, 0x19, 0x39,
    0x5, 0x25, 0x15, 0x35, 0xd, 0x2d, 0x1d, 0x3d,
    0x3, 0x23, 0x13, 0x33, 0xb, 0x2b, 0x1b, 0x3b,
    0x7, 0x27, 0x17, 0x37, 0xf, 0x2f, 0x1f, 0x3f
};

/*
    This function generates the input data and also updates the
    input data arrays used by the various FFT kernels
*/
int count=0;
void generateInput () {
    int i;

    /* Normalize input image data, scale by N for FFT, and convert from floating point
    to fixed point */
    for (i = 0; i < N; i++) {
        x_ref[i] = ((float)image[count][i]/(float)255);
    }

    /* Copy the reference image input data into the input array */
    for (i = 0; i < N; i++) {
        x_32x32[2*i] = x_ref[i];
        x_32x32[2*i + 1] = 0;
    }
}

/* Function for generating Specialized sequence of twiddle factors */
void gen_twiddle_fft_sp (float *w, int n)
{
    int i, j, k;
    double x_t, y_t, theta1, theta2, theta3;

    for (j = 1, k = 0; j <= n >> 2; j = j << 2)
    {
        for (i = 0; i < n >> 2; i += j)
        {
            theta1 = 2 * PI * i / n;
            x_t = cos (theta1);
            y_t = sin (theta1);
            w[k] = (float) x_t;
            w[k + 1] = (float) y_t;

            theta2 = 4 * PI * i / n;
            x_t = cos (theta2);
            y_t = sin (theta2);
            w[k + 2] = (float) x_t;
            w[k + 3] = (float) y_t;

            theta3 = 6 * PI * i / n;
            x_t = cos (theta3);
            y_t = sin (theta3);
            w[k + 4] = (float) x_t;
            w[k + 5] = (float) y_t;
            k += 6;
        }
    }
}

void FFTTest()
{
    int i,j;

    /* Genarate twiddle factors */
    gen_twiddle_fft_sp(w_32x32, N);

    /* Do FFTs across each row of the input image */
    for(count =0;count<N;count++)
    {
        /* Generate the input data */
        generateInput ();

        /* Call FFT routine */
        DSPF_sp_fftSPxSP(N, x_32x32, w_32x32, y_32x32, brev, 4, 0, N);

        for (i = 0, j = 0; j < N; i+=2, j++)
        {
            colreal[count][j] = y_32x32[i];
            colimag[count][j] = y_32x32[i + 1];
        }
    }

    /* Do FFTs across each column of the output array of the previous row FFTs*/
    for(count =0;count<N;count++)
    {
        for (i = 0, j = 0; j < N; i+=2, j++) {
            x_32x32[i] = colreal[j][count];
            x_32x32[i + 1] = colimag[j][count];
        }

        /* Call FFT routine */
        DSPF_sp_fftSPxSP(N, x_32x32, w_32x32, y_32x32, brev, 4, 0, N);

        for (i = 0, j = 0; j < N; i+=2, j++) {
            colreal[j][count] = y_32x32[i];
            colimag[j][count] = y_32x32[i + 1];
        }
    }

    /* Do IFFT across each row of the previous 2D-FFT output array */
    for(count =0;count<N;count++)
    {
        /* Generate the input data */
        for (i = 0, j = 0; j < N; i+=2, j++) {
            x_32x32[i] = colreal[count][j];
            x_32x32[i + 1] = colimag[count][j];
        }

        /* Call FFT routine */
        DSPF_sp_ifftSPxSP(N, x_32x32, w_32x32, y_32x32, brev, 4, 0, N);

        for (i = 0, j = 0; j < N; i+=2, j++)
        {
            colreal[count][j] = y_32x32[i];
            colimag[count][j] = y_32x32[i + 1];
        }
    }

    /* Do IFFT across each column of the output array of the previous row IFFTs */
    for(count =0;count<N;count++)
    {
        for (i = 0, j = 0; j < N; i+=2, j++) {
            x_32x32[i] = colreal[j][count];
            x_32x32[i + 1] = colimag[j][count];
        }

        /* Call FFT routine */
        DSPF_sp_ifftSPxSP(N, x_32x32, w_32x32, y_32x32, brev, 4, 0, N);

        for (i = 0, j = 0; j < N; i+=2, j++) {
            colreal[j][count] = y_32x32[i];
            colimag[j][count] = y_32x32[i + 1];
        }
    }

    /* Convert output from fixed point to 8-bit integer to get final image output */
    for (i = 0; i < N; i++)
    {
        for (j = 0; j < N; j++)
        {
            image[i][j] = colreal[i][j] * (float)255;
        }
    }
}

What am I doing wrong?