#ifdef _MSC_VER

#define _CRT_SECURE_NO_WARNINGS

#endif

#include <assert.h>

#include <stdarg.h>

#include <stddef.h>

#include <stdlib.h>

#include <string.h>

#include <errno.h>

#include "private.h"

#include "gpuarray/config.h"

#include "gpuarray/array.h"

#include "gpuarray/error.h"

#include "gpuarray/kernel.h"

#include "gpuarray/util.h"

#include "util/strb.h"

#include "util/xxhash.h"

struct extcopy_args {

int itype;

int otype;

};

static int extcopy_eq(cache_key_t _k1, cache_key_t _k2) {

struct extcopy_args *k1 = _k1;

struct extcopy_args *k2 = _k2;

return k1->itype == k2->itype && k1->otype == k2->otype;

}

static void extcopy_free(cache_key_t k) {

free(k);

}

static uint32_t extcopy_hash(cache_key_t k) {

return XXH32(k, sizeof(struct extcopy_args), 42);

}

static int ga_extcopy(GpuArray *dst, const GpuArray *src) {

struct extcopy_args a, *aa;

gpucontext *ctx = gpudata_context(dst->data);

GpuElemwise *k = NULL;

void *args[2];

if (ctx != gpudata_context(src->data))

return GA_INVALID_ERROR;

a.itype = src->typecode;

a.otype = dst->typecode;

if (ctx->extcopy_cache != NULL)

k = cache_get(ctx->extcopy_cache, &a);

if (k == NULL) {

gpuelemwise_arg gargs[2];

gargs[0].name = "src";

gargs[0].typecode = src->typecode;

gargs[0].flags = GE_READ;

gargs[1].name = "dst";

gargs[1].typecode = dst->typecode;

gargs[1].flags = GE_WRITE;

k = GpuElemwise_new(ctx, "", "dst = src", 2, gargs, 0, 0);

if (k == NULL)

return GA_MISC_ERROR;

aa = memdup(&a, sizeof(a));

if (aa == NULL) {

GpuElemwise_free(k);

return GA_MEMORY_ERROR;

}

if (ctx->extcopy_cache == NULL)

ctx->extcopy_cache = cache_twoq(4, 8, 8, 2, extcopy_eq, extcopy_hash,

extcopy_free,

(cache_freev_fn)GpuElemwise_free);

if (ctx->extcopy_cache == NULL)

return GA_MISC_ERROR;

if (cache_add(ctx->extcopy_cache, aa, k) != 0)

return GA_MISC_ERROR;

}

args[0] = (void *)src;

args[1] = (void *)dst;

return GpuElemwise_call(k, args, GE_BROADCAST);

}

/* Value below which a size_t multiplication will never overflow. */

#define MUL_NO_OVERFLOW (1ULL << (sizeof(size_t) * 4))

void GpuArray_fix_flags(GpuArray *a) {

/* Only keep the writable flag */

a->flags &= GA_WRITEABLE;

/* Set the other flags if applicable */

if (GpuArray_is_c_contiguous(a)) a->flags |= GA_C_CONTIGUOUS;

if (GpuArray_is_f_contiguous(a)) a->flags |= GA_F_CONTIGUOUS;

if (GpuArray_is_aligned(a)) a->flags |= GA_ALIGNED;

}

int GpuArray_empty(GpuArray *a, gpucontext *ctx, int typecode,

unsigned int nd, const size_t *dims, ga_order ord) {

size_t size = gpuarray_get_elsize(typecode);

unsigned int i;

int res = GA_NO_ERROR;

if (ord == GA_ANY_ORDER)

ord = GA_C_ORDER;

if (ord != GA_C_ORDER && ord != GA_F_ORDER)

return GA_VALUE_ERROR;

for (i = 0; i < nd; i++) {

size_t d = dims[i];

/* Check for overflow */

if ((d >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&

d > 0 && SIZE_MAX / d < size)

return GA_VALUE_ERROR;

size *= d;

}

a->data = gpudata_alloc(ctx, size, NULL, 0, &res);

if (a->data == NULL) return res;

a->nd = nd;

a->offset = 0;

a->typecode = typecode;

a->dimensions = calloc(nd, sizeof(size_t));

a->strides = calloc(nd, sizeof(ssize_t));

/* F/C distinction comes later */

a->flags = GA_BEHAVED;

if (a->dimensions == NULL || a->strides == NULL) {

GpuArray_clear(a);

return GA_MEMORY_ERROR;

}

/* Mult will not overflow since calloc succeded */

memcpy(a->dimensions, dims, sizeof(size_t)*nd);

size = gpuarray_get_elsize(typecode);

/* mults will not overflow, checked on entry */

switch (ord) {

case GA_C_ORDER:

for (i = nd; i > 0; i--) {

a->strides[i-1] = size;

size *= a->dimensions[i-1];

}

a->flags |= GA_C_CONTIGUOUS;

break;

case GA_F_ORDER:

for (i = 0; i < nd; i++) {

a->strides[i] = size;

size *= a->dimensions[i];

}

a->flags |= GA_F_CONTIGUOUS;

break;

default:

assert(0); /* cannot be reached */

}

if (a->nd <= 1)

a->flags |= GA_F_CONTIGUOUS|GA_C_CONTIGUOUS;

return GA_NO_ERROR;

}

int GpuArray_zeros(GpuArray *a, gpucontext *ctx,

int typecode, unsigned int nd, const size_t *dims,

ga_order ord) {

int err;

err = GpuArray_empty(a, ctx, typecode, nd, dims, ord);

if (err != GA_NO_ERROR)

return err;

err = gpudata_memset(a->data, a->offset, 0);

if (err != GA_NO_ERROR) {

GpuArray_clear(a);

}

return err;

}

int GpuArray_fromdata(GpuArray *a, gpudata *data, size_t offset, int typecode,

unsigned int nd, const size_t *dims,

const ssize_t *strides, int writeable) {

if (gpuarray_get_type(typecode)->typecode != typecode)

return GA_VALUE_ERROR;

assert(data != NULL);

a->data = data;

gpudata_retain(a->data);

a->nd = nd;

a->offset = offset;

a->typecode = typecode;

a->dimensions = calloc(nd, sizeof(size_t));

a->strides = calloc(nd, sizeof(ssize_t));

a->flags = (writeable ? GA_WRITEABLE : 0);

if (a->dimensions == NULL || a->strides == NULL) {

GpuArray_clear(a);

return GA_MEMORY_ERROR;

}

memcpy(a->dimensions, dims, nd*sizeof(size_t));

memcpy(a->strides, strides, nd*sizeof(ssize_t));

GpuArray_fix_flags(a);

return GA_NO_ERROR;

}

int GpuArray_copy_from_host(GpuArray *a, gpucontext *ctx, void *buf,

int typecode, unsigned int nd, const size_t *dims,

const ssize_t *strides) {

char *base = (char *)buf;

size_t offset = 0;

size_t size = gpuarray_get_elsize(typecode);

gpudata *b;

int err;

unsigned int i;

for (i = 0; i < nd; i++) {

if (dims[i] == 0) {

size = 0;

base = (char *)buf;

break;

}

if (strides[i] < 0)

base += (dims[i]-1) * strides[i];

else

size += (dims[i]-1) * strides[i];

}

offset = (char *)buf - base;

size += offset;

b = gpudata_alloc(ctx, size, base, GA_BUFFER_INIT, &err);

if (b == NULL) return err;

err = GpuArray_fromdata(a, b, offset, typecode, nd, dims, strides, 1);

gpudata_release(b);

return err;

}

int GpuArray_view(GpuArray *v, const GpuArray *a) {

v->data = a->data;

gpudata_retain(a->data);

v->nd = a->nd;

v->offset = a->offset;

v->typecode = a->typecode;

v->flags = a->flags;

v->dimensions = calloc(v->nd, sizeof(size_t));

v->strides = calloc(v->nd, sizeof(ssize_t));

if (v->dimensions == NULL || v->strides == NULL) {

GpuArray_clear(v);

return GA_MEMORY_ERROR;

}

memcpy(v->dimensions, a->dimensions, v->nd*sizeof(size_t));

memcpy(v->strides, a->strides, v->nd*sizeof(ssize_t));

return GA_NO_ERROR;

}

int GpuArray_sync(GpuArray *a) {

return gpudata_sync(a->data);

}

int GpuArray_index_inplace(GpuArray *a, const ssize_t *starts,

const ssize_t *stops, const ssize_t *steps) {

unsigned int i, new_i;

unsigned int new_nd = a->nd;

size_t *newdims;

ssize_t *newstrs;

size_t new_offset = a->offset;

if ((starts == NULL) || (stops == NULL) || (steps == NULL))

return GA_VALUE_ERROR;

for (i = 0; i < a->nd; i++) {

if (steps[i] == 0) new_nd -= 1;

}

newdims = calloc(new_nd, sizeof(size_t));

newstrs = calloc(new_nd, sizeof(ssize_t));

if (newdims == NULL || newstrs == NULL) {

free(newdims);

free(newstrs);

return GA_MEMORY_ERROR;

}

new_i = 0;

for (i = 0; i < a->nd; i++) {

if (starts[i] < -1 || (starts[i] > 0 &&

(size_t)starts[i] > a->dimensions[i])) {

free(newdims);

free(newstrs);

return GA_VALUE_ERROR;

}

if (steps[i] == 0 &&

(starts[i] == -1 || (size_t)starts[i] >= a->dimensions[i])) {

free(newdims);

free(newstrs);

return GA_VALUE_ERROR;

}

new_offset += starts[i] * a->strides[i];

if (steps[i] != 0) {

if ((stops[i] < -1 || (stops[i] > 0 &&

(size_t)stops[i] > a->dimensions[i])) ||

(stops[i]-starts[i])/steps[i] < 0) {

free(newdims);

free(newstrs);

return GA_VALUE_ERROR;

}

newstrs[new_i] = steps[i] * a->strides[i];

newdims[new_i] = (stops[i]-starts[i]+steps[i]-

(steps[i] < 0? -1 : 1))/steps[i];

new_i++;

}

}

a->nd = new_nd;

a->offset = new_offset;

free(a->dimensions);

a->dimensions = newdims;

free(a->strides);

a->strides = newstrs;

GpuArray_fix_flags(a);

return GA_NO_ERROR;

}

int GpuArray_index(GpuArray *r, const GpuArray *a, const ssize_t *starts,

const ssize_t *stops, const ssize_t *steps) {

int err;

err = GpuArray_view(r, a);

if (err != GA_NO_ERROR) return err;

err = GpuArray_index_inplace(r, starts, stops, steps);

if (err != GA_NO_ERROR) GpuArray_clear(r);

return err;

}

static int gen_take1_kernel(GpuKernel *k, gpucontext *ctx, char **err_str,

GpuArray *a, const GpuArray *v,

const GpuArray *ind, int addr32) {

strb sb = STRB_STATIC_INIT;

int *atypes;

char *sz, *ssz;

unsigned int i, i2;

unsigned int nargs, apos;

int flags = GA_USE_CLUDA;

int res;

nargs = 9 + 2 * v->nd;

atypes = calloc(nargs, sizeof(int));

if (atypes == NULL)

return GA_MEMORY_ERROR;

if (addr32) {

sz = "ga_uint";

ssz = "ga_int";

} else {

sz = "ga_size";

ssz = "ga_ssize";

}

apos = 0;

strb_appendf(&sb, "KERNEL void take1(GLOBAL_MEM %s *r, ga_size r_off, "

"GLOBAL_MEM const %s *v, ga_size v_off,",

gpuarray_get_type(a->typecode)->cluda_name,

gpuarray_get_type(v->typecode)->cluda_name);

atypes[apos++] = GA_BUFFER;

atypes[apos++] = GA_SIZE;

atypes[apos++] = GA_BUFFER;

atypes[apos++] = GA_SIZE;

for (i = 0; i < v->nd; i++) {

strb_appendf(&sb, " ga_ssize s%u, ga_size d%u,", i, i);

atypes[apos++] = GA_SSIZE;

atypes[apos++] = GA_SIZE;

}

strb_appendf(&sb, " GLOBAL_MEM const %s *ind, ga_size i_off, "

"ga_size n0, ga_size n1, GLOBAL_MEM int* err) {\n",

gpuarray_get_type(ind->typecode)->cluda_name);

atypes[apos++] = GA_BUFFER;

atypes[apos++] = GA_SIZE;

atypes[apos++] = GA_SIZE;

atypes[apos++] = GA_SIZE;

atypes[apos++] = GA_BUFFER;

assert(apos == nargs);

strb_appendf(&sb, " const %s idx0 = LDIM_0 * GID_0 + LID_0;\n"

" const %s numThreads0 = LDIM_0 * GDIM_0;\n"

" const %s idx1 = LDIM_1 * GID_1 + LID_1;\n"

" const %s numThreads1 = LDIM_1 * GDIM_1;\n"

" %s i0, i1;\n", sz, sz, sz, sz, sz);

strb_appends(&sb, " if (idx0 >= n0 || idx1 >= n1) return;\n");

strb_appendf(&sb, " r = (GLOBAL_MEM %s *)(((GLOBAL_MEM char *)r) + r_off);\n"

" ind = (GLOBAL_MEM %s *)(((GLOBAL_MEM char *)ind) + i_off);\n",

gpuarray_get_type(a->typecode)->cluda_name,

gpuarray_get_type(ind->typecode)->cluda_name);

strb_appendf(&sb, " for (i0 = idx0; i0 < n0; i0 += numThreads0) {\n"

" %s ii0 = ind[i0];\n"

" %s pos0 = v_off;\n"

" if (ii0 < 0) ii0 += d0;\n"

" if ((ii0 < 0) || (ii0 >= d0)) {\n"

" *err = -1;\n"

" continue;\n"

" }\n"

" pos0 += ii0 * (%s)s0;\n"

" for (i1 = idx1; i1 < n1; i1 += numThreads1) {\n"

" %s p = pos0;\n", ssz, sz, sz, sz);

if (v->nd > 1) {

strb_appendf(&sb, " %s pos, ii = i1;\n", sz);

for (i2 = v->nd; i2 > 1; i2--) {

i = i2 - 1;

if (i > 1)

strb_appendf(&sb, " pos = ii %% (%s)d%u;\n"

" ii /= (%s)d%u;\n", sz, i, sz, i);

else

strb_appends(&sb, " pos = ii;\n");

strb_appendf(&sb, " p += pos * (%s)s%u;\n", ssz, i);

}

}

strb_appendf(&sb, " r[i0*((%s)n1) + i1] = *((GLOBAL_MEM %s *)(((GLOBAL_MEM char *)v) + p));\n",

sz, gpuarray_get_type(v->typecode)->cluda_name);

strb_appends(&sb, " }\n"

" }\n"

"}\n");

if (strb_error(&sb)) {

res = GA_MEMORY_ERROR;

goto bail;

}

flags |= gpuarray_type_flags(a->typecode, v->typecode, GA_BYTE, -1);

res = GpuKernel_init(k, ctx, 1, (const char **)&sb.s, &sb.l, "take1",

nargs, atypes, flags, err_str);

bail:

free(atypes);

strb_clear(&sb);

return res;

}

int GpuArray_take1(GpuArray *a, const GpuArray *v, const GpuArray *i,

int check_error) {

size_t n[2], ls[2] = {0, 0}, gs[2] = {0, 0};

size_t pl;

gpudata *errbuf;

#if DEBUG

char *errstr = NULL;

#endif

GpuKernel k;

unsigned int j;

unsigned int argp;

int err, kerr = 0;

int addr32 = 0;

if (!GpuArray_ISWRITEABLE(a))

return GA_INVALID_ERROR;

if (!GpuArray_ISALIGNED(a) || !GpuArray_ISALIGNED(v) ||

!GpuArray_ISALIGNED(i))

return GA_UNALIGNED_ERROR;

/* a and i have to be C contiguous */

if (!GpuArray_IS_C_CONTIGUOUS(a) || !GpuArray_IS_C_CONTIGUOUS(i))

return GA_INVALID_ERROR;

/* Check that the dimensions match namely a[0] == i[0] and a[>0] == v[>0] */

if (v->nd == 0 || a->nd == 0 || i->nd != 1 || a->nd != v->nd ||

a->dimensions[0] != i->dimensions[0])

return GA_INVALID_ERROR;

n[0] = i->dimensions[0];

n[1] = 1;

for (j = 1; j < v->nd; j++) {

if (a->dimensions[j] != v->dimensions[j])

return GA_INVALID_ERROR;

n[1] *= v->dimensions[j];

}

if (n[0] * n[1] < SADDR32_MAX) {

addr32 = 1;

}

err = gpudata_property(v->data, GA_CTX_PROP_ERRBUF, &errbuf);

if (err != GA_NO_ERROR)

return err;

err = gen_take1_kernel(&k, GpuArray_context(a),

#if DEBUG

&errstr,

#else

NULL,

#endif

a, v, i, addr32);

#if DEBUG

if (errstr != NULL) {

fprintf(stderr, "%s\n", errstr);

free(errstr);

}

#endif

if (err != GA_NO_ERROR)

return err;

err = GpuKernel_sched(&k, n[0]*n[1], &gs[1], &ls[1]);

if (err != GA_NO_ERROR)

goto out;

/* This may not be the best scheduling, but it's good enough */

err = gpukernel_property(k.k, GA_KERNEL_PROP_PREFLSIZE, &pl);

ls[0] = ls[1] / pl;

ls[1] = pl;

if (n[1] > n[0]) {

pl = ls[0];

ls[0] = ls[1];

ls[1] = pl;

gs[0] = 1;

} else {

gs[0] = gs[1];

gs[1] = 1;

}

argp = 0;

GpuKernel_setarg(&k, argp++, a->data);

GpuKernel_setarg(&k, argp++, (void *)&a->offset);

GpuKernel_setarg(&k, argp++, v->data);

/* The cast is to avoid a warning about const */

GpuKernel_setarg(&k, argp++, (void *)&v->offset);

for (j = 0; j < v->nd; j++) {

GpuKernel_setarg(&k, argp++, &v->strides[j]);

GpuKernel_setarg(&k, argp++, &v->dimensions[j]);

}

GpuKernel_setarg(&k, argp++, i->data);

GpuKernel_setarg(&k, argp++, (void *)&i->offset);

GpuKernel_setarg(&k, argp++, &n[0]);

GpuKernel_setarg(&k, argp++, &n[1]);

GpuKernel_setarg(&k, argp++, errbuf);

err = GpuKernel_call(&k, 2, gs, ls, 0, NULL);

if (check_error && err == GA_NO_ERROR) {

err = gpudata_read(&kerr, errbuf, 0, sizeof(int));

if (err == GA_NO_ERROR && kerr != 0) {

err = GA_VALUE_ERROR;

kerr = 0;

/* We suppose this will not fail */

gpudata_write(errbuf, 0, &kerr, sizeof(int));

}

}

out:

GpuKernel_clear(&k);

return err;

}

int GpuArray_setarray(GpuArray *a, const GpuArray *v) {

GpuArray tv;

size_t sz;

ssize_t *strs;

unsigned int i, off;

int err = GA_NO_ERROR;

int simple_move = 1;

if (a->nd < v->nd)

return GA_VALUE_ERROR;

if (!GpuArray_ISWRITEABLE(a))

return GA_VALUE_ERROR;

if (!GpuArray_ISALIGNED(v) || !GpuArray_ISALIGNED(a))

return GA_UNALIGNED_ERROR;

off = a->nd - v->nd;

for (i = 0; i < v->nd; i++) {

if (v->dimensions[i] != a->dimensions[i+off]) {

if (v->dimensions[i] != 1)

return GA_VALUE_ERROR;

else

simple_move = 0;

}

}

if (simple_move && GpuArray_ISONESEGMENT(a) && GpuArray_ISONESEGMENT(v) &&

GpuArray_ISFORTRAN(a) == GpuArray_ISFORTRAN(v) &&

a->typecode == v->typecode &&

a->nd == v->nd) {

sz = gpuarray_get_elsize(a->typecode);

for (i = 0; i < a->nd; i++) sz *= a->dimensions[i];

return gpudata_move(a->data, a->offset, v->data, v->offset, sz);

}

strs = calloc(a->nd, sizeof(ssize_t));

if (strs == NULL)

return GA_MEMORY_ERROR;

for (i = off; i < a->nd; i++) {

if (v->dimensions[i-off] == a->dimensions[i]) {

strs[i] = v->strides[i-off];

}

}

memcpy(&tv, v, sizeof(GpuArray));

tv.nd = a->nd;

tv.dimensions = a->dimensions;

tv.strides = strs;

if (tv.nd != 0)

GpuArray_fix_flags(&tv);

err = ga_extcopy(a, &tv);

free(strs);

return err;

}

int GpuArray_reshape(GpuArray *res, const GpuArray *a, unsigned int nd,

const size_t *newdims, ga_order ord, int nocopy) {

int err;

err = GpuArray_view(res, a);

if (err != GA_NO_ERROR) return err;

err = GpuArray_reshape_inplace(res, nd, newdims, ord);

if (err == GA_COPY_ERROR && !nocopy) {

GpuArray_clear(res);

GpuArray_copy(res, a, ord);

err = GpuArray_reshape_inplace(res, nd, newdims, ord);

}

if (err != GA_NO_ERROR) GpuArray_clear(res);

return err;

}

int GpuArray_reshape_inplace(GpuArray *a, unsigned int nd,

const size_t *newdims, ga_order ord) {

ssize_t *newstrides;

size_t *tmpdims;

size_t np;

size_t op;

size_t newsize = 1;

size_t oldsize = 1;

unsigned int ni = 0;

unsigned int oi = 0;

unsigned int nj = 1;

unsigned int oj = 1;

unsigned int nk;

unsigned int ok;

unsigned int i;

if (ord == GA_ANY_ORDER && GpuArray_ISFORTRAN(a) && a->nd > 1)

ord = GA_F_ORDER;

for (i = 0; i < a->nd; i++) {

oldsize *= a->dimensions[i];

}

for (i = 0; i < nd; i++) {

size_t d = newdims[i];

/* Check for overflow */

if ((d >= MUL_NO_OVERFLOW || newsize >= MUL_NO_OVERFLOW) &&

d > 0 && SIZE_MAX / d < newsize)

return GA_INVALID_ERROR;

newsize *= d;

}

if (newsize != oldsize) return GA_INVALID_ERROR;

/* If the source and desired layouts are the same, then just copy

strides and dimensions */

if ((ord == GA_C_ORDER && GpuArray_CHKFLAGS(a, GA_C_CONTIGUOUS)) ||

(ord == GA_F_ORDER && GpuArray_CHKFLAGS(a, GA_F_CONTIGUOUS))) {

goto do_final_copy;

}

newstrides = calloc(nd, sizeof(ssize_t));

if (newstrides == NULL)

return GA_MEMORY_ERROR;

while (ni < nd && oi < a->nd) {

np = newdims[ni];

op = a->dimensions[oi];

while (np != op) {

if (np < op) {

np *= newdims[nj++];

} else {

op *= a->dimensions[oj++];

}

}

for (ok = oi; ok < oj - 1; ok++) {

if (ord == GA_F_ORDER) {

if (a->strides[ok+1] != (ssize_t)a->dimensions[ok]*a->strides[ok])

goto need_copy;

} else {

if (a->strides[ok] != (ssize_t)a->dimensions[ok+1]*a->strides[ok+1])

goto need_copy;

}

}

if (ord == GA_F_ORDER) {

newstrides[ni] = a->strides[oi];

for (nk = ni + 1; nk < nj; nk++) {

newstrides[nk] = newstrides[nk - 1]*newdims[nk - 1];

}

} else {

newstrides[nj-1] = a->strides[oj-1];

for (nk = nj-1; nk > ni; nk--) {

newstrides[nk-1] = newstrides[nk]*newdims[nk];

}

}

ni = nj++;

oi = oj++;

}

/* Fixup trailing ones */

if (ord == GA_F_ORDER) {

for (i = nj-1; i < nd; i++) {

newstrides[i] = newstrides[i-1] * newdims[i-1];

}

} else {

for (i = nj-1; i < nd; i++) {

newstrides[i] = gpuarray_get_elsize(a->typecode);

}

}

/* We can reuse newstrides since it was allocated in this function.

Can't do the same with newdims (which is a parameter). */

tmpdims = calloc(nd, sizeof(size_t));

if (tmpdims == NULL) {

return GA_MEMORY_ERROR;

}

memcpy(tmpdims, newdims, nd*sizeof(size_t));

a->nd = nd;

free(a->dimensions);

free(a->strides);

a->dimensions = tmpdims;

a->strides = newstrides;

goto fix_flags;

need_copy:

free(newstrides);

return GA_COPY_ERROR;

do_final_copy:

tmpdims = calloc(nd, sizeof(size_t));

newstrides = calloc(nd, sizeof(ssize_t));

if (tmpdims == NULL || newstrides == NULL) {

free(tmpdims);

free(newstrides);

return GA_MEMORY_ERROR;

}

memcpy(tmpdims, newdims, nd*sizeof(size_t));

if (nd > 0) {

if (ord == GA_F_ORDER) {

newstrides[0] = gpuarray_get_elsize(a->typecode);

for (i = 1; i < nd; i++) {

newstrides[i] = newstrides[i-1] * tmpdims[i-1];

}

} else {

newstrides[nd-1] = gpuarray_get_elsize(a->typecode);

for (i = nd-1; i > 0; i--) {

newstrides[i-1] = newstrides[i] * tmpdims[i];

}

}

}

free(a->dimensions);

free(a->strides);

a->nd = nd;

a->dimensions = tmpdims;

a->strides = newstrides;

fix_flags:

GpuArray_fix_flags(a);

return GA_NO_ERROR;

}

int GpuArray_transpose(GpuArray *res, const GpuArray *a,

const unsigned int *new_axes) {

int err;

err = GpuArray_view(res, a);

if (err != GA_NO_ERROR) return err;

err = GpuArray_transpose_inplace(res, new_axes);

if (err != GA_NO_ERROR) GpuArray_clear(res);

return err;

}

int GpuArray_transpose_inplace(GpuArray *a, const unsigned int *new_axes) {

size_t *newdims;

ssize_t *newstrs;

unsigned int i;

unsigned int j;

unsigned int k;

newdims = calloc(a->nd, sizeof(size_t));

newstrs = calloc(a->nd, sizeof(ssize_t));

if (newdims == NULL || newstrs == NULL) {

free(newdims);

free(newstrs);

return GA_MEMORY_ERROR;

}

for (i = 0; i < a->nd; i++) {

if (new_axes == NULL) {

j = a->nd - i - 1;

} else {

j = new_axes[i];

// Repeated axes will lead to a broken output

for (k = 0; k < i; k++)

if (j == new_axes[k]) {

free(newdims);

free(newstrs);

return GA_VALUE_ERROR;

}

}

newdims[i] = a->dimensions[j];

newstrs[i] = a->strides[j];

}

free(a->dimensions);

free(a->strides);

a->dimensions = newdims;

a->strides = newstrs;

GpuArray_fix_flags(a);

return GA_NO_ERROR;

}

void GpuArray_clear(GpuArray *a) {

if (a->data)

gpudata_release(a->data);

free(a->dimensions);

free(a->strides);

memset(a, 0, sizeof(*a));

}

int GpuArray_share(const GpuArray *a, const GpuArray *b) {

if (a->data != b->data) return 0;

/* XXX: redefine buffer_share to mean: is it possible to share?

and use offset to make sure */

return gpudata_share(a->data, b->data, NULL);

}

gpucontext *GpuArray_context(const GpuArray *a) {

return gpudata_context(a->data);

}

int GpuArray_move(GpuArray *dst, const GpuArray *src) {

size_t sz;

unsigned int i;

if (!GpuArray_ISWRITEABLE(dst))

return GA_VALUE_ERROR;

if (!GpuArray_ISALIGNED(src) || !GpuArray_ISALIGNED(dst))

return GA_UNALIGNED_ERROR;

if (src->nd != dst->nd)

return GA_VALUE_ERROR;

for (i = 0; i < src->nd; i++) {

if (src->dimensions[i] != dst->dimensions[i])

return GA_VALUE_ERROR;

}

if (!GpuArray_ISONESEGMENT(dst) || !GpuArray_ISONESEGMENT(src) ||

GpuArray_ISFORTRAN(dst) != GpuArray_ISFORTRAN(src) ||

dst->typecode != src->typecode) {

return ga_extcopy(dst, src);

}

sz = gpuarray_get_elsize(dst->typecode);

for (i = 0; i < dst->nd; i++) sz *= dst->dimensions[i];

return gpudata_move(dst->data, dst->offset, src->data, src->offset, sz);

}

int GpuArray_write(GpuArray *dst, const void *src, size_t src_sz) {

if (!GpuArray_ISWRITEABLE(dst))

return GA_VALUE_ERROR;

if (!GpuArray_ISONESEGMENT(dst))

return GA_UNSUPPORTED_ERROR;

return gpudata_write(dst->data, dst->offset, src, src_sz);

}

int GpuArray_read(void *dst, size_t dst_sz, const GpuArray *src) {

if (!GpuArray_ISONESEGMENT(src))

return GA_UNSUPPORTED_ERROR;

return gpudata_read(dst, src->data, src->offset, dst_sz);

}

int GpuArray_memset(GpuArray *a, int data) {

if (!GpuArray_ISONESEGMENT(a))

return GA_UNSUPPORTED_ERROR;

return gpudata_memset(a->data, a->offset, data);

}

int GpuArray_copy(GpuArray *res, const GpuArray *a, ga_order order) {

int err;

err = GpuArray_empty(res, GpuArray_context(a), a->typecode,

a->nd, a->dimensions, order);

if (err != GA_NO_ERROR) return err;

err = GpuArray_move(res, a);

if (err != GA_NO_ERROR)

GpuArray_clear(res);

return err;

}

int GpuArray_transfer(GpuArray *res, const GpuArray *a) {

size_t sz;

unsigned int i;

if (!GpuArray_ISONESEGMENT(res))

return GA_UNSUPPORTED_ERROR;

if (!GpuArray_ISONESEGMENT(a))

return GA_UNSUPPORTED_ERROR;

if (res->typecode != a->typecode)

return GA_UNSUPPORTED_ERROR;

sz = gpuarray_get_elsize(a->typecode);

for (i = 0; i < a->nd; i++) sz *= a->dimensions[i];

return gpudata_transfer(res->data, res->offset, a->data, a->offset, sz);

}

int GpuArray_split(GpuArray **rs, const GpuArray *a, size_t n, size_t *p,

unsigned int axis) {

size_t i;

ssize_t *starts, *stops, *steps;

int err;

starts = calloc(a->nd, sizeof(ssize_t));

stops = calloc(a->nd, sizeof(ssize_t));

steps = calloc(a->nd, sizeof(ssize_t));

if (starts == NULL || stops == NULL || steps == NULL) {

free(starts);

free(stops);

free(steps);

return GA_MEMORY_ERROR;

}

for (i = 0; i < a->nd; i++) {

starts[i] = 0;

stops[i] = a->dimensions[i];

steps[i] = 1;

}

for (i = 0; i <= n; i++) {

if (i > 0)

starts[axis] = p[i-1];

else

starts[axis] = 0;

if (i < n)

stops[axis] = p[i];

else

stops[axis] = a->dimensions[axis];

err = GpuArray_index(rs[i], a, starts, stops, steps);

if (err != GA_NO_ERROR)

break;

}

free(starts);

free(stops);

free(steps);

if (err != GA_NO_ERROR) {

size_t ii;

for (ii = 0; ii < i; ii++)

GpuArray_clear(rs[ii]);

}

return err;

}

int GpuArray_concatenate(GpuArray *r, const GpuArray **as, size_t n,

unsigned int axis, int restype) {

size_t *dims, *res_dims;

size_t i, res_off;

unsigned int p;

int res_flags;

int err = GA_NO_ERROR;

if (axis >= as[0]->nd)

return GA_VALUE_ERROR;

dims = calloc(as[0]->nd, sizeof(size_t));

if (dims == NULL)

return GA_MEMORY_ERROR;

for (p = 0; p < as[0]->nd; p++) {

dims[p] = as[0]->dimensions[p];

}

if (!GpuArray_ISALIGNED(as[0])) {

err = GA_UNALIGNED_ERROR;

goto afterloop;

}

for (i = 1; i < n; i++) {

if (!GpuArray_ISALIGNED(as[i])) {

err = GA_UNALIGNED_ERROR;

goto afterloop;

}

if (as[i]->nd != as[0]->nd) {

err = GA_VALUE_ERROR;

goto afterloop;

}

for (p = 0; p < as[0]->nd; p++) {

if (p != axis && dims[p] != as[i]->dimensions[p]) {

err = GA_VALUE_ERROR;

goto afterloop;

} else if (p == axis) {

dims[p] += as[i]->dimensions[p];

}

}

}

afterloop:

if (err != GA_NO_ERROR) {

free(dims);

return err;

}

err = GpuArray_empty(r, GpuArray_context(as[0]), restype,

as[0]->nd, dims, GA_ANY_ORDER);

free(dims);