size_t nels = ggml_nelements(tensor);

std::vector<float> data(nels);

{

// parallel initialization

static const size_t n_threads = N_THREADS;

// static RNG initialization (revisit if n_threads stops being constant)

static std::vector<std::default_random_engine> generators = []() {

std::random_device rd;

std::vector<std::default_random_engine> vec;

vec.reserve(n_threads);

//for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(1234 + i); } // fixed seed

for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(rd()); }

return vec;

}();

auto init_thread = [&](size_t ith, size_t start, size_t end) {

std::uniform_real_distribution<float> distribution(min, max);

auto & gen = generators[ith];

for (size_t i = start; i < end; i++) {

data[i] = distribution(gen);

}

};

if (n_threads == 1) {

init_thread(0, 0, nels);

} else {

std::vector<std::future<void>> tasks;

tasks.reserve(n_threads);

for (size_t i = 0; i < n_threads; i++) {

size_t start = i*nels/n_threads;

size_t end = (i+1)*nels/n_threads;

tasks.push_back(std::async(std::launch::async, init_thread, i, start, end));

}

for (auto & t : tasks) {

t.get();

}

}

}

if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) {

ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float));

} else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {

GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0);

// dummy importance matrix

std::vector<float> imatrix(tensor->ne[0], 1.0f);

const float * im = imatrix.data();

if (!ggml_quantize_requires_imatrix(tensor->type)) {

// when the imatrix is optional, we want to test both quantization with and without imatrix

// use one of the random numbers to decide

if (data[0] > 0.5f*(min + max)) {

im = nullptr;

}

}

std::vector<uint8_t> dataq(ggml_row_size(tensor->type, nels));

{

// parallel quantization by block

size_t blck_size = ggml_blck_size(tensor->type);

size_t n_blocks = nels / blck_size;

auto quantize_thread = [&](size_t start, size_t end) {

ggml_quantize_chunk(tensor->type, data.data(), dataq.data(),

start * blck_size, end - start, blck_size, im);

};

const size_t min_blocks_per_thread = 1;

const size_t n_quant_threads = std::min<size_t>(std::max<size_t>(N_THREADS/2, 1),

std::max<size_t>(1, n_blocks / min_blocks_per_thread));

if (n_quant_threads == 1) {

// single-threaded quantization: do all blocks in the current thread

quantize_thread(0, n_blocks);

} else {

std::vector<std::future<void>> tasks;

tasks.reserve(n_quant_threads);

for (size_t i = 0; i < n_quant_threads; i++) {

size_t start = i*n_blocks/n_quant_threads;

size_t end = (i+1)*n_blocks/n_quant_threads;

tasks.push_back(std::async(std::launch::async, quantize_thread, start, end));

}

for (auto & t : tasks) {

t.get();

}

}

}

ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size());

} else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) {

// This is going to create some weird integers though.

ggml_backend_tensor_set(tensor, data.data(), 0, ggml_nbytes(tensor));

} else if (tensor->type == GGML_TYPE_I64) {

// Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful.

const size_t nbytes_half = ggml_nbytes(tensor)/2;

ggml_backend_tensor_set(tensor, data.data(), 0*nbytes_half, nbytes_half);

ggml_backend_tensor_set(tensor, data.data(), 1*nbytes_half, nbytes_half);

} else {

GGML_ABORT("fatal error");

}

GGML_ASSERT(tensor->type == GGML_TYPE_F16);

GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne);

std::vector<float> data_f32(ne0*ne1*ne2*ne3);

std::vector<ggml_fp16_t> data_f16(ne0*ne1*ne2*ne3);

std::random_device rd;

std::mt19937 gen(rd());

std::uniform_real_distribution<float> dis(min, max);

for (size_t i = 0; i < data_f32.size(); i++) {

data_f32[i] = dis(gen);

}

// block size

const int blck0 = 128;

const int blck1 = 64;

// number of INF/zero blocks

const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1);

for (int b = 0; b < n_inf_zero_blocks; b++) {

const int p3 = (rd() % ne3);

const int p2 = (rd() % ne2);

const int p1 = (rd() % ne1);

const int p0 = (rd() % ne0);

bool inf = rd() & 1;

for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) {

const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0;

for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) {

data_f32[idx + i0] = inf ? -INFINITY : 0.0f;

}

}

}

ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3);

ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));

GGML_ASSERT(tensor->type == GGML_TYPE_F32);

GGML_ASSERT(tensor->ne[0] == tensor->ne[1]);

GGML_TENSOR_LOCALS(int32_t, ne, tensor, ne);

GGML_TENSOR_LOCALS(size_t, nb, tensor, nb);

std::vector<float> data_f32(ne0*ne1*ne2*ne3);

std::random_device rd;

std::mt19937 gen(rd());

std::uniform_real_distribution<float> dis(min, max);

for (int64_t i3 = 0; i3 < ne3; i3++) {

for (int64_t i2 = 0; i2 < ne2; i2++) {

for (int64_t i1 = 0; i1 < ne1; i1++) {

for (int64_t i0 = 0; i0 < ne0; i0++) {

int64_t idx = (i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3) / sizeof(float);

if (i0 <= i1) {

data_f32[idx] = dis(gen);

} else {

data_f32[idx] = 0.0f;

}

}

}

}

}

ggml_backend_tensor_set(tensor, data_f32.data(), 0, ggml_nbytes(tensor));

std::vector<float> tv;

tv.reserve(ggml_nelements(t));

std::vector<uint8_t> buf(ggml_nbytes(t));

ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t));

const auto * tt = ggml_get_type_traits(t->type);

size_t bs = ggml_blck_size(t->type);

std::vector<float> vq(ggml_blck_size(t->type));

bool quantized = ggml_is_quantized(t->type);

// access elements by index to avoid gaps in views

for (int64_t i3 = 0; i3 < t->ne[3]; i3++) {

for (int64_t i2 = 0; i2 < t->ne[2]; i2++) {

for (int64_t i1 = 0; i1 < t->ne[1]; i1++) {

for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) {

size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0];

if (t->type == GGML_TYPE_F16) {

tv.push_back(ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i]));

} else if (t->type == GGML_TYPE_BF16) {

tv.push_back(ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i]));

} else if (t->type == GGML_TYPE_F32) {

tv.push_back(*(float *) &buf[i]);

} else if (t->type == GGML_TYPE_I64) {

tv.push_back((float)*(int64_t *) &buf[i]);

} else if (t->type == GGML_TYPE_I32) {

tv.push_back((float)*(int32_t *) &buf[i]);

} else if (t->type == GGML_TYPE_I16) {

tv.push_back((float)*(int16_t *) &buf[i]);

} else if (t->type == GGML_TYPE_I8) {

tv.push_back((float)*(int8_t *) &buf[i]);

} else if (quantized) {

tt->to_float(&buf[i], vq.data(), bs);

tv.insert(tv.end(), vq.begin(), vq.end());

} else {

GGML_ABORT("fatal error");

}

}

}

}

}

return tv;

double mse_a_b = 0.0;

double mse_a_0 = 0.0;

for (size_t i = 0; i < n; i++) {

float a_i = a[i];

float b_i = b[i];

mse_a_b += (a_i - b_i) * (a_i - b_i);

mse_a_0 += a_i * a_i;

}

return mse_a_b / mse_a_0;

std::unordered_map<T, size_t> set_a;

std::unordered_map<T, size_t> set_b;

for (size_t i = 0; i < n; ++i) {

set_a[a[i]]++;

set_b[b[i]]++;

}

size_t diff = 0;

for (const auto & p : set_a) {

const int64_t na = p.second;

const int64_t nb = set_b.find(p.first) != set_b.end() ? set_b.at(p.first) : 0;

diff += std::abs(na - nb);

}

for (const auto & p : set_b) {

if (set_a.find(p.first) == set_a.end()) {

diff += p.second;

}

}

return (double) diff / (2*n);

double sum = 0.0f;

size_t nvalid = 0;

for (size_t i = 0; i < n; i++) {

if (!expected_vals.empty()) {

bool matches_any = false;

for (const float & ev : expected_vals) {

if (fabsf(a[i] - ev) < 1e-3f) {

matches_any = true;

break;

}

}

if (!matches_any) {

continue;

}

}

const float asymm = (a[i] - b[i]) / (a[i] + b[i]);

sum += fabsf(asymm);

nvalid++;

}

return sum/nvalid;

std::string s = "[";

for (size_t i = 0; i < N; i++) {

if (i > 0) {

s += ",";

}

s += var_to_str(x[i]);

}

s += "]";

return s;

std::string s = "[";

for (size_t i = 0; i < N; i++) {

if (i > 0) {

s += ",";

}

s += var_to_str(x[i]);

}

s += "]";

return s;

switch (pool) {

case GGML_OP_POOL_AVG: return "avg";

case GGML_OP_POOL_MAX: return "max";

default: return std::to_string(pool);

}

std::string str;

switch (mode & 0xFF) {

case GGML_SCALE_MODE_NEAREST: str = "nearest"; break;

case GGML_SCALE_MODE_BILINEAR: str = "bilinear"; break;

case GGML_SCALE_MODE_BICUBIC: str = "bicubic"; break;

default: str = std::to_string(mode); break;

}

if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) {

str += "|align_corners";

}

if (mode & GGML_SCALE_FLAG_ANTIALIAS) {

str += "|antialias";

}

return str;

ggml_backend_dev_t dev = ggml_backend_get_device(backend);

ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);

auto get_features = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");

if (!get_features) {

return false;

}

const ggml_backend_feature * features = get_features(reg);

if (!features) {

return false;

}

for (const ggml_backend_feature * f = features; f->name; ++f) {

if (strcmp(f->name, feature_name) == 0 && strcmp(f->value, "1") == 0) {

return true;

}

}

return false;

MODE_TEST,

MODE_PERF,

MODE_GRAD,

MODE_SUPPORT,

if (s == "console") {

format = CONSOLE;

} else if (s == "sql") {

format = SQL;

} else if (s == "csv") {

format = CSV;

} else {

return false;

}

return true;

std::string test_time;

std::string build_commit;

std::string backend_name;

std::string op_name;

std::string op_params;

std::string test_mode;

bool supported;

bool passed;

std::string error_message;

double time_us;

double flops;

double bandwidth_gb_s;

size_t memory_kb;

int n_runs;

std::string device_description;

std::string backend_reg_name;

test_result() {

// Initialize with default values

time_us = 0.0;

flops = 0.0;

bandwidth_gb_s = 0.0;

memory_kb = 0;

n_runs = 0;

supported = false;

passed = false;

// Set test time

time_t t = time(NULL);

char buf[32];

std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));

test_time = buf;

// Set build info

build_commit = ggml_commit();

}

test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params,

const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "",

double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0,

int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") :

backend_name(backend_name),

op_name(op_name),

op_params(op_params),

test_mode(test_mode),

supported(supported),

passed(passed),

error_message(error_message),

time_us(time_us),

flops(flops),

bandwidth_gb_s(bandwidth_gb_s),

memory_kb(memory_kb),

n_runs(n_runs),

device_description(device_description),

backend_reg_name(backend_reg_name) {

// Set test time

time_t t = time(NULL);

char buf[32];

std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));

test_time = buf;

// Set build info

build_commit = ggml_commit();

}

static const std::vector<std::string> & get_fields() {

static const std::vector<std::string> fields = {

"test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported",

"passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs",

"device_description", "backend_reg_name"

};

return fields;

}

enum field_type { STRING, BOOL, INT, FLOAT };

static field_type get_field_type(const std::string & field) {

if (field == "supported" || field == "passed") {

return BOOL;

}

if (field == "memory_kb" || field == "n_runs") {

return INT;

}

if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") {

return FLOAT;

}

return STRING;

}

std::vector<std::string> get_values() const {

return { test_time,

build_commit,

backend_name,

op_name,

op_params,

test_mode,

std::to_string(supported),

std::to_string(passed),

error_message,

std::to_string(time_us),

std::to_string(flops),

std::to_string(bandwidth_gb_s),

std::to_string(memory_kb),

std::to_string(n_runs),

device_description,

backend_reg_name };

}

std::string op_name;

std::string op_params;

std::string backend_name;

test_status_t status = test_status_t::OK;

std::string failure_reason;

// Additional information fields that were previously in separate structs

std::string error_component;

std::string error_details;

// Gradient info

int64_t gradient_index = -1;

std::string gradient_param_name;

float gradient_value = 0.0f;

// MAA error info

double maa_error = 0.0;

double maa_threshold = 0.0;

// Flags for different types of information

bool has_error = false;

bool has_gradient_info = false;

bool has_maa_error = false;

bool is_compare_failure = false;

bool is_large_tensor_skip = false;

test_operation_info() = default;

test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name,

test_status_t status = test_status_t::OK, const std::string & failure_reason = "") :

op_name(op_name),

op_params(op_params),

backend_name(backend_name),

status(status),

failure_reason(failure_reason) {}

// Set error information

void set_error(const std::string & component, const std::string & details) {

has_error = true;

error_component = component;

error_details = details;

if (status == test_status_t::OK) {

status = test_status_t::FAIL;

}

}

// Set gradient information

void set_gradient_info(int64_t index, const std::string & param_name, float value) {

has_gradient_info = true;

gradient_index = index;

gradient_param_name = param_name;

gradient_value = value;

if (status == test_status_t::OK) {

status = test_status_t::FAIL;

}

}

// Set MAA error information

void set_maa_error(double error, double threshold) {

has_maa_error = true;

maa_error = error;

maa_threshold = threshold;

if (status == test_status_t::OK) {

status = test_status_t::FAIL;

}

}

// Set compare failure

void set_compare_failure() {

is_compare_failure = true;

if (status == test_status_t::OK) {

status = test_status_t::FAIL;

}

}

// Set large tensor skip

void set_large_tensor_skip() { is_large_tensor_skip = true; }

size_t tests_passed;

size_t tests_total;

bool is_backend_summary = false; // true for backend summary, false for test summary

test_summary_info() = default;

test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) :

tests_passed(tests_passed),

tests_total(tests_total),

is_backend_summary(is_backend_summary) {}

size_t device_count;

testing_start_info() = default;

testing_start_info(size_t device_count) : device_count(device_count) {}

size_t device_index;

size_t total_devices;

std::string device_name;

bool skipped = false;

std::string skip_reason;

std::string description;

size_t memory_total_mb = 0;

size_t memory_free_mb = 0;

bool has_memory_info = false;

backend_init_info() = default;

backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false,

const std::string & skip_reason = "", const std::string & description = "",

size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) :

device_index(device_index),

total_devices(total_devices),

device_name(device_name),

skipped(skipped),

skip_reason(skip_reason),

description(description),

memory_total_mb(memory_total_mb),

memory_free_mb(memory_free_mb),

has_memory_info(has_memory_info) {}

std::string backend_name;

test_status_t status;

backend_status_info() = default;

backend_status_info(const std::string & backend_name, test_status_t status) :

backend_name(backend_name),

status(status) {}

size_t backends_passed;

size_t backends_total;

bool all_passed;

overall_summary_info() = default;

overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) :

backends_passed(backends_passed),

backends_total(backends_total),

all_passed(all_passed) {}

virtual ~printer() {}

FILE * fout = stdout;

virtual void print_header() {}

virtual void print_test_result(const test_result & result) = 0;

virtual void print_footer() {}

virtual void print_operation(const test_operation_info & info) { (void) info; }

virtual void print_summary(const test_summary_info & info) { (void) info; }

virtual void print_testing_start(const testing_start_info & info) { (void) info; }

virtual void print_backend_init(const backend_init_info & info) { (void) info; }

virtual void print_backend_status(const backend_status_info & info) { (void) info; }

virtual void print_overall_summary(const overall_summary_info & info) { (void) info; }

virtual void print_failed_tests(const std::vector<std::string> & failed_tests) { (void) failed_tests; }

void print_test_result(const test_result & result) override {

if (result.test_mode == "test") {

print_test_console(result);

} else if (result.test_mode == "perf") {

print_perf_console(result);

} else if (result.test_mode == "support") {

print_support_console(result);

}

}

void print_operation(const test_operation_info & info) override {

printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str());

fflush(stdout);

// Handle large tensor skip first

if (info.is_large_tensor_skip) {

printf("skipping large tensors for speed \n");

return;

}

// Handle not supported status

if (info.status == test_status_t::NOT_SUPPORTED) {

if (!info.failure_reason.empty()) {

printf("not supported [%s]\n", info.failure_reason.c_str());

} else {

printf("not supported [%s]\n", info.backend_name.c_str());

}

return;

}

// Handle errors and additional information

if (info.has_error) {

if (info.error_component == "allocation") {

fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str());

} else if (info.error_component == "backend") {

fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str());

} else {

fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());

}

}

// Handle gradient info

if (info.has_gradient_info) {

printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,

info.gradient_param_name.c_str(), info.gradient_value);

}

// Handle MAA error

if (info.has_maa_error) {

printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);

}

// Handle compare failure

if (info.is_compare_failure) {

printf("compare failed ");

}

// Print final status

if (info.status == test_status_t::OK) {

printf("\033[1;32mOK\033[0m\n");

} else {

printf("\033[1;31mFAIL\033[0m\n");

}

}

void print_summary(const test_summary_info & info) override {

if (info.is_backend_summary) {

printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total);

} else {

printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total);

}

}

void print_backend_status(const backend_status_info & info) override {

printf(" Backend %s: ", info.backend_name.c_str());

if (info.status == test_status_t::OK) {

printf("\033[1;32mOK\033[0m\n");

} else {

printf("\033[1;31mFAIL\033[0m\n");

}

}

void print_testing_start(const testing_start_info & info) override {

printf("Testing %zu devices\n\n", info.device_count);

}

void print_backend_init(const backend_init_info & info) override {

printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());

if (info.skipped) {

printf(" %s\n", info.skip_reason.c_str());

return;

}

if (!info.description.empty()) {

printf(" Device description: %s\n", info.description.c_str());

}

if (info.has_memory_info) {

printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);

}

printf("\n");

}

void print_overall_summary(const overall_summary_info & info) override {

printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total);

if (info.all_passed) {

printf("\033[1;32mOK\033[0m\n");

} else {

printf("\033[1;31mFAIL\033[0m\n");

}

}

void print_failed_tests(const std::vector<std::string> & failed_tests) override {

if (failed_tests.empty()) {

return;

}

printf("\nFailing tests:\n");

for (const auto & test_name : failed_tests) {

printf(" %s\n", test_name.c_str());

}

}

private:

void print_test_console(const test_result & result) {

printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());

fflush(stdout);

if (!result.supported) {

printf("not supported [%s] ", result.backend_name.c_str());

printf("\n");

return;

}

if (result.passed) {

printf("\033[1;32mOK\033[0m\n");

} else {

printf("\033[1;31mFAIL\033[0m\n");

}

}

void print_perf_console(const test_result & result) {

int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());

fflush(stdout);

if (!result.supported) {

printf("not supported\n");

return;

}

// align while also leaving some margin for variations in parameters

int align = 8;

int last = (len + align - 1) / align * align;

if (last - len < 5) {

last += align;

}

printf("%*s", last - len, "");

printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);

if (result.flops > 0) {

auto format_flops = [](double flops) -> std::string {

char buf[256];

if (flops >= 1e12) {

snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12);

} else if (flops >= 1e9) {

snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9);

} else if (flops >= 1e6) {

snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6);

} else {

snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3);

}

return buf;

};

uint64_t op_flops_per_run = result.flops * result.time_us / 1e6;

printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(),

format_flops(result.flops).c_str());

} else {

printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s);

}

printf("\n");

}

void print_support_console(const test_result & result) {

printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());

fflush(stdout);