const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };

uint8_t highbits = static_cast<uint8_t>(src) >> 4;

return lookup[highbits];

std::string result;

for (size_t pos = 0; ; pos += search.length()) {

auto new_pos = s.find(search, pos);

if (new_pos == std::string::npos) {

result += s.substr(pos, s.size() - pos);

break;

}

result += s.substr(pos, new_pos - pos) + replace;

pos = new_pos;

}

s = std::move(result);

// Check for non-negative tolerance

if (abs_tol < 0.0) {

throw std::invalid_argument("Tolerance must be non-negative");

}

// Exact equality check

if (a == b) {

return true;

}

// Check for infinities

if (std::isinf(a) || std::isinf(b)) {

return false;

}

// Regular comparison using the provided absolute tolerance

return std::fabs(b - a) <= abs_tol;

char zero = 0;

for (size_t i = 0; i < n; ++i) {

file.write(&zero, 1);

}

va_list ap;

va_list ap2;

va_start(ap, fmt);

va_copy(ap2, ap);

int size = vsnprintf(NULL, 0, fmt, ap);

GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT

std::vector<char> buf(size + 1);

int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);

GGML_ASSERT(size2 == size);

va_end(ap2);

va_end(ap);

return std::string(buf.data(), size);

LLM_ARCH_LLAMA,

LLM_ARCH_FALCON,

LLM_ARCH_BAICHUAN,

LLM_ARCH_GPT2,

LLM_ARCH_GPTJ,

LLM_ARCH_GPTNEOX,

LLM_ARCH_MPT,

LLM_ARCH_STARCODER,

LLM_ARCH_PERSIMMON,

LLM_ARCH_REFACT,

LLM_ARCH_BLOOM,

LLM_ARCH_STABLELM,

LLM_ARCH_QWEN,

LLM_ARCH_PHI2,

LLM_ARCH_PLAMO,

LLM_ARCH_UNKNOWN,

LLM_KV_GENERAL_ARCHITECTURE,

LLM_KV_GENERAL_QUANTIZATION_VERSION,

LLM_KV_GENERAL_ALIGNMENT,

LLM_KV_GENERAL_NAME,

LLM_KV_GENERAL_AUTHOR,

LLM_KV_GENERAL_URL,

LLM_KV_GENERAL_DESCRIPTION,

LLM_KV_GENERAL_LICENSE,

LLM_KV_GENERAL_SOURCE_URL,

LLM_KV_GENERAL_SOURCE_HF_REPO,

LLM_KV_CONTEXT_LENGTH,

LLM_KV_EMBEDDING_LENGTH,

LLM_KV_BLOCK_COUNT,

LLM_KV_FEED_FORWARD_LENGTH,

LLM_KV_USE_PARALLEL_RESIDUAL,

LLM_KV_TENSOR_DATA_LAYOUT,

LLM_KV_EXPERT_COUNT,

LLM_KV_EXPERT_USED_COUNT,

LLM_KV_ATTENTION_HEAD_COUNT,

LLM_KV_ATTENTION_HEAD_COUNT_KV,

LLM_KV_ATTENTION_MAX_ALIBI_BIAS,

LLM_KV_ATTENTION_CLAMP_KQV,

LLM_KV_ATTENTION_LAYERNORM_EPS,

LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,

LLM_KV_ROPE_DIMENSION_COUNT,

LLM_KV_ROPE_FREQ_BASE,

LLM_KV_ROPE_SCALE_LINEAR,

LLM_KV_ROPE_SCALING_TYPE,

LLM_KV_ROPE_SCALING_FACTOR,

LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,

LLM_KV_ROPE_SCALING_FINETUNED,

LLM_KV_TOKENIZER_MODEL,

LLM_KV_TOKENIZER_LIST,

LLM_KV_TOKENIZER_TOKEN_TYPE,

LLM_KV_TOKENIZER_SCORES,

LLM_KV_TOKENIZER_MERGES,

LLM_KV_TOKENIZER_BOS_ID,

LLM_KV_TOKENIZER_EOS_ID,

LLM_KV_TOKENIZER_UNK_ID,

LLM_KV_TOKENIZER_SEP_ID,

LLM_KV_TOKENIZER_PAD_ID,

LLM_KV_TOKENIZER_ADD_BOS,

LLM_KV_TOKENIZER_ADD_EOS,

LLM_KV_TOKENIZER_HF_JSON,

LLM_KV_TOKENIZER_RWKV,

LLM_KV(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_kv kv) const {

return ::format(LLM_KV_NAMES[kv].c_str(), LLM_ARCH_NAMES[arch].c_str());

}

LLM_TENSOR_TOKEN_EMBD,

LLM_TENSOR_TOKEN_EMBD_NORM,

LLM_TENSOR_POS_EMBD,

LLM_TENSOR_OUTPUT,

LLM_TENSOR_OUTPUT_NORM,

LLM_TENSOR_ROPE_FREQS,

LLM_TENSOR_ATTN_Q,

LLM_TENSOR_ATTN_K,

LLM_TENSOR_ATTN_V,

LLM_TENSOR_ATTN_QKV,

LLM_TENSOR_ATTN_OUT,

LLM_TENSOR_ATTN_NORM,

LLM_TENSOR_ATTN_NORM_2,

LLM_TENSOR_ATTN_ROT_EMBD,

LLM_TENSOR_FFN_GATE_INP,

LLM_TENSOR_FFN_NORM,

LLM_TENSOR_FFN_GATE,

LLM_TENSOR_FFN_DOWN,

LLM_TENSOR_FFN_UP,

LLM_TENSOR_FFN_DOWN_EXP,

LLM_TENSOR_FFN_GATE_EXP,

LLM_TENSOR_FFN_UP_EXP,

LLM_TENSOR_ATTN_Q_NORM,

LLM_TENSOR_ATTN_K_NORM,

for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT

if (kv.second == name) {

return kv.first;

}

}

return LLM_ARCH_UNKNOWN;

LLM_TN(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_tensor tensor) const {

return LLM_TENSOR_NAMES[arch].at(tensor);

}

std::string operator()(llm_tensor tensor, const std::string & suffix) const {

return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix;

}

std::string operator()(llm_tensor tensor, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid);

}

std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;

}

std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix;

}

for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {

if (kv.second == name) {

return kv.first;

}

}

return LLAMA_ROPE_SCALING_UNSPECIFIED;

switch (type) {

case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);

case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);

case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);

case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);

case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);

case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);

case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);

case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);

case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);

case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);

case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";

default: return format("unknown type %d", type);

}

const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);

switch (type) {

case GGUF_TYPE_STRING:

return gguf_get_val_str(ctx_gguf, i);

case GGUF_TYPE_ARRAY:

{

const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);

int arr_n = gguf_get_arr_n(ctx_gguf, i);

const void * data = gguf_get_arr_data(ctx_gguf, i);

std::stringstream ss;

ss << "[";

for (int j = 0; j < arr_n; j++) {

if (arr_type == GGUF_TYPE_STRING) {

std::string val = gguf_get_arr_str(ctx_gguf, i, j);

// escape quotes

replace_all(val, "\\", "\\\\");

replace_all(val, "\"", "\\\"");

ss << '"' << val << '"';

} else if (arr_type == GGUF_TYPE_ARRAY) {

ss << "???";

} else {

ss << gguf_data_to_str(arr_type, data, j);

}

if (j < arr_n - 1) {

ss << ", ";

}

}

ss << "]";

return ss.str();

}

default:

return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);

}

struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);

if (plan.work_size > 0) {

buf.resize(plan.work_size);

plan.work_data = buf.data();

}

ggml_graph_compute(graph, &plan);

LPSTR buf;

size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,

NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);

if (!size) {

return "FormatMessageA failed";

}

std::string ret(buf, size);

LocalFree(buf);

return ret;

// use FILE * so we don't have to re-open the file to mmap

FILE * fp;

size_t size;

llama_file(const char * fname, const char * mode) {

fp = std::fopen(fname, mode);

if (fp == NULL) {

throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));

}

seek(0, SEEK_END);

size = tell();

seek(0, SEEK_SET);

}

size_t tell() const {

#ifdef _WIN32

__int64 ret = _ftelli64(fp);

#else

long ret = std::ftell(fp);

#endif

GGML_ASSERT(ret != -1); // this really shouldn't fail

return (size_t) ret;

}

void seek(size_t offset, int whence) const {

#ifdef _WIN32

int ret = _fseeki64(fp, (__int64) offset, whence);

#else

int ret = std::fseek(fp, (long) offset, whence);

#endif

GGML_ASSERT(ret == 0); // same

}

void read_raw(void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

std::size_t ret = std::fread(ptr, len, 1, fp);

if (ferror(fp)) {

throw std::runtime_error(format("read error: %s", strerror(errno)));

}

if (ret != 1) {

throw std::runtime_error("unexpectedly reached end of file");

}

}

uint32_t read_u32() const {

uint32_t ret;

read_raw(&ret, sizeof(ret));

return ret;

}

void write_raw(const void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

size_t ret = std::fwrite(ptr, len, 1, fp);

if (ret != 1) {

throw std::runtime_error(format("write error: %s", strerror(errno)));

}

}

void write_u32(std::uint32_t val) const {

write_raw(&val, sizeof(val));

}

~llama_file() {

if (fp) {

std::fclose(fp);

}

}

void * addr;

size_t size;

llama_mmap(const llama_mmap &) = delete;

#ifdef _POSIX_MAPPED_FILES

static constexpr bool SUPPORTED = true;

// list of mapped fragments (first_offset, last_offset)

std::vector<std::pair<size_t, size_t>> mapped_fragments;

llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {

size = file->size;

int fd = fileno(file->fp);

int flags = MAP_SHARED;

// prefetch/readahead impairs performance on NUMA systems

if (numa) { prefetch = 0; }

#ifdef __linux__

// advise the kernel to read the file sequentially (increases readahead)

if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {

LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",

strerror(errno));

}

if (prefetch) { flags |= MAP_POPULATE; }

#endif

addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);

if (addr == MAP_FAILED) { // NOLINT

throw std::runtime_error(format("mmap failed: %s", strerror(errno)));

}

if (prefetch > 0) {

// advise the kernel to preload the mapped memory

if (posix_madvise(addr, std::min(file->size, prefetch), POSIX_MADV_WILLNEED)) {

LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",

strerror(errno));

}

}

if (numa) {

// advise the kernel not to use readahead

// (because the next page might not belong on the same node)

if (posix_madvise(addr, file->size, POSIX_MADV_RANDOM)) {

LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",

strerror(errno));

}

}

// initialize list of mapped_fragments

mapped_fragments.emplace_back(0, file->size);

}

static void align_range(size_t * first, size_t * last, size_t page_size) {

// align first to the next page

size_t offset_in_page = *first & (page_size - 1);

size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;

*first += offset_to_page;

// align last to the previous page

*last = *last & ~(page_size - 1);

if (*last <= *first) {

*last = *first;

}

}

// partially unmap the file in the range [first, last)

void unmap_fragment(size_t first, size_t last) {

// note: this function must not be called multiple times with overlapping ranges

// otherwise, there is a risk of invalidating addresses that have been repurposed for other mappings

int page_size = sysconf(_SC_PAGESIZE);

align_range(&first, &last, page_size);

size_t len = last - first;

if (len == 0) {

return;

}

GGML_ASSERT(first % page_size == 0);

GGML_ASSERT(last % page_size == 0);

GGML_ASSERT(last > first);

void * next_page_start = (uint8_t *) addr + first;

// unmap the range

if (munmap(next_page_start, len)) {

LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));

}

// update the list of mapped fragments to avoid unmapping the same range again in the destructor

std::vector<std::pair<size_t, size_t>> new_mapped_fragments;

for (const auto & frag : mapped_fragments) {

if (frag.first < first && frag.second > last) {

// the range is in the middle of the fragment, split it

new_mapped_fragments.emplace_back(frag.first, first);

new_mapped_fragments.emplace_back(last, frag.second);

} else if (frag.first < first && frag.second > first) {

// the range starts in the middle of the fragment

new_mapped_fragments.emplace_back(frag.first, first);

} else if (frag.first < last && frag.second > last) {

// the range ends in the middle of the fragment

new_mapped_fragments.emplace_back(last, frag.second);

} else if (frag.first >= first && frag.second <= last) {

// the range covers the entire fragment

} else {

// the range is outside the fragment

new_mapped_fragments.push_back(frag);

}

}

mapped_fragments = std::move(new_mapped_fragments);

}

~llama_mmap() {

for (const auto & frag : mapped_fragments) {

if (munmap((char *) addr + frag.first, frag.second - frag.first)) {

LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));

}

}

}

#elif defined(_WIN32)

static constexpr bool SUPPORTED = true;

llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false) {

GGML_UNUSED(numa);

size = file->size;

HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));

HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

if (hMapping == NULL) {

DWORD error = GetLastError();

throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));

}

addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);

DWORD error = GetLastError();

CloseHandle(hMapping);

if (addr == NULL) {

throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));

}

if (prefetch > 0) {

// PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it

BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);

HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");

// may fail on pre-Windows 8 systems

pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));

if (pPrefetchVirtualMemory) {

// advise the kernel to preload the mapped memory

WIN32_MEMORY_RANGE_ENTRY range;

range.VirtualAddress = addr;

range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);

if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {

LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

}

}

void unmap_fragment(size_t first, size_t last) {

// not supported

GGML_UNUSED(first);

GGML_UNUSED(last);