const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };

uint8_t highbits = static_cast<uint8_t>(src) >> 4;

return lookup[highbits];

std::string result;

for (size_t pos = 0; ; pos += search.length()) {

auto new_pos = s.find(search, pos);

if (new_pos == std::string::npos) {

result += s.substr(pos, s.size() - pos);

break;

}

result += s.substr(pos, new_pos - pos) + replace;

pos = new_pos;

}

s = std::move(result);

// Check for non-negative tolerance

if (abs_tol < 0.0) {

throw std::invalid_argument("Tolerance must be non-negative");

}

// Exact equality check

if (a == b) {

return true;

}

// Check for infinities

if (std::isinf(a) || std::isinf(b)) {

return false;

}

// Regular comparison using the provided absolute tolerance

return std::fabs(b - a) <= abs_tol;

char zero = 0;

for (size_t i = 0; i < n; ++i) {

file.write(&zero, 1);

}

va_list ap;

va_list ap2;

va_start(ap, fmt);

va_copy(ap2, ap);

int size = vsnprintf(NULL, 0, fmt, ap);

GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT

std::vector<char> buf(size + 1);

int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);

GGML_ASSERT(size2 == size);

va_end(ap2);

va_end(ap);

return std::string(buf.data(), size);

LLM_ARCH_LLAMA,

LLM_ARCH_FALCON,

LLM_ARCH_BAICHUAN,

LLM_ARCH_GPT2,

LLM_ARCH_GPTJ,

LLM_ARCH_GPTNEOX,

LLM_ARCH_MPT,

LLM_ARCH_STARCODER,

LLM_ARCH_PERSIMMON,

LLM_ARCH_REFACT,

LLM_ARCH_BLOOM,

LLM_ARCH_UNKNOWN,

LLM_KV_GENERAL_ARCHITECTURE,

LLM_KV_GENERAL_QUANTIZATION_VERSION,

LLM_KV_GENERAL_ALIGNMENT,

LLM_KV_GENERAL_NAME,

LLM_KV_GENERAL_AUTHOR,

LLM_KV_GENERAL_URL,

LLM_KV_GENERAL_DESCRIPTION,

LLM_KV_GENERAL_LICENSE,

LLM_KV_GENERAL_SOURCE_URL,

LLM_KV_GENERAL_SOURCE_HF_REPO,

LLM_KV_CONTEXT_LENGTH,

LLM_KV_EMBEDDING_LENGTH,

LLM_KV_BLOCK_COUNT,

LLM_KV_FEED_FORWARD_LENGTH,

LLM_KV_USE_PARALLEL_RESIDUAL,

LLM_KV_TENSOR_DATA_LAYOUT,

LLM_KV_ATTENTION_HEAD_COUNT,

LLM_KV_ATTENTION_HEAD_COUNT_KV,

LLM_KV_ATTENTION_MAX_ALIBI_BIAS,

LLM_KV_ATTENTION_CLAMP_KQV,

LLM_KV_ATTENTION_LAYERNORM_EPS,

LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,

LLM_KV_ROPE_DIMENSION_COUNT,

LLM_KV_ROPE_FREQ_BASE,

LLM_KV_ROPE_SCALE_LINEAR,

LLM_KV_TOKENIZER_MODEL,

LLM_KV_TOKENIZER_LIST,

LLM_KV_TOKENIZER_TOKEN_TYPE,

LLM_KV_TOKENIZER_SCORES,

LLM_KV_TOKENIZER_MERGES,

LLM_KV_TOKENIZER_BOS_ID,

LLM_KV_TOKENIZER_EOS_ID,

LLM_KV_TOKENIZER_UNK_ID,

LLM_KV_TOKENIZER_SEP_ID,

LLM_KV_TOKENIZER_PAD_ID,

LLM_KV_TOKENIZER_HF_JSON,

LLM_KV_TOKENIZER_RWKV,

LLM_KV(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_kv kv) const {

return ::format(LLM_KV_NAMES[kv].c_str(), LLM_ARCH_NAMES[arch].c_str());

}

LLM_TENSOR_TOKEN_EMBD,

LLM_TENSOR_TOKEN_EMBD_NORM,

LLM_TENSOR_POS_EMBD,

LLM_TENSOR_OUTPUT,

LLM_TENSOR_OUTPUT_NORM,

LLM_TENSOR_ROPE_FREQS,

LLM_TENSOR_ATTN_Q,

LLM_TENSOR_ATTN_K,

LLM_TENSOR_ATTN_V,

LLM_TENSOR_ATTN_QKV,

LLM_TENSOR_ATTN_OUT,

LLM_TENSOR_ATTN_NORM,

LLM_TENSOR_ATTN_NORM_2,

LLM_TENSOR_ATTN_ROT_EMBD,

LLM_TENSOR_FFN_GATE,

LLM_TENSOR_FFN_DOWN,

LLM_TENSOR_FFN_UP,

LLM_TENSOR_FFN_NORM,

LLM_TENSOR_ATTN_Q_NORM,

LLM_TENSOR_ATTN_K_NORM,

for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT

if (kv.second == name) {

return kv.first;

}

}

return LLM_ARCH_UNKNOWN;

LLM_TN(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_tensor tensor) const {

return LLM_TENSOR_NAMES[arch].at(tensor);

}

std::string operator()(llm_tensor tensor, const std::string & suffix) const {

return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix;

}

std::string operator()(llm_tensor tensor, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid);

}

std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;

}

struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);

if (plan.work_size > 0) {

buf.resize(plan.work_size);

plan.work_data = buf.data();

}

ggml_graph_compute(graph, &plan);

LPSTR buf;

size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,

NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);

if (!size) {

return "FormatMessageA failed";

}

std::string ret(buf, size);

LocalFree(buf);

return ret;

void * data = NULL;

size_t size = 0;

// fallback to malloc / free

// useful in cases where CUDA can try to allocate PINNED memory

bool fallback = false;

void resize(size_t n) {

llama_host_free(data);

data = llama_host_malloc(n);

if (!data) {

fallback = true;

data = malloc(n);

} else {

fallback = false;

}

GGML_ASSERT(data);

size = n;

}

~llama_buffer() {

if (data) {

if (fallback) { // NOLINT

free(data);

} else {

llama_host_free(data);

}

}

data = NULL;

}

// use FILE * so we don't have to re-open the file to mmap

FILE * fp;

size_t size;

llama_file(const char * fname, const char * mode) {

fp = std::fopen(fname, mode);

if (fp == NULL) {

throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));

}

seek(0, SEEK_END);

size = tell();

seek(0, SEEK_SET);

}

size_t tell() const {

#ifdef _WIN32

__int64 ret = _ftelli64(fp);

#else

long ret = std::ftell(fp);

#endif

GGML_ASSERT(ret != -1); // this really shouldn't fail

return (size_t) ret;

}

void seek(size_t offset, int whence) const {

#ifdef _WIN32

int ret = _fseeki64(fp, (__int64) offset, whence);

#else

int ret = std::fseek(fp, (long) offset, whence);

#endif

GGML_ASSERT(ret == 0); // same

}

void read_raw(void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

std::size_t ret = std::fread(ptr, len, 1, fp);

if (ferror(fp)) {

throw std::runtime_error(format("read error: %s", strerror(errno)));

}

if (ret != 1) {

throw std::runtime_error(std::string("unexpectedly reached end of file"));

}

}

uint32_t read_u32() const {

uint32_t ret;

read_raw(&ret, sizeof(ret));

return ret;

}

void write_raw(const void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

size_t ret = std::fwrite(ptr, len, 1, fp);

if (ret != 1) {

throw std::runtime_error(format("write error: %s", strerror(errno)));

}

}

void write_u32(std::uint32_t val) const {

write_raw(&val, sizeof(val));

}

~llama_file() {

if (fp) {

std::fclose(fp);

}

}

void * addr;

size_t size;

llama_mmap(const llama_mmap &) = delete;

#ifdef _POSIX_MAPPED_FILES

static constexpr bool SUPPORTED = true;

llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {

size = file->size;

int fd = fileno(file->fp);

int flags = MAP_SHARED;

// prefetch/readahead impairs performance on NUMA systems

if (numa) { prefetch = 0; }

#ifdef __linux__

if (prefetch) { flags |= MAP_POPULATE; }

#endif

addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);

if (addr == MAP_FAILED) {

throw std::runtime_error(format("mmap failed: %s", strerror(errno)));

}

if (prefetch > 0) {

// Advise the kernel to preload the mapped memory

if (posix_madvise(addr, std::min(file->size, prefetch), POSIX_MADV_WILLNEED)) {

fprintf(stderr, "warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",

strerror(errno));

}

}

if (numa) {

// advise the kernel not to use readahead

// (because the next page might not belong on the same node)

if (posix_madvise(addr, file->size, POSIX_MADV_RANDOM)) {

fprintf(stderr, "warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",

strerror(errno));

}

}

}

~llama_mmap() {

munmap(addr, size);

}

#elif defined(_WIN32)

static constexpr bool SUPPORTED = true;

llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {

(void) numa;

size = file->size;

HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));

HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

DWORD error = GetLastError();

if (hMapping == NULL) {

throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));

}

addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);

error = GetLastError();

CloseHandle(hMapping);

if (addr == NULL) {

throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));

}

if (prefetch) {

// PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it

BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);

HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");

// may fail on pre-Windows 8 systems

pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));

if (pPrefetchVirtualMemory) {

// advise the kernel to preload the mapped memory

WIN32_MEMORY_RANGE_ENTRY range;

range.VirtualAddress = addr;

range.NumberOfBytes = (SIZE_T)size;

if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {

fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

}

}

~llama_mmap() {

if (!UnmapViewOfFile(addr)) {

fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

#else

static constexpr bool SUPPORTED = false;

llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {

(void) file;

(void) prefetch;

(void) numa;

throw std::runtime_error(std::string("mmap not supported"));

}

#endif

void * addr = NULL;

size_t size = 0;

bool failed_already = false;

llama_mlock() {}

llama_mlock(const llama_mlock &) = delete;

~llama_mlock() {

if (size) {

raw_unlock(addr, size);

}

}

void init(void * ptr) {

GGML_ASSERT(addr == NULL && size == 0); // NOLINT

addr = ptr;

}

void grow_to(size_t target_size) {

GGML_ASSERT(addr);

if (failed_already) {

return;

}

size_t granularity = lock_granularity();

target_size = (target_size + granularity - 1) & ~(granularity - 1);

if (target_size > size) {

if (raw_lock((uint8_t *) addr + size, target_size - size)) {

size = target_size;

} else {

failed_already = true;

}

}

}

#ifdef _POSIX_MEMLOCK_RANGE

static constexpr bool SUPPORTED = true;

static size_t lock_granularity() {

return (size_t) sysconf(_SC_PAGESIZE);

}

#ifdef __APPLE__

#define MLOCK_SUGGESTION \

"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \

"decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n"

#else

#define MLOCK_SUGGESTION \

"Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"

#endif

bool raw_lock(const void * addr, size_t size) const {

if (!mlock(addr, size)) {

return true;

}

char* errmsg = std::strerror(errno);

bool suggest = (errno == ENOMEM);

// Check if the resource limit is fine after all

struct rlimit lock_limit;

if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {

suggest = false;

}

if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) {

suggest = false;

}

fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",

size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");

return false;

}

#undef MLOCK_SUGGESTION

static void raw_unlock(void * addr, size_t size) {

if (munlock(addr, size)) {

fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));

}

}

#elif defined(_WIN32)

static constexpr bool SUPPORTED = true;

static size_t lock_granularity() {

SYSTEM_INFO si;

GetSystemInfo(&si);

return (size_t) si.dwPageSize;

}

bool raw_lock(void * ptr, size_t len) const {

for (int tries = 1; ; tries++) {

if (VirtualLock(ptr, len)) {

return true;

}

if (tries == 2) {

fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",

len, size, llama_format_win_err(GetLastError()).c_str());

return false;

}

// It failed but this was only the first try; increase the working

// set size and try again.

SIZE_T min_ws_size, max_ws_size;

if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {

fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

return false;

}

// Per MSDN: "The maximum number of pages that a process can lock

// is equal to the number of pages in its minimum working set minus

// a small overhead."

// Hopefully a megabyte is enough overhead:

size_t increment = len + 1048576;

// The minimum must be <= the maximum, so we need to increase both:

min_ws_size += increment;

max_ws_size += increment;

if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {

fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

return false;

}

}

}

static void raw_unlock(void * ptr, size_t len) {

if (!VirtualUnlock(ptr, len)) {

fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

#else

static constexpr bool SUPPORTED = false;

static size_t lock_granularity() {

return (size_t) 65536;

}

bool raw_lock(const void * addr, size_t len) const {

fprintf(stderr, "warning: mlock not supported on this system\n");

return false;

}

static void raw_unlock(const void * addr, size_t len) {}

#endif

std::vector<char> result(8, 0);

const int n_tokens = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size());

if (n_tokens < 0) {

result.resize(-n_tokens);

int check = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size());

GGML_ASSERT(check == -n_tokens);

}

else {

result.resize(n_tokens);

}

return std::string(result.data(), result.size());

// We save the log callback globally

ggml_log_callback log_callback = llama_log_callback_default;

void * log_callback_user_data = nullptr;