messages: Anthropic.Beta.Messages.BetaMessageParam[],

tools: Anthropic.Beta.Messages.BetaToolUnion[],

): Promise<number | null> {

try {

const result = await countMessagesTokensWithAPI(messages, tools)

if (result !== null) {

return result

}

logForDebugging(

`countTokensWithFallback: API returned null, trying haiku fallback (${tools.length} tools)`,

)

} catch (err) {

logForDebugging(`countTokensWithFallback: API failed: ${errorMessage(err)}`)

logError(err)

}

try {

const fallbackResult = await countTokensViaHaikuFallback(messages, tools)

if (fallbackResult === null) {

logForDebugging(

`countTokensWithFallback: haiku fallback also returned null (${tools.length} tools)`,

)

}

return fallbackResult

} catch (err) {

logForDebugging(

`countTokensWithFallback: haiku fallback failed: ${errorMessage(err)}`,

)

logError(err)

return null

}

name: string

tokens: number

color: keyof Theme

/** When true, these tokens are deferred and don't count toward context usage */

isDeferred?: boolean

color: keyof Theme

isFilled: boolean

categoryName: string

tokens: number

percentage: number

squareFullness: number // 0-1 representing how full this individual square is

path: string

type: string

tokens: number

name: string

serverName: string

tokens: number

isLoaded?: boolean

name: string

tokens: number

isLoaded: boolean

agentType: string

source: SettingSource | 'built-in' | 'plugin'

tokens: number

readonly totalCommands: number

readonly includedCommands: number

readonly tokens: number

name: string

source: SettingSource | 'plugin'

tokens: number

/** Total number of available skills */

readonly totalSkills: number

/** Number of skills included within token budget */

readonly includedSkills: number

/** Total tokens consumed by skills */

readonly tokens: number

/** Individual skill details */

readonly skillFrontmatter: SkillFrontmatter[]

readonly categories: ContextCategory[]

readonly totalTokens: number

readonly maxTokens: number

readonly rawMaxTokens: number

readonly percentage: number

readonly gridRows: GridSquare[][]

readonly model: string

readonly memoryFiles: MemoryFile[]

readonly mcpTools: McpTool[]

/** Ant-only: per-tool breakdown of deferred built-in tools */

readonly deferredBuiltinTools?: DeferredBuiltinTool[]

/** Ant-only: per-tool breakdown of always-loaded built-in tools */

readonly systemTools?: SystemToolDetail[]

/** Ant-only: per-section breakdown of system prompt */

readonly systemPromptSections?: SystemPromptSectionDetail[]

readonly agents: Agent[]

readonly slashCommands?: SlashCommandInfo

/** Skill statistics */

readonly skills?: SkillInfo

readonly autoCompactThreshold?: number

readonly isAutoCompactEnabled: boolean

messageBreakdown?: {

toolCallTokens: number

toolResultTokens: number

attachmentTokens: number

assistantMessageTokens: number

userMessageTokens: number

toolCallsByType: Array<{

name: string

callTokens: number

resultTokens: number

}>

attachmentsByType: Array<{ name: string; tokens: number }>

}

/** Actual token usage from last API response (if available) */

readonly apiUsage: {

input_tokens: number

output_tokens: number

cache_creation_input_tokens: number

cache_read_input_tokens: number

} | null

tools: Tools,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

agentInfo: AgentDefinitionsResult | null,

model?: string,

): Promise<number> {

const toolSchemas = await Promise.all(

tools.map(tool =>

toolToAPISchema(tool, {

getToolPermissionContext,

tools,

agents: agentInfo?.activeAgents ?? [],

model,

}),

),

)

const result = await countTokensWithFallback([], toolSchemas)

if (result === null || result === 0) {

const toolNames = tools.map(t => t.name).join(', ')

logForDebugging(

`countToolDefinitionTokens returned ${result} for ${tools.length} tools: ${toolNames.slice(0, 100)}${toolNames.length > 100 ? '...' : ''}`,

)

}

return result ?? 0

// Try to find first markdown heading

const headingMatch = content.match(/^#+\s+(.+)$/m)

if (headingMatch) {

return headingMatch[1]!.trim()

}

// Fall back to a truncated preview of the first non-empty line

const firstLine = content.split('\n').find(l => l.trim().length > 0) ?? ''

return firstLine.length > 40 ? firstLine.slice(0, 40) + '…' : firstLine

effectiveSystemPrompt: readonly string[],

): Promise<{

systemPromptTokens: number

systemPromptSections: SystemPromptSectionDetail[]

}> {

// Get system context (gitStatus, etc.) which is always included

const systemContext = await getSystemContext()

// Build named entries: system prompt parts + system context values

// Skip empty strings and the global-cache boundary marker

const namedEntries: Array<{ name: string; content: string }> = [

...effectiveSystemPrompt

.filter(

content =>

content.length > 0 && content !== SYSTEM_PROMPT_DYNAMIC_BOUNDARY,

)

.map(content => ({ name: extractSectionName(content), content })),

...Object.entries(systemContext)

.filter(([, content]) => content.length > 0)

.map(([name, content]) => ({ name, content })),

]

if (namedEntries.length < 1) {

return { systemPromptTokens: 0, systemPromptSections: [] }

}

const systemTokenCounts = await Promise.all(

namedEntries.map(({ content }) =>

countTokensWithFallback([{ role: 'user', content }], []),

),

)

const systemPromptSections: SystemPromptSectionDetail[] = namedEntries.map(

(entry, i) => ({

name: entry.name,

tokens: systemTokenCounts[i] || 0,

}),

)

const systemPromptTokens = systemTokenCounts.reduce(

(sum: number, tokens) => sum + (tokens || 0),

0,

)

return { systemPromptTokens, systemPromptSections }

memoryFileDetails: MemoryFile[]

claudeMdTokens: number

}> {

// Simple mode disables CLAUDE.md loading, so don't report tokens for them

if (isEnvTruthy(process.env.CLAUDE_CODE_SIMPLE)) {

return { memoryFileDetails: [], claudeMdTokens: 0 }

}

const memoryFilesData = filterInjectedMemoryFiles(await getMemoryFiles())

const memoryFileDetails: MemoryFile[] = []

let claudeMdTokens = 0

if (memoryFilesData.length < 1) {

return {

memoryFileDetails: [],

claudeMdTokens: 0,

}

}

const claudeMdTokenCounts = await Promise.all(

memoryFilesData.map(async file => {

const tokens = await countTokensWithFallback(

[{ role: 'user', content: file.content }],

[],

)

return { file, tokens: tokens || 0 }

}),

)

for (const { file, tokens } of claudeMdTokenCounts) {

claudeMdTokens += tokens

memoryFileDetails.push({

path: file.path,

type: file.type,

tokens,

})

}

return { claudeMdTokens, memoryFileDetails }

tools: Tools,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

agentInfo: AgentDefinitionsResult | null,

model?: string,

messages?: Message[],

): Promise<{

builtInToolTokens: number

deferredBuiltinDetails: DeferredBuiltinTool[]

deferredBuiltinTokens: number

systemToolDetails: SystemToolDetail[]

}> {

const builtInTools = tools.filter(tool => !tool.isMcp)

if (builtInTools.length < 1) {

return {

builtInToolTokens: 0,

deferredBuiltinDetails: [],

deferredBuiltinTokens: 0,

systemToolDetails: [],

}

}

// Check if tool search is enabled

const { isToolSearchEnabled } = await import('./toolSearch.js')

const { isDeferredTool } = await import('../tools/ToolSearchTool/prompt.js')

const isDeferred = await isToolSearchEnabled(

model ?? '',

tools,

getToolPermissionContext,

agentInfo?.activeAgents ?? [],

'analyzeBuiltIn',

)

// Separate always-loaded and deferred builtin tools using dynamic isDeferredTool check

const alwaysLoadedTools = builtInTools.filter(t => !isDeferredTool(t))

const deferredBuiltinTools = builtInTools.filter(t => isDeferredTool(t))

// Count always-loaded tools

const alwaysLoadedTokens =

alwaysLoadedTools.length > 0

? await countToolDefinitionTokens(

alwaysLoadedTools,

getToolPermissionContext,

agentInfo,

model,

)

: 0

// Build per-tool breakdown for always-loaded tools (ant-only, proportional

// split of the bulk count based on rough schema size estimation). Excludes

// SkillTool since its tokens are shown in the separate Skills category.

let systemToolDetails: SystemToolDetail[] = []

if (process.env.USER_TYPE === 'ant') {

const toolsForBreakdown = alwaysLoadedTools.filter(

t => !toolMatchesName(t, SKILL_TOOL_NAME),

)

if (toolsForBreakdown.length > 0) {

const estimates = toolsForBreakdown.map(t =>

roughTokenCountEstimation(jsonStringify(t.inputSchema ?? {})),

)

const estimateTotal = estimates.reduce((s, e) => s + e, 0) || 1

const distributable = Math.max(

0,

alwaysLoadedTokens - TOOL_TOKEN_COUNT_OVERHEAD,

)

systemToolDetails = toolsForBreakdown

.map((t, i) => ({

name: t.name,

tokens: Math.round((estimates[i]! / estimateTotal) * distributable),

}))

.sort((a, b) => b.tokens - a.tokens)

}

}

// Count deferred builtin tools individually for details

const deferredBuiltinDetails: DeferredBuiltinTool[] = []

let loadedDeferredTokens = 0

let totalDeferredTokens = 0

if (deferredBuiltinTools.length > 0 && isDeferred) {

// Find which deferred tools have been used in messages

const loadedToolNames = new Set<string>()

if (messages) {

const deferredToolNameSet = new Set(deferredBuiltinTools.map(t => t.name))

for (const msg of messages) {

if (msg.type === 'assistant') {

for (const block of msg.message.content) {

if (

'type' in block &&

block.type === 'tool_use' &&

'name' in block &&

typeof block.name === 'string' &&

deferredToolNameSet.has(block.name)

) {

loadedToolNames.add(block.name)

}

}

}

}

}

// Count each deferred tool

const tokensByTool = await Promise.all(

deferredBuiltinTools.map(t =>

countToolDefinitionTokens(

[t],

getToolPermissionContext,

agentInfo,

model,

),

),

)

for (const [i, tool] of deferredBuiltinTools.entries()) {

const tokens = Math.max(

0,

(tokensByTool[i] || 0) - TOOL_TOKEN_COUNT_OVERHEAD,

)

const isLoaded = loadedToolNames.has(tool.name)

deferredBuiltinDetails.push({

name: tool.name,

tokens,

isLoaded,

})

totalDeferredTokens += tokens

if (isLoaded) {

loadedDeferredTokens += tokens

}

}

} else if (deferredBuiltinTools.length > 0) {

// Tool search not enabled - count deferred tools as regular

const deferredTokens = await countToolDefinitionTokens(

deferredBuiltinTools,

getToolPermissionContext,

agentInfo,

model,

)

return {

builtInToolTokens: alwaysLoadedTokens + deferredTokens,

deferredBuiltinDetails: [],

deferredBuiltinTokens: 0,

systemToolDetails,

}

}

return {

// When deferred, only count always-loaded tools + any loaded deferred tools

builtInToolTokens: alwaysLoadedTokens + loadedDeferredTokens,

deferredBuiltinDetails,

deferredBuiltinTokens: totalDeferredTokens - loadedDeferredTokens,

systemToolDetails,

}

tools: Tools,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

agentInfo: AgentDefinitionsResult | null,

): Promise<{

slashCommandTokens: number

commandInfo: { totalCommands: number; includedCommands: number }

}> {

const info = await getSlashCommandInfo(getCwd())

const slashCommandTool = findSkillTool(tools)

if (!slashCommandTool) {

return {

slashCommandTokens: 0,

commandInfo: { totalCommands: 0, includedCommands: 0 },

}

}

const slashCommandTokens = await countToolDefinitionTokens(

[slashCommandTool],

getToolPermissionContext,

agentInfo,

)

return {

slashCommandTokens,

commandInfo: {

totalCommands: info.totalCommands,

includedCommands: info.includedCommands,

},

}

tools: Tools,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

agentInfo: AgentDefinitionsResult | null,

): Promise<{

skillTokens: number

skillInfo: {

totalSkills: number

includedSkills: number

skillFrontmatter: SkillFrontmatter[]

}

}> {

try {

const skills = await getLimitedSkillToolCommands(getCwd())

const slashCommandTool = findSkillTool(tools)

if (!slashCommandTool) {

return {

skillTokens: 0,

skillInfo: { totalSkills: 0, includedSkills: 0, skillFrontmatter: [] },

}

}

// NOTE: This counts the entire SlashCommandTool (which includes both commands AND skills).

// This is the same tool counted by countSlashCommandTokens(), but we track it separately

// here for display purposes. These tokens should NOT be added to context categories

// to avoid double-counting.

const skillTokens = await countToolDefinitionTokens(

[slashCommandTool],

getToolPermissionContext,

agentInfo,

)

// Calculate per-skill token estimates based on frontmatter only

// (name, description, whenToUse) since full content is only loaded on invocation

const skillFrontmatter: SkillFrontmatter[] = skills.map(skill => ({

name: getCommandName(skill),

source: (skill.type === 'prompt' ? skill.source : 'plugin') as

| SettingSource

| 'plugin',

tokens: estimateSkillFrontmatterTokens(skill),

}))

return {

skillTokens,

skillInfo: {

totalSkills: skills.length,

includedSkills: skills.length,

skillFrontmatter,

},

}

} catch (error) {

logError(toError(error))

// Return zero values rather than failing the entire context analysis

return {

skillTokens: 0,

skillInfo: { totalSkills: 0, includedSkills: 0, skillFrontmatter: [] },

}

}

tools: Tools,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

agentInfo: AgentDefinitionsResult | null,

model: string,

messages?: Message[],

): Promise<{

mcpToolTokens: number

mcpToolDetails: McpTool[]

deferredToolTokens: number

loadedMcpToolNames: Set<string>

}> {

const mcpTools = tools.filter(tool => tool.isMcp)

const mcpToolDetails: McpTool[] = []

// Single bulk API call for all MCP tools (instead of N individual calls)

const totalTokensRaw = await countToolDefinitionTokens(

mcpTools,

getToolPermissionContext,

agentInfo,

model,

)

// Subtract the single overhead since we made one bulk call

const totalTokens = Math.max(

0,

(totalTokensRaw || 0) - TOOL_TOKEN_COUNT_OVERHEAD,

)

// Estimate per-tool proportions for display using local estimation.

// Include name + description + input schema to match what toolToAPISchema

// sends — otherwise tools with similar schemas but different descriptions

// get identical counts (MCP tools share the same base Zod inputSchema).

const estimates = await Promise.all(

mcpTools.map(async t =>

roughTokenCountEstimation(

jsonStringify({

name: t.name,

description: await t.prompt({

getToolPermissionContext,

tools,

agents: agentInfo?.activeAgents ?? [],

}),

input_schema: t.inputJSONSchema ?? {},

}),

),

),

)

const estimateTotal = estimates.reduce((s, e) => s + e, 0) || 1

const mcpToolTokensByTool = estimates.map(e =>

Math.round((e / estimateTotal) * totalTokens),

)

// Check if tool search is enabled - if so, MCP tools are deferred

// isToolSearchEnabled handles threshold calculation internally for TstAuto mode

const { isToolSearchEnabled } = await import('./toolSearch.js')

const { isDeferredTool } = await import('../tools/ToolSearchTool/prompt.js')

const isDeferred = await isToolSearchEnabled(

model,

tools,

getToolPermissionContext,

agentInfo?.activeAgents ?? [],

'analyzeMcp',

)

// Find MCP tools that have been used in messages (loaded via ToolSearchTool)

const loadedMcpToolNames = new Set<string>()

if (isDeferred && messages) {

const mcpToolNameSet = new Set(mcpTools.map(t => t.name))

for (const msg of messages) {

if (msg.type === 'assistant') {

for (const block of msg.message.content) {

if (

'type' in block &&

block.type === 'tool_use' &&

'name' in block &&

typeof block.name === 'string' &&

mcpToolNameSet.has(block.name)

) {

loadedMcpToolNames.add(block.name)

}

}

}

}

}

// Build tool details with isLoaded flag

for (const [i, tool] of mcpTools.entries()) {

mcpToolDetails.push({

name: tool.name,

serverName: tool.name.split('__')[1] || 'unknown',

tokens: mcpToolTokensByTool[i]!,

isLoaded: loadedMcpToolNames.has(tool.name) || !isDeferredTool(tool),

})

}

// Calculate loaded vs deferred tokens

let loadedTokens = 0

let deferredTokens = 0

for (const detail of mcpToolDetails) {

if (detail.isLoaded) {

loadedTokens += detail.tokens

} else if (isDeferred) {

deferredTokens += detail.tokens

}

}

return {

// When deferred but some tools are loaded, count loaded tokens

mcpToolTokens: isDeferred ? loadedTokens : totalTokens,

mcpToolDetails,

// Track deferred tokens separately for display

deferredToolTokens: deferredTokens,

loadedMcpToolNames,

}

activeAgents: AgentDefinition[]

}): Promise<{

agentTokens: number

agentDetails: Agent[]

}> {

const customAgents = agentDefinitions.activeAgents.filter(

a => a.source !== 'built-in',

)

const agentDetails: Agent[] = []

let agentTokens = 0

const tokenCounts = await Promise.all(

customAgents.map(agent =>

countTokensWithFallback(

[

{

role: 'user',

content: [agent.agentType, agent.whenToUse].join(' '),

},

],

[],

),

),

)

for (const [i, agent] of customAgents.entries()) {

const tokens = tokenCounts[i] || 0

agentTokens += tokens || 0

agentDetails.push({

agentType: agent.agentType,

source: agent.source,

tokens: tokens || 0,

})

}

return { agentTokens, agentDetails }

totalTokens: number

toolCallTokens: number

toolResultTokens: number

attachmentTokens: number

assistantMessageTokens: number

userMessageTokens: number

toolCallsByType: Map<string, number>

toolResultsByType: Map<string, number>

attachmentsByType: Map<string, number>

msg: AssistantMessage | NormalizedAssistantMessage,

breakdown: MessageBreakdown,

): void {

// Process each content block individually

for (const block of msg.message.content) {

const blockStr = jsonStringify(block)

const blockTokens = roughTokenCountEstimation(blockStr)

if ('type' in block && block.type === 'tool_use') {

breakdown.toolCallTokens += blockTokens

const toolName = ('name' in block ? block.name : undefined) || 'unknown'

breakdown.toolCallsByType.set(

toolName,

(breakdown.toolCallsByType.get(toolName) || 0) + blockTokens,

)

} else {

// Text blocks or other non-tool content

breakdown.assistantMessageTokens += blockTokens

}

}

msg: UserMessage | NormalizedUserMessage,

breakdown: MessageBreakdown,

toolUseIdToName: Map<string, string>,

): void {

// Handle both string and array content

if (typeof msg.message.content === 'string') {

// Simple string content

const tokens = roughTokenCountEstimation(msg.message.content)

breakdown.userMessageTokens += tokens

return

}

// Process each content block individually

for (const block of msg.message.content) {

const blockStr = jsonStringify(block)

const blockTokens = roughTokenCountEstimation(blockStr)

if ('type' in block && block.type === 'tool_result') {

breakdown.toolResultTokens += blockTokens

const toolUseId = 'tool_use_id' in block ? block.tool_use_id : undefined

const toolName =

(toolUseId ? toolUseIdToName.get(toolUseId) : undefined) || 'unknown'

breakdown.toolResultsByType.set(

toolName,

(breakdown.toolResultsByType.get(toolName) || 0) + blockTokens,

)

} else {

// Text blocks or other non-tool content

breakdown.userMessageTokens += blockTokens

}

}

msg: AttachmentMessage,

breakdown: MessageBreakdown,

): void {

const contentStr = jsonStringify(msg.attachment)

const tokens = roughTokenCountEstimation(contentStr)

breakdown.attachmentTokens += tokens

const attachType = msg.attachment.type || 'unknown'

breakdown.attachmentsByType.set(

attachType,

(breakdown.attachmentsByType.get(attachType) || 0) + tokens,

)

messages: Message[],

): Promise<MessageBreakdown> {

const microcompactResult = await microcompactMessages(messages)

// Initialize tracking

const breakdown: MessageBreakdown = {

totalTokens: 0,

toolCallTokens: 0,

toolResultTokens: 0,

attachmentTokens: 0,

assistantMessageTokens: 0,

userMessageTokens: 0,

toolCallsByType: new Map<string, number>(),

toolResultsByType: new Map<string, number>(),

attachmentsByType: new Map<string, number>(),

}

// Build a map of tool_use_id to tool_name for easier lookup

const toolUseIdToName = new Map<string, string>()

for (const msg of microcompactResult.messages) {

if (msg.type === 'assistant') {

for (const block of msg.message.content) {

if ('type' in block && block.type === 'tool_use') {

const toolUseId = 'id' in block ? block.id : undefined

const toolName =

('name' in block ? block.name : undefined) || 'unknown'

if (toolUseId) {

toolUseIdToName.set(toolUseId, toolName)

}

}

}

}

}

// Process each message for detailed breakdown

for (const msg of microcompactResult.messages) {

if (msg.type === 'assistant') {

processAssistantMessage(msg, breakdown)

} else if (msg.type === 'user') {

processUserMessage(msg, breakdown, toolUseIdToName)

} else if (msg.type === 'attachment') {

processAttachment(msg, breakdown)

}

}

// Calculate total tokens using the API for accuracy

const approximateMessageTokens = await countTokensWithFallback(

normalizeMessagesForAPI(microcompactResult.messages).map(_ => {

if (_.type === 'assistant') {

return {

// Important: strip out fields like id, etc. -- the counting API errors if they're present

role: 'assistant',

content: _.message.content,

}

}

return _.message

}),

[],

)

breakdown.totalTokens = approximateMessageTokens ?? 0

return breakdown

messages: Message[],

model: string,

getToolPermissionContext: () => Promise<ToolPermissionContext>,

tools: Tools,

agentDefinitions: AgentDefinitionsResult,

terminalWidth?: number,

toolUseContext?: Pick<ToolUseContext, 'options'>,

mainThreadAgentDefinition?: AgentDefinition,

/** Original messages before microcompact, used to extract API usage */

originalMessages?: Message[],

): Promise<ContextData> {

const runtimeModel = getRuntimeMainLoopModel({

permissionMode: (await getToolPermissionContext()).mode,

mainLoopModel: model,

})

// Get context window size

const contextWindow = getContextWindowForModel(runtimeModel, getSdkBetas())

// Build the effective system prompt using the shared utility

const defaultSystemPrompt = await getSystemPrompt(tools, runtimeModel)

const effectiveSystemPrompt = buildEffectiveSystemPrompt({

mainThreadAgentDefinition,

toolUseContext: toolUseContext ?? {

options: {} as ToolUseContext['options'],

},

customSystemPrompt: toolUseContext?.options.customSystemPrompt,

defaultSystemPrompt,

appendSystemPrompt: toolUseContext?.options.appendSystemPrompt,