diff --git a/packages/types/src/__tests__/lite-llm.test.ts b/packages/types/src/__tests__/lite-llm.test.ts new file mode 100644 index 0000000000..03424b2e57 --- /dev/null +++ b/packages/types/src/__tests__/lite-llm.test.ts @@ -0,0 +1,47 @@ +import { isLiteLLMPreserveReasoningModel, LITELLM_PRESERVE_REASONING_MODEL_IDS } from "../providers/lite-llm.js" + +describe("LiteLLM preserveReasoning model detection", () => { + it("matches every explicitly listed model id", () => { + for (const modelId of LITELLM_PRESERVE_REASONING_MODEL_IDS) { + expect(isLiteLLMPreserveReasoningModel(modelId)).toBe(true) + } + }) + + it("does not contain duplicate model ids", () => { + expect(new Set(LITELLM_PRESERVE_REASONING_MODEL_IDS).size).toBe(LITELLM_PRESERVE_REASONING_MODEL_IDS.length) + }) + + it("matches provider-prefixed routed model names by their final segment", () => { + expect(isLiteLLMPreserveReasoningModel("deepseek/deepseek-reasoner")).toBe(true) + expect(isLiteLLMPreserveReasoningModel("bedrock/moonshot.kimi-k2-thinking")).toBe(true) + expect(isLiteLLMPreserveReasoningModel("fireworks_ai/accounts/fireworks/models/kimi-k2p7-code")).toBe(true) + }) + + it("matches case-insensitively", () => { + expect(isLiteLLMPreserveReasoningModel("MiniMax-M2.7-Highspeed")).toBe(true) + expect(isLiteLLMPreserveReasoningModel("GLM-5.2")).toBe(true) + }) + + it("does not match model ids that merely contain a known family as a substring", () => { + expect(isLiteLLMPreserveReasoningModel("deepseek-v4-mini")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("mimo-v2.6")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("kimi-k2.6")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("kimi-k2.7-code")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("minimax-m4")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("minimax-m1")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("glm-4.7-flash")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("glm-4.7-flashx")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("glm-5-flash")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("glm-4.8")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("qwen3.5-plus")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("qwen3.7-mini")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("qwen3.6-max")).toBe(false) + }) + + it("does not match unrelated model names", () => { + expect(isLiteLLMPreserveReasoningModel("gpt-4")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("claude-3-opus")).toBe(false) + expect(isLiteLLMPreserveReasoningModel("")).toBe(false) + expect(isLiteLLMPreserveReasoningModel(undefined)).toBe(false) + }) +}) diff --git a/packages/types/src/providers/lite-llm.ts b/packages/types/src/providers/lite-llm.ts index 14a68cfc3c..a63f3971f5 100644 --- a/packages/types/src/providers/lite-llm.ts +++ b/packages/types/src/providers/lite-llm.ts @@ -13,3 +13,83 @@ export const litellmDefaultModelInfo: ModelInfo = { cacheWritesPrice: 3.75, cacheReadsPrice: 0.3, } + +/** + * LiteLLM is a gateway: it fronts arbitrary underlying models and its + * `/v1/model/info` response carries no reasoning-related capability flags + * (no `preserveReasoning` equivalent). The underlying model identity is only + * visible as text in the model alias (`model_name`) or the routed target + * (`litellm_params.model`, e.g. `deepseek/deepseek-reasoner`, + * `bedrock/moonshot.kimi-k2-thinking`, `fireworks_ai/.../kimi-k2p7-code`). + * + * Rather than matching model-family substrings with a regex (which can + * over-match unrelated aliases, e.g. a family fragment appearing inside a + * longer unrelated model id), this is an explicit list of the exact model + * ids that set `preserveReasoning: true` in their native provider config + * (see deepseek.ts, mimo.ts, moonshot.ts, bedrock.ts, fireworks.ts, zai.ts, + * minimax.ts, opencode-go.ts). The same behavior is inferred for a + * LiteLLM-routed alias of the same underlying model. Keep this list in sync + * with those registries. This is still best-effort: unrecognized aliases or + * renamed deployments will not match, and callers should treat it as a + * heuristic, not a source of truth. + */ +export const LITELLM_PRESERVE_REASONING_MODEL_IDS = [ + // deepseek.ts + "deepseek-v4-flash", + "deepseek-v4-pro", + "deepseek-reasoner", + + // mimo.ts, opencode-go.ts + "mimo-v2.5", + "mimo-v2.5-pro", + + // moonshot.ts, bedrock.ts, fireworks.ts + "kimi-k2-thinking", + "moonshot.kimi-k2-thinking", + "kimi-k2p7-code", + + // zai.ts + "glm-4.7", + "glm-5", + "glm-5.1", + "glm-5.2", + "glm-5-turbo", + + // bedrock.ts, minimax.ts, opencode-go.ts + "minimax.minimax-m2", + "minimax-m2", + "minimax-m2-stable", + "minimax-m2.1", + "minimax-m2.1-highspeed", + "minimax-m2.5", + "minimax-m2.5-highspeed", + "minimax-m2.7", + "minimax-m2.7-highspeed", + "minimax-m3", + + // opencode-go.ts + "qwen3.6-plus", + "qwen3.7-plus", + "qwen3.7-max", +] as const + +const LITELLM_PRESERVE_REASONING_MODEL_ID_SET = new Set(LITELLM_PRESERVE_REASONING_MODEL_IDS) + +/** + * Checks whether `modelName` (a LiteLLM alias or routed `litellm_params.model` + * value) identifies a model that requires `preserveReasoning: true`. + * Provider-prefixed routed names (e.g. `deepseek/deepseek-reasoner`, + * `fireworks_ai/accounts/fireworks/models/kimi-k2p7-code`) are matched by + * their final slash-delimited segment. + */ +export function isLiteLLMPreserveReasoningModel(modelName: string | undefined): boolean { + const normalized = modelName?.trim().toLowerCase() + + if (!normalized) { + return false + } + + const modelId = normalized.split("/").pop() + + return modelId !== undefined && LITELLM_PRESERVE_REASONING_MODEL_ID_SET.has(modelId) +} diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index ab2f261058..a5898513c5 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -1124,6 +1124,145 @@ describe("LiteLLMHandler", () => { }) }) + describe("preserveReasoning message conversion", () => { + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "ok" } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + } + }, + } + + it("uses convertToR1Format (merging tool-result text) when the model info sets preserveReasoning", async () => { + const optionsWithReasoning: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "deepseek-reasoner-alias", + } + handler = new LiteLLMHandler(optionsWithReasoning) + + vi.spyOn(handler as any, "fetchModel").mockResolvedValue({ + id: "deepseek-reasoner-alias", + info: { ...litellmDefaultModelInfo, preserveReasoning: true }, + }) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: "Hello" }, + { + role: "assistant", + content: [ + { type: "text", text: "I'll help." }, + { type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } }, + ], + // DeepSeek-style interleaved thinking: must be echoed back in the next request. + reasoning_content: "Let me check the file first.", + } as Anthropic.Messages.MessageParam & { reasoning_content: string }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_123", content: "file contents" }, + { type: "text", text: "Thanks, continue." }, + ], + }, + ] + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const generator = handler.createMessage(systemPrompt, messages) + for await (const _chunk of generator) { + // Consume + } + + const createCall = mockCreate.mock.calls[0][0] + + // convertToR1Format with mergeToolResultText folds the trailing text into the + // tool message instead of appending a separate user message. + const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool") + expect(toolMessage).toBeDefined() + expect(toolMessage.content).toBe("file contents\n\nThanks, continue.") + + const trailingUserMessage = createCall.messages.find( + (msg: any) => msg.role === "user" && msg.content === "Thanks, continue.", + ) + expect(trailingUserMessage).toBeUndefined() + + // The whole point of routing through convertToR1Format: reasoning_content + // must survive on the assistant message so the model doesn't reject the + // follow-up request for missing prior reasoning. + const assistantMessage = createCall.messages.find( + (msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0, + ) + expect(assistantMessage).toBeDefined() + expect(assistantMessage.reasoning_content).toBe("Let me check the file first.") + }) + + it("uses convertToOpenAiMessages (no merging) when the model info does not set preserveReasoning", async () => { + vi.spyOn(handler as any, "fetchModel").mockResolvedValue({ + id: litellmDefaultModelId, + info: { ...litellmDefaultModelInfo, preserveReasoning: undefined }, + }) + + const systemPrompt = "You are a helpful assistant" + // Task.buildCleanConversationHistory() (src/core/task/Task.ts) already strips the + // reasoning content block before messages reach this handler when the model's + // preserveReasoning is not true, so no reasoning_content/reasoning block is present + // on the assistant message here — this input reflects what the handler actually + // receives in that case. + const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: "Hello" }, + { + role: "assistant", + content: [ + { type: "text", text: "I'll help." }, + { type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_123", content: "file contents" }, + { type: "text", text: "Thanks, continue." }, + ], + }, + ] + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const generator = handler.createMessage(systemPrompt, messages) + for await (const _chunk of generator) { + // Consume + } + + const createCall = mockCreate.mock.calls[0][0] + + const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool") + expect(toolMessage).toBeDefined() + expect(toolMessage.content).toBe("file contents") + + const trailingUserMessage = createCall.messages.find( + (msg: any) => + msg.role === "user" && + (msg.content === "Thanks, continue." || + (Array.isArray(msg.content) && + msg.content.some((part: any) => part.text === "Thanks, continue."))), + ) + expect(trailingUserMessage).toBeDefined() + + // No reasoning_content is sent to the API in this branch, matching what + // buildCleanConversationHistory already stripped upstream. + const assistantMessage = createCall.messages.find( + (msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0, + ) + expect(assistantMessage).toBeDefined() + expect(assistantMessage.reasoning_content).toBeUndefined() + }) + }) + describe("session ID header", () => { const mockStream = { async *[Symbol.asyncIterator]() { diff --git a/src/api/providers/fetchers/__tests__/litellm.spec.ts b/src/api/providers/fetchers/__tests__/litellm.spec.ts index c05cda8839..8e6d49ddae 100644 --- a/src/api/providers/fetchers/__tests__/litellm.spec.ts +++ b/src/api/providers/fetchers/__tests__/litellm.spec.ts @@ -697,4 +697,117 @@ describe("getLiteLLMModels", () => { description: "model-with-only-max-output-tokens via LiteLLM proxy", }) }) + + describe("preserveReasoning inference", () => { + it("sets preserveReasoning: true when only the routed model (not the alias) matches a known reasoning model id", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "my-deepseek-alias", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "deepseek/deepseek-reasoner", + }, + }, + { + model_name: "my-kimi-alias", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "bedrock/moonshot.kimi-k2-thinking", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["my-deepseek-alias"]).toMatchObject({ preserveReasoning: true }) + expect(result["my-kimi-alias"]).toMatchObject({ preserveReasoning: true }) + }) + + it("omits preserveReasoning when the routed model does not match a known reasoning model id", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "gpt-4-turbo", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "openai/gpt-4-turbo", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["gpt-4-turbo"]).not.toHaveProperty("preserveReasoning") + }) + + it("matches against the model alias even when the routed model name does not match", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "glm-5.2", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "zai/some-custom-deployment", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["glm-5.2"]).toMatchObject({ preserveReasoning: true }) + }) + + it("does not match a model id that merely contains a known family as a substring", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "glm-5-flash", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + }, + litellm_params: { + model: "zai/glm-5-flash", + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["glm-5-flash"]).not.toHaveProperty("preserveReasoning") + }) + }) }) diff --git a/src/api/providers/fetchers/litellm.ts b/src/api/providers/fetchers/litellm.ts index de895d01fb..83373e5163 100644 --- a/src/api/providers/fetchers/litellm.ts +++ b/src/api/providers/fetchers/litellm.ts @@ -1,6 +1,7 @@ import axios from "axios" import type { ModelRecord } from "@roo-code/types" +import { isLiteLLMPreserveReasoningModel } from "@roo-code/types" import { DEFAULT_HEADERS } from "../constants" /** @@ -40,6 +41,11 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise if (!modelName || !modelInfo || !litellmModelName) continue + // LiteLLM's /v1/model/info never reports reasoning capability flags, so infer + // preserveReasoning from explicit model ids in either the alias or routed model name. + const preservesReasoning = + isLiteLLMPreserveReasoningModel(modelName) || isLiteLLMPreserveReasoningModel(litellmModelName) + models[modelName] = { maxTokens: modelInfo.max_output_tokens || modelInfo.max_tokens || 8192, contextWindow: modelInfo.max_input_tokens || 200000, @@ -55,6 +61,7 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise cacheReadsPrice: modelInfo.cache_read_input_token_cost ? modelInfo.cache_read_input_token_cost * 1000000 : undefined, + ...(preservesReasoning && { preserveReasoning: true }), description: `${modelName} via LiteLLM proxy`, } } diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index 49cc1b115b..74a610b073 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -9,6 +9,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" +import { convertToR1Format } from "../transform/r1-format" import { GEMINI_THOUGHT_SIGNATURE_BYPASS } from "../transform/gemini-format" import { sanitizeOpenAiCallId } from "../../utils/tool-id" @@ -117,9 +118,19 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa ): ApiStream { const { id: modelId, info } = await this.fetchModel() - const openAiMessages = convertToOpenAiMessages(messages, { - normalizeToolCallId: sanitizeOpenAiCallId, - }) + // Models that require reasoning_content to be echoed back during tool-call + // continuations (see LITELLM_PRESERVE_REASONING_MODEL_IDS) need convertToR1Format: + // it merges consecutive same-role messages and, via mergeToolResultText, folds + // text following tool_results into the last tool message so a user message + // never gets inserted mid-turn and causes the model to drop prior reasoning_content. + const openAiMessages = info.preserveReasoning + ? convertToR1Format(messages, { + normalizeToolCallId: sanitizeOpenAiCallId, + mergeToolResultText: true, + }) + : convertToOpenAiMessages(messages, { + normalizeToolCallId: sanitizeOpenAiCallId, + }) // Prepare messages with cache control if enabled and supported let systemMessage: OpenAI.Chat.ChatCompletionMessageParam