212 lines
13 KiB
Python
212 lines
13 KiB
Python
|
import sys
|
||
|
|
|
||
|
|
lines = []
|
||
|
|
lines.append("routes['POST:/v1/chat/stream'] = async (req, res) => {")
|
||
|
|
lines.append(" const userId = requireAuth(req);")
|
||
|
|
lines.append(" const body = await parseJsonBody(req);")
|
||
|
|
lines.append(" const { apiKey, model, providerName, systemPrompt } = await prepareChatRequest(userId, body);")
|
||
|
|
lines.append(" const startMs = Date.now();")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" if (METRICS_ENABLED) metrics.incCounter('chat_requests_total', { model, stream: 'true' });")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" log('info', 'Chat \u6d41\u5f0f\u5f00\u59cb', { userId, model, provider: providerName, hasFiles: body.messages.some(m => Array.isArray(m.content)), base_url: body.base_url || body.baseUrl || null });")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 Anthropic: \u6682\u4e0d\u652f\u6301\u6d41\u5f0f\u5de5\u5177\u5faa\u73af\uff0c\u4fdd\u7559\u539f\u903b\u8f91 \u2500\u2500\u2500")
|
||
|
|
lines.append(" if (providerName === 'anthropic' && !body.base_url && !body.baseUrl) {")
|
||
|
|
lines.append(" const result = await proxyChat({")
|
||
|
|
lines.append(" apiKey, model,")
|
||
|
|
lines.append(" messages: body.messages,")
|
||
|
|
lines.append(" maxTokens: body.max_tokens || body.maxTokens,")
|
||
|
|
lines.append(" stream: true,")
|
||
|
|
lines.append(" baseUrl: body.base_url || body.baseUrl,")
|
||
|
|
lines.append(" systemPrompt,")
|
||
|
|
lines.append(" }, res);")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" const latencyMs = Date.now() - startMs;")
|
||
|
|
lines.append(" const sUsage = result?.usage || {};")
|
||
|
|
lines.append(" logUsage(userId, '/v1/chat/stream', sUsage.tokensIn || 0, sUsage.tokensOut || 0, model, latencyMs);")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" if (result?.fullText) {")
|
||
|
|
lines.append(" const qc = checkResponseQuality(result.fullText);")
|
||
|
|
lines.append(" log('info', 'Chat \u6d41\u5f0f\u5b8c\u6210', { userId, model, provider: 'anthropic', latencyMs, quality: qc.score, qualityFlags: qc.flags });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" if (result && !result.streamed && !res.headersSent) {")
|
||
|
|
lines.append(" json(res, 502, { error: '\u4e0a\u6e38 API \u8fd4\u56de\u9519\u8bef', upstream_status: result.status, detail: result.data });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" return;")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 OpenAI \u517c\u5bb9 provider: \u5de5\u5177\u8c03\u7528\u5faa\u73af + \u6700\u7ec8\u6d41\u5f0f\u8f93\u51fa \u2500\u2500\u2500")
|
||
|
|
lines.append(" const config = getProviderConfig(providerName, body.base_url || body.baseUrl);")
|
||
|
|
lines.append(" const toolContext = { userId, projectId: body.project_id || undefined };")
|
||
|
|
lines.append(" toolExecutor.setLogger(log);")
|
||
|
|
lines.append(" const toolDefs = (body.enable_tools !== false) ? toolExecutor.getToolsForProvider(providerName) : null;")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" const MAX_TOOL_ROUNDS = 5;")
|
||
|
|
lines.append(" let toolRound = 0;")
|
||
|
|
lines.append(" const messages = body.messages.slice(); // \u5de5\u5177\u5faa\u73af\u4f7f\u7528\u7684\u6d88\u606f\u5217\u8868\u526f\u672c")
|
||
|
|
lines.append(" const allToolCalls = []; // \u8bb0\u5f55\u6240\u6709\u5de5\u5177\u8c03\u7528\uff0c\u7528\u4e8e\u524d\u7aef\u6c47\u603b\u5c55\u793a")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u53d1\u9001\u4e00\u6761 SSE \u4e8b\u4ef6\u7ed9\u524d\u7aef\uff0c\u786e\u4fdd headers \u5df2\u53d1\u51fa")
|
||
|
|
lines.append(" function sendSSE(data) {")
|
||
|
|
lines.append(" const line = 'data: ' + JSON.stringify(data) + '\\n\\n';")
|
||
|
|
lines.append(" if (!res.headersSent) {")
|
||
|
|
lines.append(" res.writeHead(200, {")
|
||
|
|
lines.append(" 'Content-Type': 'text/event-stream',")
|
||
|
|
lines.append(" 'Cache-Control': 'no-cache',")
|
||
|
|
lines.append(" 'Connection': 'keep-alive',")
|
||
|
|
lines.append(" 'X-Accel-Buffering': 'no',")
|
||
|
|
lines.append(" });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" res.write(line);")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u4e2d\u95f4\u5de5\u5177\u5faa\u73af\u8f6e\u4f7f\u7528\u975e\u6d41\u5f0f\u8bf7\u6c42\uff0c\u907f\u514d\u6d41\u5f0f\u62fc\u63a5\u590d\u6742\u5ea6")
|
||
|
|
lines.append(" async function callLLMNonStream(msgs) {")
|
||
|
|
lines.append(" const reqBody = config.buildBody({")
|
||
|
|
lines.append(" model, messages: msgs,")
|
||
|
|
lines.append(" maxTokens: body.max_tokens || body.maxTokens,")
|
||
|
|
lines.append(" stream: false,")
|
||
|
|
lines.append(" systemPrompt,")
|
||
|
|
lines.append(" });")
|
||
|
|
lines.append(" if (toolDefs) {")
|
||
|
|
lines.append(" reqBody.tools = toolDefs;")
|
||
|
|
lines.append(" reqBody.tool_choice = 'auto';")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" return sendLLMRequest(")
|
||
|
|
lines.append(" { name: providerName, baseUrl: config.baseUrl },")
|
||
|
|
lines.append(" apiKey, reqBody, null, false")
|
||
|
|
lines.append(" );")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 \u5de5\u5177\u8c03\u7528\u5faa\u73af \u2500\u2500\u2500")
|
||
|
|
lines.append(" while (toolRound < MAX_TOOL_ROUNDS) {")
|
||
|
|
lines.append(" let result;")
|
||
|
|
lines.append(" try {")
|
||
|
|
lines.append(" result = await callLLMNonStream(messages);")
|
||
|
|
lines.append(" } catch (e) {")
|
||
|
|
lines.append(" log('error', 'Stream \u5de5\u5177\u5faa\u73af LLM \u8bf7\u6c42\u5931\u8d25', { userId, round: toolRound, error: e.message });")
|
||
|
|
lines.append(" if (!res.headersSent) json(res, 502, { error: 'LLM \u8bf7\u6c42\u5931\u8d25: ' + e.message });")
|
||
|
|
lines.append(" else res.end();")
|
||
|
|
lines.append(" return;")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" if (result.status >= 400) {")
|
||
|
|
lines.append(" log('error', 'Stream \u5de5\u5177\u5faa\u73af\u4e0a\u6e38\u9519\u8bef', { userId, round: toolRound, status: result.status });")
|
||
|
|
lines.append(" if (!res.headersSent) json(res, 502, { error: '\u4e0a\u6e38 API \u8fd4\u56de\u9519\u8bef', upstream_status: result.status, detail: result.data });")
|
||
|
|
lines.append(" else res.end();")
|
||
|
|
lines.append(" return;")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" const responseData = result.data;")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u5728\u8c03\u7528 processOpenAIToolCalls \u524d\uff0c\u5148\u8bb0\u5f55 messages \u957f\u5ea6")
|
||
|
|
lines.append(" const msgLenBefore = messages.length;")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u68c0\u6d4b\u5e76\u6267\u884c\u5de5\u5177\u8c03\u7528\uff08\u4f1a\u4fee\u6539 messages \u6570\u7ec4\uff0c\u8ffd\u52a0 assistant + tool \u6d88\u606f\uff09")
|
||
|
|
lines.append(" const toolInfo = await toolExecutor.processOpenAIToolCalls(responseData, messages, toolContext);")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" if (!toolInfo.hasToolCalls) {")
|
||
|
|
lines.append(" // \u6ca1\u6709\u5de5\u5177\u8c03\u7528 \u2014 \u8fd9\u662f\u6700\u7ec8\u54cd\u5e94\uff0c\u8df3\u51fa\u5faa\u73af\u8fdb\u884c\u6d41\u5f0f\u8f93\u51fa")
|
||
|
|
lines.append(" break;")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 \u6709\u5de5\u5177\u8c03\u7528: \u5411\u524d\u7aef\u63a8\u9001\u72b6\u6001\u901a\u77e5 \u2500\u2500\u2500")
|
||
|
|
lines.append(" toolRound++;")
|
||
|
|
lines.append(" log('info', 'Stream Tool Use \u5faa\u73af', { userId, round: toolRound, toolCount: toolInfo.toolCallCount, model });")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // processOpenAIToolCalls \u8ffd\u52a0\u4e86: [assistant(tool_calls), tool_result_1, ...]")
|
||
|
|
lines.append(" const assistantMsg = messages[msgLenBefore];")
|
||
|
|
lines.append(" if (assistantMsg && assistantMsg.tool_calls) {")
|
||
|
|
lines.append(" for (let i = 0; i < assistantMsg.tool_calls.length; i++) {")
|
||
|
|
lines.append(" const tc = assistantMsg.tool_calls[i];")
|
||
|
|
lines.append(" let parsedArgs = {};")
|
||
|
|
lines.append(" try { parsedArgs = JSON.parse(tc.function.arguments || '{}'); } catch (_) {}")
|
||
|
|
lines.append(" const toolResultMsg = messages[msgLenBefore + 1 + i];")
|
||
|
|
lines.append(" let toolResult = {};")
|
||
|
|
lines.append(" try { toolResult = JSON.parse((toolResultMsg && toolResultMsg.content) || '{}'); } catch (_) {}")
|
||
|
|
lines.append(" // \u5411\u524d\u7aef\u53d1\u9001\u5de5\u5177\u6267\u884c\u72b6\u6001\uff08running -> done\uff09")
|
||
|
|
lines.append(" sendSSE({ type: 'tool_status', tool: tc.function.name, status: 'running', args: parsedArgs });")
|
||
|
|
lines.append(" sendSSE({ type: 'tool_status', tool: tc.function.name, status: 'done', result: toolResult });")
|
||
|
|
lines.append(" allToolCalls.push({ name: tc.function.name, args: parsedArgs, result: toolResult });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 \u53d1\u9001\u5de5\u5177\u8c03\u7528\u6c47\u603b\uff08\u5982\u679c\u6709\u5de5\u5177\u8c03\u7528\uff09\u2500\u2500\u2500")
|
||
|
|
lines.append(" if (allToolCalls.length > 0) {")
|
||
|
|
lines.append(" sendSSE({ type: 'tool_calls_summary', calls: allToolCalls });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" // \u2500\u2500\u2500 \u6700\u7ec8\u8f6e: \u6d41\u5f0f\u8f93\u51fa\u7ed9\u5ba2\u6237\u7aef \u2500\u2500\u2500")
|
||
|
|
lines.append(" const finalReqBody = config.buildBody({")
|
||
|
|
lines.append(" model, messages,")
|
||
|
|
lines.append(" maxTokens: body.max_tokens || body.maxTokens,")
|
||
|
|
lines.append(" stream: true,")
|
||
|
|
lines.append(" systemPrompt,")
|
||
|
|
lines.append(" });")
|
||
|
|
lines.append(" // \u6700\u7ec8\u8f6e\u4e0d\u9644\u5e26 tools\uff0c\u907f\u514d\u518d\u6b21\u89e6\u53d1\u5de5\u5177\u8c03\u7528")
|
||
|
|
lines.append("")
|
||
|
|
lines.append(" try {")
|
||
|
|
lines.append(" if (res.headersSent) {")
|
||
|
|
lines.append(" // headers \u5df2\u53d1\uff08\u6709\u5de5\u5177\u8c03\u7528\u8f6e\uff09: sendLLMRequest \u65e0\u6cd5\u518d writeHead\uff0c\u6539\u7528\u975e\u6d41\u5f0f\u540e\u624b\u52a8\u53d1 SSE")
|
||
|
|
lines.append(" const nr = await sendLLMRequest(")
|
||
|
|
lines.append(" { name: providerName, baseUrl: config.baseUrl },")
|
||
|
|
lines.append(" apiKey, { ...finalReqBody, stream: false }, null, false")
|
||
|
|
lines.append(" );")
|
||
|
|
lines.append(" const latencyMs = Date.now() - startMs;")
|
||
|
|
lines.append(" if (nr.status >= 400) {")
|
||
|
|
lines.append(" res.write('data: ' + JSON.stringify({ type: 'error', error: '\u6700\u7ec8\u54cd\u5e94\u5931\u8d25' }) + '\\n\\n');")
|
||
|
|
lines.append(" res.end();")
|
||
|
|
lines.append(" return;")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" const finalText = (nr.data && nr.data.choices && nr.data.choices[0] &&")
|
||
|
|
lines.append(" nr.data.choices[0].message && nr.data.choices[0].message.content) || '';")
|
||
|
|
lines.append(" // \u5c06\u6700\u7ec8\u6587\u672c\u62c6\u5206\u4e3a 20 \u5b57\u7b26\u7684\u5757\uff0c\u6a21\u62df\u6d41\u5f0f\u683c\u5f0f\u8f93\u51fa")
|
||
|
|
lines.append(" const CHUNK_SIZE = 20;")
|
||
|
|
lines.append(" for (let i = 0; i < finalText.length; i += CHUNK_SIZE) {")
|
||
|
|
lines.append(" const piece = finalText.slice(i, i + CHUNK_SIZE);")
|
||
|
|
lines.append(" const fakeChunk = { choices: [{ delta: { content: piece }, index: 0, finish_reason: null }] };")
|
||
|
|
lines.append(" res.write('data: ' + JSON.stringify(fakeChunk) + '\\n\\n');")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" res.write('data: [DONE]\\n\\n');")
|
||
|
|
lines.append(" res.end();")
|
||
|
|
lines.append(" const sUsage = config.parseUsage(nr.data);")
|
||
|
|
lines.append(" logUsage(userId, '/v1/chat/stream', sUsage.input_tokens || 0, sUsage.output_tokens || 0, model, latencyMs);")
|
||
|
|
lines.append(" if (finalText) {")
|
||
|
|
lines.append(" const qc = checkResponseQuality(finalText);")
|
||
|
|
lines.append(" log('info', 'Chat \u6d41\u5f0f\u5b8c\u6210(\u5de5\u5177\u540e)', { userId, model, provider: providerName, latencyMs, toolRounds: toolRound, quality: qc.score });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" } else {")
|
||
|
|
lines.append(" // \u65e0\u5de5\u5177\u8c03\u7528\uff0c\u76f4\u63a5\u6d41\u5f0f\u8f93\u51fa\uff08\u884c\u4e3a\u4e0e\u539f\u4ee3\u7801\u5b8c\u5168\u4e00\u81f4\uff09")
|
||
|
|
lines.append(" const finalResult = await sendLLMRequest(")
|
||
|
|
lines.append(" { name: providerName, baseUrl: config.baseUrl },")
|
||
|
|
lines.append(" apiKey, finalReqBody, res, true")
|
||
|
|
lines.append(" );")
|
||
|
|
lines.append(" const latencyMs = Date.now() - startMs;")
|
||
|
|
lines.append(" const sUsage = finalResult?.usage || {};")
|
||
|
|
lines.append(" logUsage(userId, '/v1/chat/stream', sUsage.input_tokens || 0, sUsage.output_tokens || 0, model, latencyMs);")
|
||
|
|
lines.append(" if (finalResult?.fullText) {")
|
||
|
|
lines.append(" const qc = checkResponseQuality(finalResult.fullText);")
|
||
|
|
lines.append(" log('info', 'Chat \u6d41\u5f0f\u5b8c\u6210', { userId, model, provider: providerName, latencyMs, toolRounds: toolRound, quality: qc.score, qualityFlags: qc.flags });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" if (finalResult && !finalResult.streamed && !res.headersSent) {")
|
||
|
|
lines.append(" json(res, 502, { error: '\u4e0a\u6e38 API \u8fd4\u56de\u9519\u8bef', upstream_status: finalResult.status, detail: finalResult.data });")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" } catch (e) {")
|
||
|
|
lines.append(" log('error', 'Stream \u6700\u7ec8\u8f6e\u5931\u8d25', { userId, error: e.message });")
|
||
|
|
lines.append(" if (!res.headersSent) json(res, 502, { error: '\u6d41\u5f0f\u8f93\u51fa\u5931\u8d25: ' + e.message });")
|
||
|
|
lines.append(" else {")
|
||
|
|
lines.append(" res.write('data: ' + JSON.stringify({ type: 'error', error: e.message }) + '\\n\\n');")
|
||
|
|
lines.append(" res.end();")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append(" }")
|
||
|
|
lines.append("};")
|
||
|
|
|
||
|
|
content = '\n'.join(lines) + '\n'
|
||
|
|
with open('/tmp/new_block.js', 'w') as f:
|
||
|
|
f.write(content)
|
||
|
|
print('OK: written', len(lines), 'lines,', len(content), 'chars')
|