main: 初始化 Face to API 项目
- 添加前端抽卡台与模型配置界面 - 添加 Express API 服务与多厂商模型调用适配 - 配置本地环境示例、构建脚本和忽略规则
This commit is contained in:
334
server/index.ts
Normal file
334
server/index.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
import "dotenv/config";
|
||||
import cors from "cors";
|
||||
import express from "express";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { z } from "zod";
|
||||
import { findModel, providerForModel, registry } from "./modelRegistry.js";
|
||||
import { saveModelConfig } from "./modelConfigStore.js";
|
||||
import { callProvider, streamProvider } from "./providerClients.js";
|
||||
import { renderPrompt } from "./template.js";
|
||||
import { normalizeUsage } from "./tokens.js";
|
||||
|
||||
const app = express();
|
||||
const port = Number(process.env.PORT ?? 8787);
|
||||
|
||||
app.use(cors());
|
||||
app.use(express.json({ limit: "1mb" }));
|
||||
|
||||
const runSchema = z.object({
|
||||
template: z.string().trim().optional(),
|
||||
systemTemplate: z.string().trim().optional(),
|
||||
userTemplate: z.string().trim().optional(),
|
||||
modelIds: z.array(z.string()).min(1),
|
||||
draws: z.number().int().min(1).max(20),
|
||||
temperature: z.number().min(0).max(2).default(0.8),
|
||||
maxTokens: z.number().int().min(64).max(8192).default(1024)
|
||||
}).superRefine((data, context) => {
|
||||
const userTemplate = data.userTemplate || data.template || "";
|
||||
if (!userTemplate.trim()) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["userTemplate"],
|
||||
message: "User prompt template is required"
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const composePrompt = (systemPrompt: string, userPrompt: string) =>
|
||||
[systemPrompt, userPrompt].filter(Boolean).join("\n\n");
|
||||
|
||||
const modelConfigSchema = z.object({
|
||||
baseUrl: z.string().trim().optional(),
|
||||
model: z.string().trim().optional(),
|
||||
apiKey: z.string().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
clearApiKey: z.boolean().optional()
|
||||
});
|
||||
|
||||
app.get("/api/health", (_request, response) => {
|
||||
response.json({ ok: true });
|
||||
});
|
||||
|
||||
app.get("/api/models", (_request, response) => {
|
||||
response.json({ models: registry() });
|
||||
});
|
||||
|
||||
app.get("/api/model-configs", (_request, response) => {
|
||||
response.json({ models: registry() });
|
||||
});
|
||||
|
||||
app.put("/api/model-configs/:modelId", (request, response) => {
|
||||
const model = findModel(request.params.modelId);
|
||||
if (!model) {
|
||||
response.status(404).json({ error: "Model not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
const parsed = modelConfigSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
response.status(400).json({ error: parsed.error.flatten() });
|
||||
return;
|
||||
}
|
||||
|
||||
saveModelConfig(model.id, parsed.data);
|
||||
const updated = findModel(model.id);
|
||||
response.json({ model: updated });
|
||||
});
|
||||
|
||||
app.post("/api/runs", async (request, response) => {
|
||||
const parsed = runSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
response.status(400).json({ error: parsed.error.flatten() });
|
||||
return;
|
||||
}
|
||||
|
||||
const {
|
||||
systemTemplate = "",
|
||||
userTemplate: explicitUserTemplate,
|
||||
template: legacyTemplate,
|
||||
modelIds,
|
||||
draws,
|
||||
temperature,
|
||||
maxTokens
|
||||
} = parsed.data;
|
||||
const userTemplate = explicitUserTemplate || legacyTemplate || "";
|
||||
const startedAt = new Date();
|
||||
const uniqueModelIds = [...new Set(modelIds)];
|
||||
const models = uniqueModelIds.map((modelId) => findModel(modelId));
|
||||
const missing = uniqueModelIds.filter((modelId, index) => !models[index]);
|
||||
|
||||
if (missing.length) {
|
||||
response.status(400).json({ error: `Unknown model: ${missing.join(", ")}` });
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = models.flatMap((model) =>
|
||||
Array.from({ length: draws }, (_, index) => ({
|
||||
model: model!,
|
||||
draw: index + 1
|
||||
}))
|
||||
);
|
||||
|
||||
const results = await Promise.all(
|
||||
jobs.map(async ({ model, draw }) => {
|
||||
const context = { draw, model, startedAt };
|
||||
const systemPrompt = systemTemplate ? renderPrompt(systemTemplate, context) : "";
|
||||
const userPrompt = renderPrompt(userTemplate, context);
|
||||
const prompt = composePrompt(systemPrompt, userPrompt);
|
||||
const provider = providerForModel(model);
|
||||
const jobStarted = Date.now();
|
||||
|
||||
try {
|
||||
const result = await callProvider({
|
||||
model,
|
||||
provider,
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
temperature,
|
||||
maxTokens
|
||||
});
|
||||
|
||||
return {
|
||||
id: `${model.id}-${draw}-${jobStarted}`,
|
||||
ok: true,
|
||||
draw,
|
||||
prompt,
|
||||
systemPrompt,
|
||||
userPrompt,
|
||||
output: result.output,
|
||||
modelId: model.id,
|
||||
modelLabel: model.label,
|
||||
providerName: model.providerName,
|
||||
rawModel: result.rawModel,
|
||||
durationMs: Date.now() - jobStarted,
|
||||
usage: result.usage
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown provider error";
|
||||
|
||||
return {
|
||||
id: `${model.id}-${draw}-${jobStarted}`,
|
||||
ok: false,
|
||||
draw,
|
||||
prompt,
|
||||
systemPrompt,
|
||||
userPrompt,
|
||||
output: "",
|
||||
error: message,
|
||||
modelId: model.id,
|
||||
modelLabel: model.label,
|
||||
providerName: model.providerName,
|
||||
rawModel: model.model,
|
||||
durationMs: Date.now() - jobStarted,
|
||||
usage: normalizeUsage(prompt, "", null)
|
||||
};
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
response.json({
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: new Date().toISOString(),
|
||||
results
|
||||
});
|
||||
});
|
||||
|
||||
app.post("/api/runs/stream", async (request, response) => {
|
||||
const parsed = runSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
response.status(400).json({ error: parsed.error.flatten() });
|
||||
return;
|
||||
}
|
||||
|
||||
const {
|
||||
systemTemplate = "",
|
||||
userTemplate: explicitUserTemplate,
|
||||
template: legacyTemplate,
|
||||
modelIds,
|
||||
draws,
|
||||
temperature,
|
||||
maxTokens
|
||||
} = parsed.data;
|
||||
const userTemplate = explicitUserTemplate || legacyTemplate || "";
|
||||
const startedAt = new Date();
|
||||
const uniqueModelIds = [...new Set(modelIds)];
|
||||
const models = uniqueModelIds.map((modelId) => findModel(modelId));
|
||||
const missing = uniqueModelIds.filter((modelId, index) => !models[index]);
|
||||
|
||||
if (missing.length) {
|
||||
response.status(400).json({ error: `Unknown model: ${missing.join(", ")}` });
|
||||
return;
|
||||
}
|
||||
|
||||
response.setHeader("Content-Type", "application/x-ndjson; charset=utf-8");
|
||||
response.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
response.setHeader("Connection", "keep-alive");
|
||||
response.setHeader("X-Accel-Buffering", "no");
|
||||
response.flushHeaders();
|
||||
|
||||
const controller = new AbortController();
|
||||
response.on("close", () => {
|
||||
if (!response.writableEnded) {
|
||||
controller.abort();
|
||||
}
|
||||
});
|
||||
|
||||
const writeEvent = (event: Record<string, unknown>) => {
|
||||
if (!response.destroyed && !response.writableEnded) {
|
||||
response.write(`${JSON.stringify(event)}\n`);
|
||||
}
|
||||
};
|
||||
|
||||
const jobs = models.flatMap((model) =>
|
||||
Array.from({ length: draws }, (_, index) => ({
|
||||
model: model!,
|
||||
draw: index + 1
|
||||
}))
|
||||
);
|
||||
|
||||
writeEvent({
|
||||
type: "run_start",
|
||||
startedAt: startedAt.toISOString(),
|
||||
totalJobs: jobs.length
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
jobs.map(async ({ model, draw }, jobIndex) => {
|
||||
const context = { draw, model, startedAt };
|
||||
const systemPrompt = systemTemplate ? renderPrompt(systemTemplate, context) : "";
|
||||
const userPrompt = renderPrompt(userTemplate, context);
|
||||
const prompt = composePrompt(systemPrompt, userPrompt);
|
||||
const provider = providerForModel(model);
|
||||
const jobStarted = Date.now();
|
||||
const id = `${model.id}-${draw}-${startedAt.getTime()}-${jobIndex}`;
|
||||
|
||||
writeEvent({
|
||||
type: "card_start",
|
||||
id,
|
||||
draw,
|
||||
prompt,
|
||||
systemPrompt,
|
||||
userPrompt,
|
||||
modelId: model.id,
|
||||
modelLabel: model.label,
|
||||
providerName: model.providerName,
|
||||
rawModel: model.model
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await streamProvider({
|
||||
model,
|
||||
provider,
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
signal: controller.signal,
|
||||
onDelta: (delta) => {
|
||||
writeEvent({
|
||||
type: "delta",
|
||||
id,
|
||||
channel: delta.channel,
|
||||
text: delta.text
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
writeEvent({
|
||||
type: "card_end",
|
||||
id,
|
||||
ok: true,
|
||||
output: result.output,
|
||||
reasoning: result.reasoning ?? "",
|
||||
rawModel: result.rawModel,
|
||||
durationMs: Date.now() - jobStarted,
|
||||
usage: result.usage
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown provider error";
|
||||
|
||||
writeEvent({
|
||||
type: "card_end",
|
||||
id,
|
||||
ok: false,
|
||||
output: "",
|
||||
reasoning: "",
|
||||
error: message,
|
||||
rawModel: model.model,
|
||||
durationMs: Date.now() - jobStarted,
|
||||
usage: normalizeUsage(prompt, "", null)
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
writeEvent({
|
||||
type: "run_end",
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: new Date().toISOString()
|
||||
});
|
||||
response.end();
|
||||
});
|
||||
|
||||
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const distPath = path.resolve(dirname, "../dist");
|
||||
|
||||
app.use(express.static(distPath));
|
||||
app.use((request, response, next) => {
|
||||
if (request.path.startsWith("/api")) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
const indexPath = path.join(distPath, "index.html");
|
||||
response.sendFile(indexPath, (error) => {
|
||||
if (error) {
|
||||
next();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`Face to API server listening on http://127.0.0.1:${port}`);
|
||||
});
|
||||
121
server/modelConfigStore.ts
Normal file
121
server/modelConfigStore.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export type SavedModelConfig = {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
enabled?: boolean;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
type ConfigFile = {
|
||||
version: 1;
|
||||
models: Record<string, SavedModelConfig>;
|
||||
};
|
||||
|
||||
export type SaveModelConfigInput = {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
enabled?: boolean;
|
||||
clearApiKey?: boolean;
|
||||
};
|
||||
|
||||
const defaultConfigPath = () =>
|
||||
path.resolve(process.cwd(), process.env.MODEL_CONFIG_PATH?.trim() || ".face-to-api/model-configs.json");
|
||||
|
||||
let cache: ConfigFile | undefined;
|
||||
|
||||
const emptyStore = (): ConfigFile => ({ version: 1, models: {} });
|
||||
|
||||
const readStore = (): ConfigFile => {
|
||||
if (cache) {
|
||||
return cache;
|
||||
}
|
||||
|
||||
const filePath = defaultConfigPath();
|
||||
if (!fs.existsSync(filePath)) {
|
||||
cache = emptyStore();
|
||||
return cache;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8")) as ConfigFile;
|
||||
cache = {
|
||||
version: 1,
|
||||
models: parsed.models ?? {}
|
||||
};
|
||||
return cache;
|
||||
} catch {
|
||||
cache = emptyStore();
|
||||
return cache;
|
||||
}
|
||||
};
|
||||
|
||||
const writeStore = (store: ConfigFile) => {
|
||||
const filePath = defaultConfigPath();
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(store, null, 2)}\n`);
|
||||
cache = store;
|
||||
};
|
||||
|
||||
const trimOrUndefined = (value?: string) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
};
|
||||
|
||||
export const getSavedModelConfig = (modelId: string): SavedModelConfig | undefined => {
|
||||
const saved = readStore().models[modelId];
|
||||
return saved ? { ...saved } : undefined;
|
||||
};
|
||||
|
||||
export const saveModelConfig = (modelId: string, input: SaveModelConfigInput) => {
|
||||
const store = readStore();
|
||||
const existing = store.models[modelId] ?? {};
|
||||
const next: SavedModelConfig = { ...existing };
|
||||
const baseUrl = trimOrUndefined(input.baseUrl);
|
||||
const model = trimOrUndefined(input.model);
|
||||
const apiKey = trimOrUndefined(input.apiKey);
|
||||
|
||||
if (input.baseUrl !== undefined) {
|
||||
if (baseUrl) {
|
||||
next.baseUrl = baseUrl;
|
||||
} else {
|
||||
delete next.baseUrl;
|
||||
}
|
||||
}
|
||||
|
||||
if (input.model !== undefined) {
|
||||
if (model) {
|
||||
next.model = model;
|
||||
} else {
|
||||
delete next.model;
|
||||
}
|
||||
}
|
||||
|
||||
if (input.clearApiKey) {
|
||||
delete next.apiKey;
|
||||
} else if (apiKey) {
|
||||
next.apiKey = apiKey;
|
||||
}
|
||||
|
||||
if (input.enabled !== undefined) {
|
||||
next.enabled = input.enabled;
|
||||
}
|
||||
|
||||
next.updatedAt = new Date().toISOString();
|
||||
store.models[modelId] = next;
|
||||
writeStore(store);
|
||||
|
||||
return { ...next };
|
||||
};
|
||||
|
||||
export const maskApiKey = (apiKey?: string) => {
|
||||
if (!apiKey) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const tail = apiKey.slice(-4);
|
||||
return `•••• ${tail}`;
|
||||
};
|
||||
261
server/modelRegistry.ts
Normal file
261
server/modelRegistry.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
export type ProviderKind = "openai-compatible" | "anthropic" | "gemini";
|
||||
|
||||
export type ProviderId =
|
||||
| "deepseek"
|
||||
| "zhipu"
|
||||
| "openai"
|
||||
| "anthropic"
|
||||
| "gemini"
|
||||
| "custom-openai";
|
||||
|
||||
export type ModelProvider = {
|
||||
id: ProviderId;
|
||||
name: string;
|
||||
kind: ProviderKind;
|
||||
baseUrl?: string;
|
||||
apiKeyEnv: string;
|
||||
apiKey?: string;
|
||||
configured: boolean;
|
||||
};
|
||||
|
||||
export type ChatModel = {
|
||||
id: string;
|
||||
label: string;
|
||||
providerId: ProviderId;
|
||||
providerName: string;
|
||||
providerKind: ProviderKind;
|
||||
model: string;
|
||||
baseUrl: string;
|
||||
enabled: boolean;
|
||||
configured: boolean;
|
||||
apiKeyConfigured: boolean;
|
||||
apiKeyPreview: string;
|
||||
apiKeySource: "model" | "provider-env" | "none";
|
||||
configUpdatedAt?: string;
|
||||
accent: "red" | "green" | "blue" | "gold" | "ink";
|
||||
};
|
||||
|
||||
import { getSavedModelConfig, maskApiKey } from "./modelConfigStore.js";
|
||||
|
||||
const env = (key: string, fallback = "") => process.env[key]?.trim() || fallback;
|
||||
|
||||
export const providers = (): ModelProvider[] => [
|
||||
{
|
||||
id: "deepseek",
|
||||
name: "DeepSeek",
|
||||
kind: "openai-compatible",
|
||||
baseUrl: env("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
|
||||
apiKeyEnv: "DEEPSEEK_API_KEY",
|
||||
configured: Boolean(env("DEEPSEEK_API_KEY"))
|
||||
},
|
||||
{
|
||||
id: "zhipu",
|
||||
name: "Zhipu GLM",
|
||||
kind: "openai-compatible",
|
||||
baseUrl: env("ZHIPU_BASE_URL", "https://open.bigmodel.cn/api/paas/v4"),
|
||||
apiKeyEnv: "ZHIPU_API_KEY",
|
||||
configured: Boolean(env("ZHIPU_API_KEY"))
|
||||
},
|
||||
{
|
||||
id: "openai",
|
||||
name: "OpenAI",
|
||||
kind: "openai-compatible",
|
||||
baseUrl: env("OPENAI_BASE_URL", "https://api.openai.com/v1"),
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
configured: Boolean(env("OPENAI_API_KEY"))
|
||||
},
|
||||
{
|
||||
id: "anthropic",
|
||||
name: "Anthropic",
|
||||
kind: "anthropic",
|
||||
baseUrl: "https://api.anthropic.com/v1",
|
||||
apiKeyEnv: "ANTHROPIC_API_KEY",
|
||||
configured: Boolean(env("ANTHROPIC_API_KEY"))
|
||||
},
|
||||
{
|
||||
id: "gemini",
|
||||
name: "Google Gemini",
|
||||
kind: "gemini",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
apiKeyEnv: "GEMINI_API_KEY",
|
||||
configured: Boolean(env("GEMINI_API_KEY"))
|
||||
}
|
||||
];
|
||||
|
||||
const resolveModel = (
|
||||
model: Omit<
|
||||
ChatModel,
|
||||
| "providerName"
|
||||
| "providerKind"
|
||||
| "configured"
|
||||
| "baseUrl"
|
||||
| "enabled"
|
||||
| "apiKeyConfigured"
|
||||
| "apiKeyPreview"
|
||||
| "apiKeySource"
|
||||
>,
|
||||
provider: ModelProvider
|
||||
): ChatModel => {
|
||||
const saved = getSavedModelConfig(model.id);
|
||||
const envApiKey = env(provider.apiKeyEnv);
|
||||
const apiKey = saved?.apiKey || envApiKey;
|
||||
const apiKeySource = saved?.apiKey ? "model" : envApiKey ? "provider-env" : "none";
|
||||
|
||||
return {
|
||||
...model,
|
||||
providerName: provider.name,
|
||||
providerKind: provider.kind,
|
||||
model: saved?.model || model.model,
|
||||
baseUrl: saved?.baseUrl || provider.baseUrl || "",
|
||||
enabled: saved?.enabled ?? true,
|
||||
configured: Boolean(apiKey),
|
||||
apiKeyConfigured: Boolean(apiKey),
|
||||
apiKeyPreview: maskApiKey(apiKey),
|
||||
apiKeySource,
|
||||
configUpdatedAt: saved?.updatedAt
|
||||
};
|
||||
};
|
||||
|
||||
export const registry = (): ChatModel[] => {
|
||||
const providerMap = new Map(providers().map((provider) => [provider.id, provider]));
|
||||
const baseModels: Array<
|
||||
Omit<
|
||||
ChatModel,
|
||||
| "providerName"
|
||||
| "providerKind"
|
||||
| "configured"
|
||||
| "baseUrl"
|
||||
| "enabled"
|
||||
| "apiKeyConfigured"
|
||||
| "apiKeyPreview"
|
||||
| "apiKeySource"
|
||||
>
|
||||
> = [
|
||||
{
|
||||
id: "deepseek-v4-pro",
|
||||
label: "DeepSeekV4-pro",
|
||||
providerId: "deepseek",
|
||||
model: env("DEEPSEEK_MODEL_V4_PRO", "DeepSeekV4-pro"),
|
||||
accent: "red"
|
||||
},
|
||||
{
|
||||
id: "deepseek-v4-flash",
|
||||
label: "DeepSeekV4-flash",
|
||||
providerId: "deepseek",
|
||||
model: env("DEEPSEEK_MODEL_V4_FLASH", "DeepSeekV4-flash"),
|
||||
accent: "red"
|
||||
},
|
||||
{
|
||||
id: "glm-5-1",
|
||||
label: "GLM5.1",
|
||||
providerId: "zhipu",
|
||||
model: env("ZHIPU_MODEL_GLM_5_1", "GLM5.1"),
|
||||
accent: "green"
|
||||
},
|
||||
{
|
||||
id: "gpt-5-5",
|
||||
label: "GPT-5.5",
|
||||
providerId: "openai",
|
||||
model: env("OPENAI_MODEL_GPT_5_5", "gpt-5.5"),
|
||||
accent: "blue"
|
||||
},
|
||||
{
|
||||
id: "claude-opus-4-7",
|
||||
label: "Claude Opus 4.7",
|
||||
providerId: "anthropic",
|
||||
model: env("ANTHROPIC_MODEL_CLAUDE_OPUS_4_7", "claude-opus-4.7"),
|
||||
accent: "gold"
|
||||
},
|
||||
{
|
||||
id: "gemini-3-1-pro",
|
||||
label: "Gemini-3.1-pro",
|
||||
providerId: "gemini",
|
||||
model: env("GEMINI_MODEL_3_1_PRO", "Gemini-3.1-pro"),
|
||||
accent: "ink"
|
||||
},
|
||||
{
|
||||
id: "gemini-3-5-flash",
|
||||
label: "gemini3.5-flash",
|
||||
providerId: "gemini",
|
||||
model: env("GEMINI_MODEL_3_5_FLASH", "gemini3.5-flash"),
|
||||
accent: "ink"
|
||||
}
|
||||
];
|
||||
|
||||
const models = baseModels.map((model) => {
|
||||
const provider = providerMap.get(model.providerId);
|
||||
if (!provider) {
|
||||
throw new Error(`Missing provider for ${model.id}`);
|
||||
}
|
||||
|
||||
return resolveModel(model, provider);
|
||||
});
|
||||
|
||||
const customIds = env("CUSTOM_OPENAI_MODEL_IDS")
|
||||
.split(",")
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
if (customIds.length && env("CUSTOM_OPENAI_BASE_URL")) {
|
||||
const customConfigured = Boolean(env("CUSTOM_OPENAI_API_KEY"));
|
||||
models.push(
|
||||
...customIds.map((modelId) =>
|
||||
resolveModel(
|
||||
{
|
||||
id: `custom-${modelId.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`,
|
||||
label: modelId,
|
||||
providerId: "custom-openai" as const,
|
||||
model: modelId,
|
||||
configUpdatedAt: undefined,
|
||||
accent: "blue" as const
|
||||
},
|
||||
{
|
||||
id: "custom-openai",
|
||||
name: env("CUSTOM_OPENAI_NAME", "Custom OpenAI-compatible"),
|
||||
kind: "openai-compatible",
|
||||
baseUrl: env("CUSTOM_OPENAI_BASE_URL"),
|
||||
apiKeyEnv: "CUSTOM_OPENAI_API_KEY",
|
||||
configured: customConfigured
|
||||
}
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return models;
|
||||
};
|
||||
|
||||
export const findModel = (modelId: string) => registry().find((model) => model.id === modelId);
|
||||
|
||||
export const providerForModel = (model: ChatModel): ModelProvider => {
|
||||
const saved = getSavedModelConfig(model.id);
|
||||
|
||||
if (model.providerId === "custom-openai") {
|
||||
const envApiKey = env("CUSTOM_OPENAI_API_KEY");
|
||||
return {
|
||||
id: "custom-openai",
|
||||
name: env("CUSTOM_OPENAI_NAME", "Custom OpenAI-compatible"),
|
||||
kind: "openai-compatible",
|
||||
baseUrl: saved?.baseUrl || env("CUSTOM_OPENAI_BASE_URL"),
|
||||
apiKeyEnv: "CUSTOM_OPENAI_API_KEY",
|
||||
apiKey: saved?.apiKey,
|
||||
configured: Boolean(saved?.apiKey || envApiKey)
|
||||
};
|
||||
}
|
||||
|
||||
const provider = providers().find((item) => item.id === model.providerId);
|
||||
if (!provider) {
|
||||
throw new Error(`Provider not found for ${model.label}`);
|
||||
}
|
||||
|
||||
const envApiKey = env(provider.apiKeyEnv);
|
||||
|
||||
return {
|
||||
...provider,
|
||||
baseUrl: saved?.baseUrl || provider.baseUrl,
|
||||
apiKey: saved?.apiKey,
|
||||
configured: Boolean(saved?.apiKey || envApiKey)
|
||||
};
|
||||
};
|
||||
|
||||
export const apiKeyForProvider = (provider: ModelProvider) => provider.apiKey || env(provider.apiKeyEnv);
|
||||
618
server/providerClients.ts
Normal file
618
server/providerClients.ts
Normal file
@@ -0,0 +1,618 @@
|
||||
import type { ChatModel, ModelProvider } from "./modelRegistry.js";
|
||||
import { apiKeyForProvider } from "./modelRegistry.js";
|
||||
import { normalizeUsage, type Usage } from "./tokens.js";
|
||||
|
||||
export type ChatRequest = {
|
||||
model: ChatModel;
|
||||
provider: ModelProvider;
|
||||
systemPrompt?: string;
|
||||
prompt: string;
|
||||
temperature: number;
|
||||
maxTokens: number;
|
||||
};
|
||||
|
||||
export type ChatResponse = {
|
||||
output: string;
|
||||
reasoning?: string;
|
||||
usage: Usage;
|
||||
rawModel: string;
|
||||
};
|
||||
|
||||
export type StreamDelta = {
|
||||
channel: "reasoning" | "output";
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type ChatStreamRequest = ChatRequest & {
|
||||
onDelta: (delta: StreamDelta) => void;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
type OpenAIResponse = {
|
||||
choices?: Array<{ message?: { content?: string }; text?: string }>;
|
||||
usage?: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type OpenAIStreamDelta = {
|
||||
content?: string;
|
||||
reasoning?: string;
|
||||
reasoning_content?: string;
|
||||
thinking?: string;
|
||||
reasoning_details?: Array<{ text?: string; content?: string }>;
|
||||
};
|
||||
|
||||
type OpenAIStreamChunk = {
|
||||
choices?: Array<{
|
||||
delta?: OpenAIStreamDelta;
|
||||
}>;
|
||||
usage?: OpenAIResponse["usage"];
|
||||
};
|
||||
|
||||
type AnthropicResponse = {
|
||||
content?: Array<{ type?: string; text?: string }>;
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type AnthropicStreamEvent = {
|
||||
type?: string;
|
||||
content_block?: {
|
||||
type?: string;
|
||||
text?: string;
|
||||
thinking?: string;
|
||||
};
|
||||
delta?: {
|
||||
type?: string;
|
||||
text?: string;
|
||||
thinking?: string;
|
||||
};
|
||||
message?: {
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
};
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type GeminiResponse = {
|
||||
candidates?: Array<{
|
||||
content?: {
|
||||
parts?: Array<{ text?: string }>;
|
||||
};
|
||||
}>;
|
||||
usageMetadata?: {
|
||||
promptTokenCount?: number;
|
||||
candidatesTokenCount?: number;
|
||||
totalTokenCount?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type GeminiStreamChunk = GeminiResponse;
|
||||
|
||||
const readErrorBody = async (response: Response) => {
|
||||
const text = await response.text();
|
||||
if (!text) {
|
||||
return response.statusText;
|
||||
}
|
||||
|
||||
try {
|
||||
const json = JSON.parse(text) as { error?: { message?: string }; message?: string };
|
||||
return json.error?.message ?? json.message ?? text;
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
};
|
||||
|
||||
const postJson = async <T>(
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
body: unknown
|
||||
): Promise<T> => {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...headers
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(await readErrorBody(response));
|
||||
}
|
||||
|
||||
return (await response.json()) as T;
|
||||
};
|
||||
|
||||
const postJsonStream = async (
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
body: unknown,
|
||||
signal?: AbortSignal
|
||||
) => {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...headers
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(await readErrorBody(response));
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("Provider returned an empty stream");
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
const readSse = async (
|
||||
response: Response,
|
||||
onEvent: (event: { event?: string; data: string }) => void
|
||||
) => {
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error("Provider stream is not readable");
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
let eventName = "";
|
||||
let dataLines: string[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (!dataLines.length) {
|
||||
eventName = "";
|
||||
return;
|
||||
}
|
||||
|
||||
onEvent({
|
||||
event: eventName || undefined,
|
||||
data: dataLines.join("\n")
|
||||
});
|
||||
eventName = "";
|
||||
dataLines = [];
|
||||
};
|
||||
|
||||
const handleLine = (rawLine: string) => {
|
||||
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
||||
if (!line) {
|
||||
flush();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.startsWith("event:")) {
|
||||
eventName = line.slice(6).trim();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.startsWith("data:")) {
|
||||
dataLines.push(line.slice(5).trimStart());
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.trim().startsWith("{")) {
|
||||
onEvent({ data: line.trim() });
|
||||
}
|
||||
};
|
||||
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() ?? "";
|
||||
for (const line of lines) {
|
||||
handleLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
buffer += decoder.decode();
|
||||
if (buffer) {
|
||||
handleLine(buffer);
|
||||
}
|
||||
flush();
|
||||
};
|
||||
|
||||
const endpoint = (provider: ModelProvider, path: string) => {
|
||||
const baseUrl = provider.baseUrl?.replace(/\/$/, "");
|
||||
if (!baseUrl) {
|
||||
throw new Error(`${provider.name} base URL is not configured`);
|
||||
}
|
||||
|
||||
return `${baseUrl}${path}`;
|
||||
};
|
||||
|
||||
const composePrompt = (systemPrompt: string | undefined, prompt: string) =>
|
||||
[systemPrompt, prompt].filter(Boolean).join("\n\n");
|
||||
|
||||
const openAICompatibleMessages = (systemPrompt: string | undefined, prompt: string) => [
|
||||
...(systemPrompt ? [{ role: "system", content: systemPrompt }] : []),
|
||||
{ role: "user", content: prompt }
|
||||
];
|
||||
|
||||
export const callOpenAICompatible = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens
|
||||
}: ChatRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
const json = await postJson<OpenAIResponse>(
|
||||
endpoint(provider, "/chat/completions"),
|
||||
{ authorization: `Bearer ${apiKey}` },
|
||||
{
|
||||
model: model.model,
|
||||
messages: openAICompatibleMessages(systemPrompt, prompt),
|
||||
temperature,
|
||||
max_tokens: maxTokens
|
||||
}
|
||||
);
|
||||
|
||||
const output = json.choices?.[0]?.message?.content ?? json.choices?.[0]?.text ?? "";
|
||||
const usage = normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens: json.usage?.prompt_tokens ?? json.usage?.input_tokens,
|
||||
outputTokens: json.usage?.completion_tokens ?? json.usage?.output_tokens,
|
||||
totalTokens: json.usage?.total_tokens,
|
||||
estimated: !json.usage
|
||||
});
|
||||
|
||||
return { output, usage, rawModel: model.model };
|
||||
};
|
||||
|
||||
const reasoningTextFromDelta = (delta?: OpenAIStreamDelta) => {
|
||||
const details = delta?.reasoning_details
|
||||
?.map((detail) => detail.text ?? detail.content ?? "")
|
||||
.join("");
|
||||
return delta?.reasoning_content ?? delta?.reasoning ?? delta?.thinking ?? details ?? "";
|
||||
};
|
||||
|
||||
export const streamOpenAICompatible = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
onDelta,
|
||||
signal
|
||||
}: ChatStreamRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
let output = "";
|
||||
let reasoning = "";
|
||||
let usage: OpenAIResponse["usage"] | undefined;
|
||||
|
||||
const response = await postJsonStream(
|
||||
endpoint(provider, "/chat/completions"),
|
||||
{ authorization: `Bearer ${apiKey}` },
|
||||
{
|
||||
model: model.model,
|
||||
messages: openAICompatibleMessages(systemPrompt, prompt),
|
||||
temperature,
|
||||
max_tokens: maxTokens,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true }
|
||||
},
|
||||
signal
|
||||
);
|
||||
|
||||
await readSse(response, ({ data }) => {
|
||||
if (data === "[DONE]") {
|
||||
return;
|
||||
}
|
||||
|
||||
const chunk = JSON.parse(data) as OpenAIStreamChunk;
|
||||
usage = chunk.usage ?? usage;
|
||||
const delta = chunk.choices?.[0]?.delta;
|
||||
const reasoningDelta = reasoningTextFromDelta(delta);
|
||||
const outputDelta = delta?.content ?? "";
|
||||
|
||||
if (reasoningDelta) {
|
||||
reasoning += reasoningDelta;
|
||||
onDelta({ channel: "reasoning", text: reasoningDelta });
|
||||
}
|
||||
|
||||
if (outputDelta) {
|
||||
output += outputDelta;
|
||||
onDelta({ channel: "output", text: outputDelta });
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
output,
|
||||
reasoning,
|
||||
usage: normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens: usage?.prompt_tokens ?? usage?.input_tokens,
|
||||
outputTokens: usage?.completion_tokens ?? usage?.output_tokens,
|
||||
totalTokens: usage?.total_tokens,
|
||||
estimated: !usage
|
||||
}),
|
||||
rawModel: model.model
|
||||
};
|
||||
};
|
||||
|
||||
export const callAnthropic = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens
|
||||
}: ChatRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
const json = await postJson<AnthropicResponse>(
|
||||
endpoint(provider, "/messages"),
|
||||
{
|
||||
"x-api-key": apiKey,
|
||||
"anthropic-version": "2023-06-01"
|
||||
},
|
||||
{
|
||||
model: model.model,
|
||||
system: systemPrompt || undefined,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature,
|
||||
max_tokens: maxTokens
|
||||
}
|
||||
);
|
||||
|
||||
const output = json.content?.map((part) => part.text ?? "").join("\n").trim() ?? "";
|
||||
const usage = normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens: json.usage?.input_tokens,
|
||||
outputTokens: json.usage?.output_tokens,
|
||||
estimated: !json.usage
|
||||
});
|
||||
|
||||
return { output, usage, rawModel: model.model };
|
||||
};
|
||||
|
||||
export const streamAnthropic = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
onDelta,
|
||||
signal
|
||||
}: ChatStreamRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
let output = "";
|
||||
let reasoning = "";
|
||||
let inputTokens: number | undefined;
|
||||
let outputTokens: number | undefined;
|
||||
|
||||
const response = await postJsonStream(
|
||||
endpoint(provider, "/messages"),
|
||||
{
|
||||
"x-api-key": apiKey,
|
||||
"anthropic-version": "2023-06-01"
|
||||
},
|
||||
{
|
||||
model: model.model,
|
||||
system: systemPrompt || undefined,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature,
|
||||
max_tokens: maxTokens,
|
||||
stream: true
|
||||
},
|
||||
signal
|
||||
);
|
||||
|
||||
await readSse(response, ({ data }) => {
|
||||
const event = JSON.parse(data) as AnthropicStreamEvent;
|
||||
inputTokens = event.message?.usage?.input_tokens ?? event.usage?.input_tokens ?? inputTokens;
|
||||
outputTokens = event.message?.usage?.output_tokens ?? event.usage?.output_tokens ?? outputTokens;
|
||||
|
||||
const startText =
|
||||
event.type === "content_block_start" && event.content_block?.type === "text"
|
||||
? event.content_block.text ?? ""
|
||||
: "";
|
||||
const startThinking =
|
||||
event.type === "content_block_start" && event.content_block?.type === "thinking"
|
||||
? event.content_block.thinking ?? ""
|
||||
: "";
|
||||
const outputDelta = event.delta?.type === "text_delta" ? event.delta.text ?? "" : "";
|
||||
const reasoningDelta =
|
||||
event.delta?.type === "thinking_delta" ? event.delta.thinking ?? "" : "";
|
||||
|
||||
if (startThinking || reasoningDelta) {
|
||||
const text = startThinking || reasoningDelta;
|
||||
reasoning += text;
|
||||
onDelta({ channel: "reasoning", text });
|
||||
}
|
||||
|
||||
if (startText || outputDelta) {
|
||||
const text = startText || outputDelta;
|
||||
output += text;
|
||||
onDelta({ channel: "output", text });
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
output,
|
||||
reasoning,
|
||||
usage: normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
estimated: inputTokens === undefined && outputTokens === undefined
|
||||
}),
|
||||
rawModel: model.model
|
||||
};
|
||||
};
|
||||
|
||||
export const callGemini = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens
|
||||
}: ChatRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
const baseUrl = provider.baseUrl?.replace(/\/$/, "");
|
||||
const url = `${baseUrl}/models/${encodeURIComponent(model.model)}:generateContent?key=${encodeURIComponent(apiKey)}`;
|
||||
const json = await postJson<GeminiResponse>(
|
||||
url,
|
||||
{},
|
||||
{
|
||||
systemInstruction: systemPrompt ? { parts: [{ text: systemPrompt }] } : undefined,
|
||||
contents: [{ role: "user", parts: [{ text: prompt }] }],
|
||||
generationConfig: {
|
||||
temperature,
|
||||
maxOutputTokens: maxTokens
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const output =
|
||||
json.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n").trim() ?? "";
|
||||
const usage = normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens: json.usageMetadata?.promptTokenCount,
|
||||
outputTokens: json.usageMetadata?.candidatesTokenCount,
|
||||
totalTokens: json.usageMetadata?.totalTokenCount,
|
||||
estimated: !json.usageMetadata
|
||||
});
|
||||
|
||||
return { output, usage, rawModel: model.model };
|
||||
};
|
||||
|
||||
export const streamGemini = async ({
|
||||
provider,
|
||||
model,
|
||||
systemPrompt,
|
||||
prompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
onDelta,
|
||||
signal
|
||||
}: ChatStreamRequest): Promise<ChatResponse> => {
|
||||
const apiKey = apiKeyForProvider(provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`${provider.name} API key is not configured (${provider.apiKeyEnv})`);
|
||||
}
|
||||
|
||||
const baseUrl = provider.baseUrl?.replace(/\/$/, "");
|
||||
const url = `${baseUrl}/models/${encodeURIComponent(model.model)}:streamGenerateContent?alt=sse&key=${encodeURIComponent(apiKey)}`;
|
||||
let output = "";
|
||||
let reasoning = "";
|
||||
let usage: GeminiResponse["usageMetadata"] | undefined;
|
||||
|
||||
const response = await postJsonStream(
|
||||
url,
|
||||
{},
|
||||
{
|
||||
systemInstruction: systemPrompt ? { parts: [{ text: systemPrompt }] } : undefined,
|
||||
contents: [{ role: "user", parts: [{ text: prompt }] }],
|
||||
generationConfig: {
|
||||
temperature,
|
||||
maxOutputTokens: maxTokens
|
||||
}
|
||||
},
|
||||
signal
|
||||
);
|
||||
|
||||
await readSse(response, ({ data }) => {
|
||||
const chunk = JSON.parse(data) as GeminiStreamChunk;
|
||||
usage = chunk.usageMetadata ?? usage;
|
||||
const parts = chunk.candidates?.[0]?.content?.parts ?? [];
|
||||
|
||||
for (const part of parts) {
|
||||
const text = part.text ?? "";
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ("thought" in part && part.thought === true) {
|
||||
reasoning += text;
|
||||
onDelta({ channel: "reasoning", text });
|
||||
} else {
|
||||
output += text;
|
||||
onDelta({ channel: "output", text });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
output,
|
||||
reasoning,
|
||||
usage: normalizeUsage(composePrompt(systemPrompt, prompt), output, {
|
||||
inputTokens: usage?.promptTokenCount,
|
||||
outputTokens: usage?.candidatesTokenCount,
|
||||
totalTokens: usage?.totalTokenCount,
|
||||
estimated: !usage
|
||||
}),
|
||||
rawModel: model.model
|
||||
};
|
||||
};
|
||||
|
||||
export const callProvider = async (request: ChatRequest): Promise<ChatResponse> => {
|
||||
switch (request.provider.kind) {
|
||||
case "openai-compatible":
|
||||
return callOpenAICompatible(request);
|
||||
case "anthropic":
|
||||
return callAnthropic(request);
|
||||
case "gemini":
|
||||
return callGemini(request);
|
||||
default: {
|
||||
const provider: never = request.provider.kind;
|
||||
throw new Error(`Unsupported provider kind: ${provider}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const streamProvider = async (request: ChatStreamRequest): Promise<ChatResponse> => {
|
||||
switch (request.provider.kind) {
|
||||
case "openai-compatible":
|
||||
return streamOpenAICompatible(request);
|
||||
case "anthropic":
|
||||
return streamAnthropic(request);
|
||||
case "gemini":
|
||||
return streamGemini(request);
|
||||
default: {
|
||||
const provider: never = request.provider.kind;
|
||||
throw new Error(`Unsupported provider kind: ${provider}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
22
server/template.ts
Normal file
22
server/template.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import type { ChatModel } from "./modelRegistry.js";
|
||||
|
||||
export type TemplateContext = {
|
||||
draw: number;
|
||||
model: ChatModel;
|
||||
startedAt: Date;
|
||||
};
|
||||
|
||||
export const renderPrompt = (template: string, context: TemplateContext) => {
|
||||
const values: Record<string, string> = {
|
||||
draw: String(context.draw),
|
||||
model: context.model.label,
|
||||
provider: context.model.providerName,
|
||||
now: context.startedAt.toLocaleString("zh-CN", { hour12: false }),
|
||||
timestamp: context.startedAt.toISOString()
|
||||
};
|
||||
|
||||
return template.replace(/\{\{\s*([a-zA-Z0-9_-]+)\s*\}\}/g, (match, key: string) => {
|
||||
return values[key] ?? match;
|
||||
});
|
||||
};
|
||||
|
||||
32
server/tokens.ts
Normal file
32
server/tokens.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
export type Usage = {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
estimated: boolean;
|
||||
};
|
||||
|
||||
export const estimateTokens = (text: string) => {
|
||||
if (!text.trim()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const ascii = text.match(/[\x00-\x7F]/g)?.length ?? 0;
|
||||
const nonAscii = text.length - ascii;
|
||||
return Math.max(1, Math.ceil(ascii / 4 + nonAscii / 1.7));
|
||||
};
|
||||
|
||||
export const normalizeUsage = (
|
||||
prompt: string,
|
||||
output: string,
|
||||
usage?: Partial<Usage> | null
|
||||
): Usage => {
|
||||
const inputTokens = usage?.inputTokens ?? estimateTokens(prompt);
|
||||
const outputTokens = usage?.outputTokens ?? estimateTokens(output);
|
||||
|
||||
return {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: usage?.totalTokens ?? inputTokens + outputTokens,
|
||||
estimated: usage?.estimated ?? !usage
|
||||
};
|
||||
};
|
||||
Reference in New Issue
Block a user