feat: 增加 siliconflow fim 的支持 (#63 )

siliconflow 已支持标准格式的 FIM 补全,特此 PR
Fix: Repair the logic for obtaining the configuration file path. (#60 )
2024-10-11 18:49:08 -07:00 · 2024-09-25 00:11:30 -07:00 · 2024-09-25 00:10:50 -07:00 · 2024-09-08 18:24:12 -07:00 · 2024-07-29 00:33:11 -07:00 · 2024-07-26 18:41:27 +08:00
5 changed files with 216 additions and 70 deletions
--- a/15
+++ b/15
@ -1,17 +1,22 @@
 FROM golang:alpine AS builder

 WORKDIR /app
+COPY . .

-ADD . .
+ENV GO111MODULE=on GOPROXY=https://goproxy.cn,direct
+RUN go mod download

-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o override
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o override

 FROM alpine:latest

-COPY --from=builder /app/override /usr/local/bin/override
+RUN apk --no-cache add ca-certificates
+
+COPY --from=builder /app/override /usr/local/bin/
+COPY config.json.example /app/config.json

 WORKDIR /app
-
-ENTRYPOINT ["/usr/local/bin/override"]
+VOLUME /app

 EXPOSE 8181
+CMD ["override"]
--- a/README.md
+++ b/README.md
@ -6,7 +6,7 @@

 ```json
    "github.copilot.advanced": {
-        "debug.overrideCAPIUrl": "http://127.0.0.1:8181",
+        "debug.overrideCAPIUrl": "http://127.0.0.1:8181/v1",
        "debug.overrideProxyUrl": "http://127.0.0.1:8181",
        "debug.chatOverrideProxyUrl": "http://127.0.0.1:8181/v1/chat/completions",
        "authProvider": "github-enterprise"
@ -28,42 +28,71 @@

 ```json
 {
-  "bind": "127.0.0.1:8181",
-  "proxy_url": "",
-  "timeout": 600,
-  "codex_api_base": "https://api-proxy.oaipro.com/v1",
-  "codex_api_key": "sk-xxx",
-  "codex_api_organization": "",
-  "codex_api_project": "",
-  "code_instruct_model": "gpt-3.5-turbo-instruct",
-  "chat_api_base": "https://api-proxy.oaipro.com/v1",
-  "chat_api_key": "sk-xxx",
-  "chat_api_organization": "",
-  "chat_api_project": "",
-  "chat_max_tokens": 4096,
-  "chat_model_default": "gpt-4o",
-  "chat_model_map": {},
-  "auth_token": ""
+ "bind": "127.0.0.1:8181",
+ "proxy_url": "",
+ "timeout": 600,
+ "codex_api_base": "https://api-proxy.oaipro.com/v1",
+ "codex_api_key": "sk-xxx",
+ "codex_api_organization": "",
+ "codex_api_project": "",
+ "codex_max_tokens": 500,
+ "code_instruct_model": "gpt-3.5-turbo-instruct",
+ "chat_api_base": "https://api-proxy.oaipro.com/v1",
+ "chat_api_key": "sk-xxx",
+ "chat_api_organization": "",
+ "chat_api_project": "",
+ "chat_max_tokens": 4096,
+ "chat_model_default": "gpt-4o",
+ "chat_model_map": {},
+ "chat_locale": "zh_CN",
+ "auth_token": ""
 }
+
 ```

 `organization` 和 `project` 除非你有，且知道怎么回事再填。

 `chat_model_map` 是个模型映射的字典。会将请求的模型映射到你想要的，如果不存在映射，则使用 `chat_model_default` 。

+`codex_max_tokens` 可以设置为你希望的最大Token数，你设置的时候最好知道自己在做什么。代码生成通常使用 `500` 即可。
+
 `chat_max_tokens` 可以设置为你希望的最大Token数，你设置的时候最好知道自己在做什么。`gpt-4o` 输出最大为 `4096`

 可以通过 `OVERRIDE_` + 大写配置项作为环境变量，可以覆盖 `config.json` 中的值。例如：`OVERRIDE_CODEX_API_KEY=sk-xxxx`

+### DeepSeek Coder 设置
+如果你希望使用 DeepSeek Coder FIM 来进行代码补全，着重修改以下配置：
+
+```json
+  "codex_api_base": "https://api.deepseek.com/beta/v1",
+  "codex_api_key": "sk-xxx",
+  "code_instruct_model": "deepseek-coder",
+```
+
+### Siliconflow 设置
+如果你希望使用 Siliconflow FIM 模型来进行代码补全，着重修改以下配置：
+
+```json
+  "codex_api_base": "https://api.siliconflow.cn/v1",
+  "codex_api_key": "sk-xxx,sk-xxx2,sk-xxx3...",
+  "code_instruct_model": "Qwen/Qwen2.5-Coder-7B-Instruct",
+```
+
+截至目前，Siliconflow 共有三个模型支持 FIM。分别是 `Qwen/Qwen2.5-Coder-7B-Instruct`、`deepseek-ai/DeepSeek-Coder-V2-Instruct` 、`deepseek-ai/DeepSeek-V2.5`。其中 `Qwen/Qwen2.5-Coder-7B-Instruct` 是免费模型，另外两个是收费模型。
+
+如果你有很多 Siliconflow API Key, 可以以英文逗号分隔填入`codex_api_key`字段, 这样可以很好的避免Siliconflow官方的 TPM RateLimit 对你编码速度影响(尤其使用收费模型时，用户级别较低，TPM 最低只有 10k)。
+
+
+
 ### 本地大模型设置
 1. 安装ollama 
 2. ollama run stable-code:code  (这个模型较小，大部分显卡都能跑)  
 或者你的显卡比较高安装这个：ollama run stable-code:3b-code-fp16
 3. 修改config.json里面的codex_api_base为http://localhost:11434/v1/chat
 4. 修改code_instruct_model为你的模型名称，stable-code:code或者stable-code:3b-code-fp16
-4. 剩下的就按照正常流程走即可。
-5. 如果调不通，请确认http://localhost:11434/v1/chat可用。
-
+5. 剩下的就按照正常流程走即可。
+6. 如果调不通，请确认http://localhost:11434/v1/chat可用。
+        
 ### 重要说明
 `codex_max_tokens` 工作并不完美，已经移除。**JetBrains IDE 完美工作**，`VSCode` 需要执行以下脚本Patch之：

--- a/config.json.example
+++ b/config.json.example
@ -6,6 +6,8 @@
  "codex_api_key": "sk-xxx",
  "codex_api_organization": "",
  "codex_api_project": "",
+  "codex_max_tokens": 500,
+  "code_instruct_model": "gpt-3.5-turbo-instruct",
  "chat_api_base": "https://api-proxy.oaipro.com/v1",
  "chat_api_key": "sk-xxx",
  "chat_api_organization": "",
@ -13,5 +15,6 @@
  "chat_max_tokens": 4096,
  "chat_model_default": "gpt-4o",
  "chat_model_map": {},
-  "chat_locale": "zh_CN"
-}
+  "chat_locale": "zh_CN",
+  "auth_token": ""
+}
--- a/localModel.go
+++ b/localModel.go
@ -1 +0,0 @@
-package main
--- a/main.go
+++ b/main.go
@ -19,12 +19,17 @@ import (
 	"strconv"
 	"strings"
 	"time"
+    "math/rand"
 )

 const DefaultInstructModel = "gpt-3.5-turbo-instruct"

 const StableCodeModelPrefix = "stable-code"

+const DeepSeekCoderModel = "deepseek-coder"
+
+var SiliconflowModels = []string{"deepseek-ai/DeepSeek-V2.5", "deepseek-ai/DeepSeek-Coder-V2-Instruct", "Qwen/Qwen2.5-Coder-7B-Instruct"}
+
 type config struct {
 	Bind                 string            `json:"bind"`
 	ProxyUrl             string            `json:"proxy_url"`
@ -33,6 +38,7 @@ type config struct {
 	CodexApiKey          string            `json:"codex_api_key"`
 	CodexApiOrganization string            `json:"codex_api_organization"`
 	CodexApiProject      string            `json:"codex_api_project"`
+	CodexMaxTokens       int               `json:"codex_max_tokens"`
 	CodeInstructModel    string            `json:"code_instruct_model"`
 	ChatApiBase          string            `json:"chat_api_base"`
 	ChatApiKey           string            `json:"chat_api_key"`
@ -46,7 +52,13 @@ type config struct {
 }

 func readConfig() *config {
-	content, err := os.ReadFile("config.json")
+	var configPath string
+	if len(os.Args) > 1 {
+		configPath = os.Args[1]
+	} else {
+		configPath = "config.json"
+	}
+	content, err := os.ReadFile(configPath)
 	if nil != err {
 		log.Fatal(err)
 	}
@ -97,6 +109,14 @@ func readConfig() *config {
 		_cfg.CodeInstructModel = DefaultInstructModel
 	}

+	if _cfg.CodexMaxTokens == 0 {
+		_cfg.CodexMaxTokens = 500
+	}
+
+	if _cfg.ChatMaxTokens == 0 {
+		_cfg.ChatMaxTokens = 4096
+	}
+
 	return _cfg
 }

@ -173,6 +193,7 @@ func AuthMiddleware(authToken string) gin.HandlerFunc {
 func (s *ProxyService) InitRoutes(e *gin.Engine) {
 	e.GET("/_ping", s.pong)
 	e.GET("/models", s.models)
+	e.GET("/v1/models", s.models)
 	authToken := s.cfg.AuthToken // replace with your dynamic value as needed
 	if authToken != "" {
 		// 鉴权
@ -180,10 +201,16 @@ func (s *ProxyService) InitRoutes(e *gin.Engine) {
 		{
 			v1.POST("/chat/completions", s.completions)
 			v1.POST("/engines/copilot-codex/completions", s.codeCompletions)
+
+			v1.POST("/v1/chat/completions", s.completions)
+			v1.POST("/v1/engines/copilot-codex/completions", s.codeCompletions)
 		}
 	} else {
 		e.POST("/v1/chat/completions", s.completions)
 		e.POST("/v1/engines/copilot-codex/completions", s.codeCompletions)
+
+		e.POST("/v1/v1/chat/completions", s.completions)
+		e.POST("/v1/v1/engines/copilot-codex/completions", s.codeCompletions)
 	}
 }

@ -206,9 +233,12 @@ func (s *ProxyService) models(c *gin.Context) {
 		"data": []gin.H{
 			{
 				"capabilities": gin.H{
-					"family": "gpt-3.5-turbo",
-					"object": "model_capabilities",
-					"type":   "chat",
+					"family":    "gpt-3.5-turbo",
+					"limits":    gin.H{"max_prompt_tokens": 12288},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"tool_calls": true},
+					"tokenizer": "cl100k_base",
+					"type":      "chat",
 				},
 				"id":      "gpt-3.5-turbo",
 				"name":    "GPT 3.5 Turbo",
@ -217,20 +247,26 @@ func (s *ProxyService) models(c *gin.Context) {
 			},
 			{
 				"capabilities": gin.H{
-					"family": "gpt-3.5-turbo",
-					"object": "model_capabilities",
-					"type":   "chat",
+					"family":    "gpt-3.5-turbo",
+					"limits":    gin.H{"max_prompt_tokens": 12288},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"tool_calls": true},
+					"tokenizer": "cl100k_base",
+					"type":      "chat",
 				},
 				"id":      "gpt-3.5-turbo-0613",
-				"name":    "GPT 3.5 Turbo (2023-06-13)",
+				"name":    "GPT 3.5 Turbo",
 				"object":  "model",
 				"version": "gpt-3.5-turbo-0613",
 			},
 			{
 				"capabilities": gin.H{
-					"family": "gpt-4",
-					"object": "model_capabilities",
-					"type":   "chat",
+					"family":    "gpt-4",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"tool_calls": true},
+					"tokenizer": "cl100k_base",
+					"type":      "chat",
 				},
 				"id":      "gpt-4",
 				"name":    "GPT 4",
@ -239,31 +275,81 @@ func (s *ProxyService) models(c *gin.Context) {
 			},
 			{
 				"capabilities": gin.H{
-					"family": "gpt-4",
-					"object": "model_capabilities",
-					"type":   "chat",
+					"family":    "gpt-4",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"tool_calls": true},
+					"tokenizer": "cl100k_base",
+					"type":      "chat",
 				},
 				"id":      "gpt-4-0613",
-				"name":    "GPT 4 (2023-06-13)",
+				"name":    "GPT 4",
 				"object":  "model",
 				"version": "gpt-4-0613",
 			},
 			{
 				"capabilities": gin.H{
-					"family": "gpt-4-turbo",
-					"object": "model_capabilities",
-					"type":   "chat",
+					"family":    "gpt-4-turbo",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"parallel_tool_calls": true, "tool_calls": true},
+					"tokenizer": "cl100k_base",
+					"type":      "chat",
 				},
 				"id":      "gpt-4-0125-preview",
-				"name":    "GPT 4 Turbo (2024-01-25 Preview)",
+				"name":    "GPT 4 Turbo",
 				"object":  "model",
 				"version": "gpt-4-0125-preview",
 			},
 			{
 				"capabilities": gin.H{
-					"family": "text-embedding-ada-002",
-					"object": "model_capabilities",
-					"type":   "embeddings",
+					"family":    "gpt-4o",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"parallel_tool_calls": true, "tool_calls": true},
+					"tokenizer": "o200k_base",
+					"type":      "chat",
+				},
+				"id":      "gpt-4o",
+				"name":    "GPT 4o",
+				"object":  "model",
+				"version": "gpt-4o-2024-05-13",
+			},
+			{
+				"capabilities": gin.H{
+					"family":    "gpt-4o",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"parallel_tool_calls": true, "tool_calls": true},
+					"tokenizer": "o200k_base",
+					"type":      "chat",
+				},
+				"id":      "gpt-4o-2024-05-13",
+				"name":    "GPT 4o",
+				"object":  "model",
+				"version": "gpt-4o-2024-05-13",
+			},
+			{
+				"capabilities": gin.H{
+					"family":    "gpt-4o",
+					"limits":    gin.H{"max_prompt_tokens": 20000},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"parallel_tool_calls": true, "tool_calls": true},
+					"tokenizer": "o200k_base",
+					"type":      "chat",
+				},
+				"id":     "gpt-4-o-preview",
+				"name":   "GPT 4o",
+				"object": "model",
+			},
+			{
+				"capabilities": gin.H{
+					"family":    "text-embedding-ada-002",
+					"limits":    gin.H{"max_inputs": 256},
+					"object":    "model_capabilities",
+					"supports":  gin.H{},
+					"tokenizer": "cl100k_base",
+					"type":      "embeddings",
 				},
 				"id":      "text-embedding-ada-002",
 				"name":    "Embedding V2 Ada",
@ -272,20 +358,12 @@ func (s *ProxyService) models(c *gin.Context) {
 			},
 			{
 				"capabilities": gin.H{
-					"family": "text-embedding-ada-002",
-					"object": "model_capabilities",
-					"type":   "embeddings",
-				},
-				"id":      "text-embedding-ada-002-index",
-				"name":    "Embedding V2 Ada (Index)",
-				"object":  "model",
-				"version": "text-embedding-ada-002",
-			},
-			{
-				"capabilities": gin.H{
-					"family": "text-embedding-3-small",
-					"object": "model_capabilities",
-					"type":   "embeddings",
+					"family":    "text-embedding-3-small",
+					"limits":    gin.H{"max_inputs": 256},
+					"object":    "model_capabilities",
+					"supports":  gin.H{"dimensions": true},
+					"tokenizer": "cl100k_base",
+					"type":      "embeddings",
 				},
 				"id":      "text-embedding-3-small",
 				"name":    "Embedding V3 small",
@ -294,9 +372,11 @@ func (s *ProxyService) models(c *gin.Context) {
 			},
 			{
 				"capabilities": gin.H{
-					"family": "text-embedding-3-small",
-					"object": "model_capabilities",
-					"type":   "embeddings",
+					"family":    "text-embedding-3-small",
+					"object":    "model_capabilities",
+					"supports":  gin.H{"dimensions": true},
+					"tokenizer": "cl100k_base",
+					"type":      "embeddings",
 				},
 				"id":      "text-embedding-3-small-inference",
 				"name":    "Embedding V3 small (Inference)",
@ -327,6 +407,12 @@ func (s *ProxyService) completions(c *gin.Context) {

 	if !gjson.GetBytes(body, "function_call").Exists() {
 		messages := gjson.GetBytes(body, "messages").Array()
+		for i, msg := range messages {
+			toolCalls := msg.Get("tool_calls").Array()
+			if len(toolCalls) == 0 {
+				body, _ = sjson.DeleteBytes(body, fmt.Sprintf("messages.%d.tool_calls", i))
+			}
+		}
 		lastIndex := len(messages) - 1
 		if !strings.Contains(messages[lastIndex].Get("content").String(), "Respond in the following locale") {
 			locale := s.cfg.ChatLocale
@ -391,10 +477,14 @@ func (s *ProxyService) completions(c *gin.Context) {
 	_, _ = io.Copy(c.Writer, resp.Body)
 }

+func contains(arr []string, str string) bool {
+    return strings.Contains(strings.Join(arr, ","), str)
+}
+
 func (s *ProxyService) codeCompletions(c *gin.Context) {
 	ctx := c.Request.Context()

-	time.Sleep(100 * time.Millisecond)
+	time.Sleep(200 * time.Millisecond)
 	if ctx.Err() != nil {
 		abortCodex(c, http.StatusRequestTimeout)
 		return
@ -411,13 +501,12 @@ func (s *ProxyService) codeCompletions(c *gin.Context) {
 	proxyUrl := s.cfg.CodexApiBase + "/completions"
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, proxyUrl, io.NopCloser(bytes.NewBuffer(body)))
 	if nil != err {
-		//
 		abortCodex(c, http.StatusInternalServerError)
 		return
 	}

 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", "Bearer "+s.cfg.CodexApiKey)
+	req.Header.Set("Authorization", "Bearer " + getRandomApiKey(s.cfg.CodexApiKey))
 	if "" != s.cfg.CodexApiOrganization {
 		req.Header.Set("OpenAI-Organization", s.cfg.CodexApiOrganization)
 	}
@ -456,17 +545,38 @@ func (s *ProxyService) codeCompletions(c *gin.Context) {
 	_, _ = io.Copy(c.Writer, resp.Body)
 }

+// 随机取一个apiKey
+func getRandomApiKey(paramStr string) string {
+    params := strings.Split(paramStr, ",")
+    rand.Seed(time.Now().UnixNano())
+    randomIndex := rand.Intn(len(params))
+	fmt.Println("Code completion API Key index:", randomIndex)
+	fmt.Println("Code completion API Key:", strings.TrimSpace(params[randomIndex]))
+    return strings.TrimSpace(params[randomIndex])
+}
+
 func ConstructRequestBody(body []byte, cfg *config) []byte {
 	body, _ = sjson.DeleteBytes(body, "extra")
 	body, _ = sjson.DeleteBytes(body, "nwo")
 	body, _ = sjson.SetBytes(body, "model", cfg.CodeInstructModel)
+
+	if int(gjson.GetBytes(body, "max_tokens").Int()) > cfg.CodexMaxTokens {
+		body, _ = sjson.SetBytes(body, "max_tokens", cfg.CodexMaxTokens)
+	}
+
 	if strings.Contains(cfg.CodeInstructModel, StableCodeModelPrefix) {
 		return constructWithStableCodeModel(body)
+	} else if strings.HasPrefix(cfg.CodeInstructModel, DeepSeekCoderModel) || contains(SiliconflowModels, cfg.CodeInstructModel) {
+		if gjson.GetBytes(body, "n").Int() > 1 {
+			body, _ = sjson.SetBytes(body, "n", 1)
+		}
 	}
+
 	if strings.HasSuffix(cfg.ChatApiBase, "chat") {
 		// @Todo  constructWithChatModel
 		// 如果code base以chat结尾则构建chatModel，暂时没有好的prompt
 	}
+
 	return body
 }
Author	SHA1	Message	Date
zhuiyue132	8603e7429e	feat: 增加 siliconflow fim 的支持 (#63 ) siliconflow 已支持标准格式的 FIM 补全,特此 PR	2024-10-11 18:49:08 -07:00
aliensb	6d9ba954dd	Fix: Repair the logic for obtaining the configuration file path. (#60 ) Fixed the logic for obtaining the configuration file path to ensure that config.json is used as the default when no command line arguments are provided.	2024-09-25 00:11:30 -07:00
今夕是何年	9ef70da47b	当message中的tool_calls字段为空数组时移除掉这个属性，防止deepseek报错。 (#57 ) Co-authored-by: liyuzhe <banyebushui@>	2024-09-25 00:10:50 -07:00
Huanzhang Hu	aef14559a1	change models api (#54 ) Co-authored-by: huhuanzhang <huhuanzhang@parkingwang.com>	2024-09-08 18:24:12 -07:00
Liu Bingyan	0685e8c153	Modify expose port (#46 ) Modify expose port to match the port in the docker-compose file.	2024-07-29 00:33:11 -07:00
wozulong	8fdd840460	update README Signed-off-by: wozulong <>	2024-07-26 18:41:27 +08:00
wozulong	19447a898a	update README Signed-off-by: wozulong <>	2024-07-26 15:21:06 +08:00
wozulong	6325a5e2f5	add deepseek-coder fim support Signed-off-by: wozulong <>	2024-07-26 15:20:18 +08:00
forose	e251e9e50b	Update Dockerfile (#39 ) Set up a proxy for Go	2024-06-18 21:20:16 -07:00
echo66677	f93a0ed85f	fix config.json.example (#36 ) Co-authored-by: Neo <Neo@zhile.io>	2024-06-18 21:19:51 -07:00