perf(common): 优化 token 计算方式，在较大 input 时使用估算避免服务器负载过高

alioth-center · Nov 25, 2024 · f476ca3 · f476ca3
1 parent 3546224
commit f476ca3
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/app/service/common.go b/app/service/common.go
@@ -98,6 +98,12 @@ func GetAvailableClient(ctx context.Context, key string, modelName string, promp
 
 func CalculatePromptToken(inputs ...string) (promptToken int64) {
 	for _, input := range inputs {
+		if len([]rune(input)) > 2500 {
+			// tokenizer has performance issue with large input length, use a simple method to calculate token
+			promptToken += int64(len([]rune(input)) / 4)
+			continue
+		}
+
 		promptToken += int64(tokenizer.MustCalToken(input))
 	}