diff --git a/packages/ai-native/package.json b/packages/ai-native/package.json index 652b91a71b..d7e3e8a272 100644 --- a/packages/ai-native/package.json +++ b/packages/ai-native/package.json @@ -40,9 +40,9 @@ "@xterm/xterm": "5.5.0", "ansi-regex": "^2.0.0", "dom-align": "^1.7.0", - "js-tiktoken": "1.0.12", "react-chat-elements": "^12.0.10", "react-highlight": "^0.15.0", + "tiktoken": "1.0.12", "web-tree-sitter": "0.22.6" }, "devDependencies": { diff --git a/packages/ai-native/src/browser/contrib/inline-completions/prompt/prompt.ts b/packages/ai-native/src/browser/contrib/inline-completions/prompt/prompt.ts index 68b8574f31..71958602c9 100644 --- a/packages/ai-native/src/browser/contrib/inline-completions/prompt/prompt.ts +++ b/packages/ai-native/src/browser/contrib/inline-completions/prompt/prompt.ts @@ -1,5 +1,4 @@ -// @ts-ignore -import { Tiktoken } from 'js-tiktoken'; +import { Tiktoken } from 'tiktoken/lite'; import { Injector } from '@opensumi/di'; @@ -59,7 +58,12 @@ export const getMarkerForSnippets = (text: string, language: string) => { return lines.map((line) => getMarkerByLanguage(line, language)).join('\n'); }; -export const getCroppedTextByLine = (text: string, maxTokenSize: number, textTokens: number[][], reverse = false) => { +export const getCroppedTextByLine = ( + text: string, + maxTokenSize: number, + textTokens: Uint32Array[], + reverse = false, +) => { const currentTokenSize = textTokens.reduce((prev, cur) => prev + cur.length, 0); if (currentTokenSize < maxTokenSize) { return text; @@ -110,7 +114,7 @@ export const getCroppedTextByLine = (text: string, maxTokenSize: number, textTok export const getCroppedText = async ( text: string, maxTokenSize: number, - textTokens: number[][], + textTokens: Uint32Array[], strategy = StrategyType.InterceptBasedOnLine, tokenizer: Tiktoken, parser?: LanguageParser, @@ -118,7 +122,7 @@ export const getCroppedText = async ( reverse = false, token?: monaco.CancellationToken, ): Promise => { - let tokens: number[]; + let tokens: Uint32Array; if (strategy === StrategyType.InterceptBasedOnLine) { // 按行进行裁剪 text = getCroppedTextByLine(text, maxTokenSize, textTokens, reverse); diff --git a/packages/ai-native/src/browser/contrib/inline-completions/prompt/tokenizer.ts b/packages/ai-native/src/browser/contrib/inline-completions/prompt/tokenizer.ts index e0a348ed6b..22cbe91807 100644 --- a/packages/ai-native/src/browser/contrib/inline-completions/prompt/tokenizer.ts +++ b/packages/ai-native/src/browser/contrib/inline-completions/prompt/tokenizer.ts @@ -1,5 +1,4 @@ -// @ts-ignore -import { Tiktoken, getEncoding } from 'js-tiktoken'; +import { Tiktoken, get_encoding } from 'tiktoken'; import { TokenizerName } from '../types'; @@ -10,7 +9,7 @@ export const getTokenizer = (tokenizerName = TokenizerName.cl100k_base) => { if (tokenizer) { return tokenizer; } - tokenizer = getEncoding('cl100k_base'); + tokenizer = get_encoding('cl100k_base'); TOKENIZER_CACHE.set(tokenizerName, tokenizer); return tokenizer; }; diff --git a/tools/dev-tool/src/webpack.js b/tools/dev-tool/src/webpack.js index 5d5394f81e..ce3b78547d 100644 --- a/tools/dev-tool/src/webpack.js +++ b/tools/dev-tool/src/webpack.js @@ -45,6 +45,9 @@ exports.createWebpackConfig = function (dir, entry, extraConfig) { cache: { type: 'filesystem', }, + experiments: { + asyncWebAssembly: true, // 启用 WebAssembly 支持 + }, resolve: { extensions: ['.ts', '.tsx', '.js', '.json', '.less'], plugins: [ diff --git a/yarn.lock b/yarn.lock index 978124f030..b9c7dd0dd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2265,9 +2265,9 @@ __metadata: "@xterm/xterm": "npm:5.5.0" ansi-regex: "npm:^2.0.0" dom-align: "npm:^1.7.0" - js-tiktoken: "npm:1.0.12" react-chat-elements: "npm:^12.0.10" react-highlight: "npm:^0.15.0" + tiktoken: "npm:1.0.12" web-tree-sitter: "npm:0.22.6" languageName: unknown linkType: soft @@ -12221,15 +12221,6 @@ __metadata: languageName: node linkType: hard -"js-tiktoken@npm:1.0.12": - version: 1.0.12 - resolution: "js-tiktoken@npm:1.0.12" - dependencies: - base64-js: "npm:^1.5.1" - checksum: 10/21aaa9302409fefc5ac18695579b04e0223cdda2566e5cc4a95de228333bbadfc2e16110fceca1824d7faa43081ef378e6bc72238a2230374a85f88638556305 - languageName: node - linkType: hard - "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0": version: 4.0.0 resolution: "js-tokens@npm:4.0.0" @@ -18889,6 +18880,13 @@ __metadata: languageName: node linkType: hard +"tiktoken@npm:1.0.12": + version: 1.0.12 + resolution: "tiktoken@npm:1.0.12" + checksum: 10/595ab2e93f1937a1af8baf057f1a2951cfd7b5d4c0986807c1fc8e1a2118800a31cec81cdbcb115c4be5eec4fd9d5c23ac23691a227658c1010aeefec5dfa3ea + languageName: node + linkType: hard + "timeago.js@npm:^4.0.2": version: 4.0.2 resolution: "timeago.js@npm:4.0.2"