Skip to content

Devika is the first open-source implementation of an Agentic Software Engineer. Initially started as an open-source alternative to Devin.

Python

Implement TicToken library in frontend (#283)

Leandro de Souza7474541c21c5
diff --git a/ui/bun.lockb b/ui/bun.lockb
index c727e41..b9271d9 100644
Binary files a/ui/bun.lockb and b/ui/bun.lockb differ
diff --git a/ui/package.json b/ui/package.json
index 60388c0..06aa039 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -16,12 +16,14 @@
"postcss-load-config": "^5.0.2",
"svelte": "^4.2.7",
"tailwindcss": "^3.3.6",
- "vite": "^5.0.3"
+ "vite": "^5.0.3",
+ "vite-plugin-wasm": "^3.3.0"
},
"type": "module",
"dependencies": {
- "socket.io-client": "^4.7.5",
"clsx": "^2.1.0",
+ "socket.io-client": "^4.7.5",
+ "tiktoken": "^1.0.13",
"xterm": "^5.3.0",
"xterm-addon-fit": "^0.8.0"
}
diff --git a/ui/src/lib/components/MessageInput.svelte b/ui/src/lib/components/MessageInput.svelte
index a454d4f..dd1a2b8 100644
--- a/ui/src/lib/components/MessageInput.svelte
+++ b/ui/src/lib/components/MessageInput.svelte
@@ -1,6 +1,7 @@
<script>
- import { API_BASE_URL, socket } from "$lib/api";
+ import { socket } from "$lib/api";
import { agentState, messages } from "$lib/store";
+ import { calculateTokens } from "$lib/token";
let isAgentActive = false;
@@ -13,7 +14,7 @@
const projectName = localStorage.getItem("selectedProject");
const selectedModel = localStorage.getItem("selectedModel");
const serachEngine = localStorage.getItem("selectedSearchEngine");
-
+
if (!projectName) {
alert("Please select a project first!");
return;
@@ -30,7 +31,7 @@
message: messageInput,
base_model: selectedModel,
project_name: projectName,
- search_engine: serachEngine
+ search_engine: serachEngine,
});
} else {
socket.emit("user-message", {
@@ -38,33 +39,18 @@
message: messageInput,
base_model: selectedModel,
project_name: projectName,
- search_engine: serachEngine
- });
+ search_engine: serachEngine,
+ });
}
messageInput = "";
}
}
- function calculateTokens(event) {
+ function setTokenSize(event) {
const prompt = event.target.value;
- fetch(`${API_BASE_URL}/api/calculate-tokens`, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify({ prompt }),
- })
- .then((response) => response.json())
- .then((data) => {
- document.querySelector(".token-count").textContent =
- `${data.token_usage} tokens`;
- })
- .catch((error) => {
- console.error("Error:", error);
- });
+ let tokens = calculateTokens(prompt);
+ document.querySelector(".token-count").textContent = `${tokens} tokens`;
}
-
-
</script>
<div class="expandable-input relative">
@@ -73,7 +59,7 @@
class="w-full p-2 border-2 rounded-lg pr-20"
placeholder="Type your message..."
bind:value={messageInput}
- on:input={calculateTokens}
+ on:input={setTokenSize}
on:keydown={(e) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
diff --git a/ui/src/lib/token.js b/ui/src/lib/token.js
new file mode 100644
index 0000000..426f74c
--- /dev/null
+++ b/ui/src/lib/token.js
@@ -0,0 +1,13 @@
+import { Tiktoken } from "tiktoken/lite";
+import cl100k_base from "tiktoken/encoders/cl100k_base.json";
+
+const encoding = new Tiktoken(
+ cl100k_base.bpe_ranks,
+ cl100k_base.special_tokens,
+ cl100k_base.pat_str
+);
+
+export function calculateTokens(text) {
+ const tokens = encoding.encode(text);
+ return tokens.length;
+}
diff --git a/ui/vite.config.js b/ui/vite.config.js
index 576a486..8a41207 100644
--- a/ui/vite.config.js
+++ b/ui/vite.config.js
@@ -1,9 +1,10 @@
-import { sveltekit } from '@sveltejs/kit/vite';
-import { defineConfig } from 'vite';
+import { sveltekit } from "@sveltejs/kit/vite";
+import { defineConfig } from "vite";
+import wasm from "vite-plugin-wasm";
export default defineConfig({
- plugins: [sveltekit()],
- server: {
- port: 3000,
- },
+ plugins: [sveltekit(), wasm()],
+ server: {
+ port: 3000,
+ },
});