LLM inference in C/C++
C++ ggml
server: remove obsolete scripts (#23870)
Xuan-Son Nguyenb5f52280fb78
diff --git a/tools/server/chat-llama2.sh b/tools/server/chat-llama2.shdeleted file mode 100755index 450445f17e..0000000000--- a/tools/server/chat-llama2.sh+++ /dev/null@@ -1,109 +0,0 @@-#!/usr/bin/env bash--API_URL="${API_URL:-http://127.0.0.1:8080}"--CHAT=(- "Hello, Assistant."- "Hello. How may I help you today?"-)--INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."--trim() {- shopt -s extglob- set -- "${1##+([[:space:]])}"- printf "%s" "${1%%+([[:space:]])}"-}--trim_trailing() {- shopt -s extglob- printf "%s" "${1%%+([[:space:]])}"-}--format_prompt() {- if [[ "${#CHAT[@]}" -eq 0 ]]; then- echo -n "[INST] <<SYS>>\n${INSTRUCTION}\n<</SYS>>"- else- LAST_INDEX=$(( ${#CHAT[@]} - 1 ))- echo -n "${CHAT[$LAST_INDEX]}\n[INST] $1 [/INST]"- fi-}--tokenize() {- curl \- --silent \- --request POST \- --url "${API_URL}/tokenize" \- --header "Content-Type: application/json" \- --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \- | jq '.tokens[]'-}--N_KEEP=$(tokenize "[INST] <<SYS>>\n${INSTRUCTION}\n<</SYS>>" | wc -l)--chat_completion() {- PROMPT="$(trim_trailing "$(format_prompt "$1")")"- DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{- prompt: .,- temperature: 0.2,- top_k: 40,- top_p: 0.9,- n_keep: $n_keep,- n_predict: 1024,- stop: ["[INST]"],- stream: true- }')"-- # Create a temporary file to hold the Python output- TEMPFILE=$(mktemp)-- exec 3< <(curl \- --silent \- --no-buffer \- --request POST \- --url "${API_URL}/completion" \- --header "Content-Type: application/json" \- --data-raw "${DATA}")-- python -c "-import json-import sys--answer = ''-while True:- line = sys.stdin.readline()- if not line:- break- if line.startswith('data: '):- json_content = line[6:].strip()- content = json.loads(json_content)['content']- sys.stdout.write(content)- sys.stdout.flush()- answer += content--answer = answer.rstrip('\n')--# Write the answer to the temporary file-with open('$TEMPFILE', 'w') as f:- f.write(answer)- " <&3-- exec 3<&--- # Read the answer from the temporary file- ANSWER=$(cat $TEMPFILE)-- # Clean up the temporary file- rm $TEMPFILE-- printf "\n"-- CHAT+=("$1" "$(trim "$ANSWER")")-}--while true; do- echo -en "\033[0;32m" # Green color- read -r -e -p "> " QUESTION- echo -en "\033[0m" # Reset color- chat_completion "${QUESTION}"-donediff --git a/tools/server/chat.mjs b/tools/server/chat.mjsdeleted file mode 100644index 4fef5655a8..0000000000--- a/tools/server/chat.mjs+++ /dev/null@@ -1,131 +0,0 @@-import * as readline from 'node:readline'-import { stdin, stdout } from 'node:process'-import { readFileSync } from 'node:fs'-import { SchemaConverter } from './public_legacy/json-schema-to-grammar.mjs'--const args = process.argv.slice(2);-const grammarJsonSchemaFile = args.find(- (_, index) => args[index - 1] === "--grammar-json-schema"-);--const no_cached_prompt = args.find(- (_, index) => args[index - 1] === "--no-cache-prompt"-) ?? "false";--const grammarFile = args.find((_, index) => args[index - 1] === "--grammar");--// Example usage: function,arguments-const grammarJsonSchemaPropOrder = args.find(- (_, index) => args[index - 1] === "--grammar-json-schema-prop-order"-);-const propOrder = grammarJsonSchemaPropOrder- ? grammarJsonSchemaPropOrder- .split(",")- .reduce((acc, cur, index) => ({ ...acc, [cur]: index }), {})- : {};--let grammar = null-if (grammarJsonSchemaFile) {- let schema = JSON.parse(readFileSync(grammarJsonSchemaFile, 'utf-8'))- const converter = new SchemaConverter({prop_order: propOrder, allow_fetch: true})- schema = await converter.resolveRefs(schema, grammarJsonSchemaFile)- converter.visit(schema, '')- grammar = converter.formatGrammar()-}-if (grammarFile) {- grammar = readFileSync(grammarFile, 'utf-8')-}--// for cached prompt-let slot_id = -1;--const API_URL = 'http://127.0.0.1:8080'--const chat = [- {- human: "Hello, Assistant.",- assistant: "Hello. How may I help you today?"- },- {- human: "Please tell me the largest city in Europe.",- assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia."- },-]--const instruction = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.`--function format_prompt(question) {- return `${instruction}\n${- chat.map(m =>`### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n")- }\n### Human: ${question}\n### Assistant:`-}--async function tokenize(content) {- const result = await fetch(`${API_URL}/tokenize`, {- method: 'POST',- body: JSON.stringify({ content })- })-- if (!result.ok) {- return []- }-- return await result.json().tokens-}--const n_keep = await tokenize(instruction).length--async function chat_completion(question) {- const result = await fetch(`${API_URL}/completion`, {- method: 'POST',- body: JSON.stringify({- prompt: format_prompt(question),- temperature: 0.2,- top_k: 40,- top_p: 0.9,- n_keep: n_keep,- n_predict: 256,- cache_prompt: no_cached_prompt === "false",- slot_id: slot_id,- stop: ["\n### Human:"], // stop completion after generating this- grammar,- stream: true,- })- })-- if (!result.ok) {- return- }-- let answer = ''-- for await (var chunk of result.body) {- const t = Buffer.from(chunk).toString('utf8')- if (t.startsWith('data: ')) {- const message = JSON.parse(t.substring(6))- slot_id = message.slot_id- answer += message.content- process.stdout.write(message.content)- if (message.stop) {- if (message.truncated) {- chat.shift()- }- break- }- }- }-- process.stdout.write('\n')- chat.push({ human: question, assistant: answer.trimStart() })-}--const rl = readline.createInterface({ input: stdin, output: stdout });--const readlineQuestion = (rl, query, options) => new Promise((resolve, reject) => {- rl.question(query, options, resolve)-});--while(true) {- const question = await readlineQuestion(rl, '> ')- await chat_completion(question)-}diff --git a/tools/server/chat.sh b/tools/server/chat.shdeleted file mode 100755index 84cea2d56a..0000000000--- a/tools/server/chat.sh+++ /dev/null@@ -1,80 +0,0 @@-#!/usr/bin/env bash--API_URL="${API_URL:-http://127.0.0.1:8080}"--CHAT=(- "Hello, Assistant."- "Hello. How may I help you today?"- "Please tell me the largest city in Europe."- "Sure. The largest city in Europe is Moscow, the capital of Russia."-)--INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."--trim() {- shopt -s extglob- set -- "${1##+([[:space:]])}"- printf "%s" "${1%%+([[:space:]])}"-}--trim_trailing() {- shopt -s extglob- printf "%s" "${1%%+([[:space:]])}"-}--format_prompt() {- echo -n "${INSTRUCTION}"- printf "\n### Human: %s\n### Assistant: %s" "${CHAT[@]}" "$1"-}--tokenize() {- curl \- --silent \- --request POST \- --url "${API_URL}/tokenize" \- --header "Content-Type: application/json" \- --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \- | jq '.tokens[]'-}--N_KEEP=$(tokenize "${INSTRUCTION}" | wc -l)--chat_completion() {- PROMPT="$(trim_trailing "$(format_prompt "$1")")"- DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{- prompt: .,- temperature: 0.2,- top_k: 40,- top_p: 0.9,- n_keep: $n_keep,- n_predict: 256,- cache_prompt: true,- stop: ["\n### Human:"],- stream: true- }')"-- ANSWER=''-- while IFS= read -r LINE; do- if [[ $LINE = data:* ]]; then- CONTENT="$(echo "${LINE:5}" | jq -r '.content')"- printf "%s" "${CONTENT}"- ANSWER+="${CONTENT}"- fi- done < <(curl \- --silent \- --no-buffer \- --request POST \- --url "${API_URL}/completion" \- --header "Content-Type: application/json" \- --data-raw "${DATA}")-- printf "\n"-- CHAT+=("$1" "$(trim "$ANSWER")")-}--while true; do- read -r -e -p "> " QUESTION- chat_completion "${QUESTION}"-done