LLM inference in C/C++
C++ ggml
tokenize : drop --stdin mutual-exclusion check (#25672)
Adrien Gallouëtaff6eb6e7503
diff --git a/common/arg.cpp b/common/arg.cppindex b6fddae00c..7bc770ab4f 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2932,7 +2932,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex).set_examples({LLAMA_EXAMPLE_TOKENIZE}));add_opt(common_arg({"--stdin"},- string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),+ string_format("read the prompt from stdin (takes precedence over -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),[](common_params & params) {params.tokenize_stdin = true;}diff --git a/tools/tokenize/tokenize.cpp b/tools/tokenize/tokenize.cppindex 23120ad2e1..77b33c4a46 100644--- a/tools/tokenize/tokenize.cpp+++ b/tools/tokenize/tokenize.cpp@@ -103,19 +103,11 @@ int main(int argc, char ** argv) {return 1;}- // which prompt source was requested?- // -p/--prompt and -f/--file both end up in params.prompt (common's -f also- // strips a single trailing newline), but -f additionally records the path- // in params.prompt_file, so we use that to tell them apart.+ // -f and -p both land in params.prompt; -f also sets prompt_file. -f and -p+ // resolve like the other tools (no mutual exclusion), --stdin takes precedence.const bool use_stdin = params.tokenize_stdin;const bool use_file = !params.prompt_file.empty();- // sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt- if (use_stdin && (use_file || !params.prompt.empty())) {- LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n");- return 1;- }-// must have some promptif (!use_stdin && !use_file && params.prompt.empty()) {LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");