Skip to content

Commit ae87863

Browse files
taronaeotaronaeo
authored
llama-bench: introduce -hf and -hff flags & use --mmap 1 by default (ggml-org#20211)
1 parent 97c64fb commit ae87863

1 file changed

Lines changed: 116 additions & 64 deletions

File tree

‎tools/llama-bench/llama-bench.cpp‎

Lines changed: 116 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <unordered_set>
2121

2222
#include "common.h"
23+
#include "download.h"
2324
#include "ggml.h"
2425
#include "llama.h"
2526

@@ -312,6 +313,9 @@ static std::vector<int> parse_int_range(const std::string & s) {
312313

313314
struct cmd_params {
314315
std::vector<std::string> model;
316+
std::vector<std::string> hf_repo;
317+
std::vector<std::string> hf_file;
318+
std::string hf_token;
315319
std::vector<int> n_prompt;
316320
std::vector<int> n_gen;
317321
std::vector<std::pair<int, int>> n_pg;
@@ -351,6 +355,9 @@ struct cmd_params {
351355

352356
static const cmd_params cmd_params_defaults = {
353357
/* model */ { "models/7B/ggml-model-q4_0.gguf" },
358+
/* hf_repo */ {},
359+
/* hf_file */ {},
360+
/* hf_token */ "",
354361
/* n_prompt */ { 512 },
355362
/* n_gen */ { 128 },
356363
/* n_pg */ {},
@@ -372,7 +379,7 @@ static const cmd_params cmd_params_defaults = {
372379
/* devices */ { {} },
373380
/* tensor_split */ { std::vector<float>(llama_max_devices(), 0.0f) },
374381
/* tensor_buft_overrides*/ { std::vector<llama_model_tensor_buft_override>{ { nullptr, nullptr } } },
375-
/* use_mmap */ { false },
382+
/* use_mmap */ { true },
376383
/* use_direct_io */ { false },
377384
/* embeddings */ { false },
378385
/* no_op_offload */ { false },
@@ -393,74 +400,57 @@ static void print_usage(int /* argc */, char ** argv) {
393400
printf("\n");
394401
printf("options:\n");
395402
printf(" -h, --help\n");
396-
printf(" --numa <distribute|isolate|numactl> numa mode (default: disabled)\n");
397-
printf(" -r, --repetitions <n> number of times to repeat each test (default: %d)\n",
398-
cmd_params_defaults.reps);
399-
printf(" --prio <-1|0|1|2|3> process/thread priority (default: %d)\n",
400-
cmd_params_defaults.prio);
401-
printf(" --delay <0...N> (seconds) delay between each test (default: %d)\n",
402-
cmd_params_defaults.delay);
403-
printf(" -o, --output <csv|json|jsonl|md|sql> output format printed to stdout (default: %s)\n",
404-
output_format_str(cmd_params_defaults.output_format));
405-
printf(" -oe, --output-err <csv|json|jsonl|md|sql> output format printed to stderr (default: %s)\n",
406-
output_format_str(cmd_params_defaults.output_format_stderr));
407-
printf(" --list-devices list available devices and exit\n");
408-
printf(" -v, --verbose verbose output\n");
409-
printf(" --progress print test progress indicators\n");
410-
printf(" --no-warmup skip warmup runs before benchmarking\n");
403+
printf(" --numa <distribute|isolate|numactl> numa mode (default: disabled)\n");
404+
printf(" -r, --repetitions <n> number of times to repeat each test (default: %d)\n", cmd_params_defaults.reps);
405+
printf(" --prio <-1|0|1|2|3> process/thread priority (default: %d)\n", cmd_params_defaults.prio);
406+
printf(" --delay <0...N> (seconds) delay between each test (default: %d)\n", cmd_params_defaults.delay);
407+
printf(" -o, --output <csv|json|jsonl|md|sql> output format printed to stdout (default: %s)\n", output_format_str(cmd_params_defaults.output_format));
408+
printf(" -oe, --output-err <csv|json|jsonl|md|sql> output format printed to stderr (default: %s)\n", output_format_str(cmd_params_defaults.output_format_stderr));
409+
printf(" --list-devices list available devices and exit\n");
410+
printf(" -v, --verbose verbose output\n");
411+
printf(" --progress print test progress indicators\n");
412+
printf(" --no-warmup skip warmup runs before benchmarking\n");
411413
if (llama_supports_rpc()) {
412-
printf(" -rpc, --rpc <rpc_servers> register RPC devices (comma separated)\n");
414+
printf(" -rpc, --rpc <rpc_servers> register RPC devices (comma separated)\n");
413415
}
414416
printf("\n");
415417
printf("test parameters:\n");
416-
printf(" -m, --model <filename> (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());
417-
printf(" -p, --n-prompt <n> (default: %s)\n",
418-
join(cmd_params_defaults.n_prompt, ",").c_str());
419-
printf(" -n, --n-gen <n> (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());
420-
printf(" -pg <pp,tg> (default: %s)\n",
421-
join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());
422-
printf(" -d, --n-depth <n> (default: %s)\n",
423-
join(cmd_params_defaults.n_depth, ",").c_str());
424-
printf(" -b, --batch-size <n> (default: %s)\n",
425-
join(cmd_params_defaults.n_batch, ",").c_str());
426-
printf(" -ub, --ubatch-size <n> (default: %s)\n",
427-
join(cmd_params_defaults.n_ubatch, ",").c_str());
428-
printf(" -ctk, --cache-type-k <t> (default: %s)\n",
429-
join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());
430-
printf(" -ctv, --cache-type-v <t> (default: %s)\n",
431-
join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());
432-
printf(" -t, --threads <n> (default: %s)\n",
433-
join(cmd_params_defaults.n_threads, ",").c_str());
434-
printf(" -C, --cpu-mask <hex,hex> (default: %s)\n",
435-
join(cmd_params_defaults.cpu_mask, ",").c_str());
436-
printf(" --cpu-strict <0|1> (default: %s)\n",
437-
join(cmd_params_defaults.cpu_strict, ",").c_str());
438-
printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());
439-
printf(" -ngl, --n-gpu-layers <n> (default: %s)\n",
440-
join(cmd_params_defaults.n_gpu_layers, ",").c_str());
441-
printf(" -ncmoe, --n-cpu-moe <n> (default: %s)\n",
442-
join(cmd_params_defaults.n_cpu_moe, ",").c_str());
443-
printf(" -sm, --split-mode <none|layer|row> (default: %s)\n",
444-
join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
445-
printf(" -mg, --main-gpu <i> (default: %s)\n",
446-
join(cmd_params_defaults.main_gpu, ",").c_str());
447-
printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n",
448-
join(cmd_params_defaults.no_kv_offload, ",").c_str());
449-
printf(" -fa, --flash-attn <0|1> (default: %s)\n",
450-
join(cmd_params_defaults.flash_attn, ",").c_str());
451-
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
452-
printf(" -mmp, --mmap <0|1> (default: %s)\n",
453-
join(cmd_params_defaults.use_mmap, ",").c_str());
454-
printf(" -dio, --direct-io <0|1> (default: %s)\n",
455-
join(cmd_params_defaults.use_direct_io, ",").c_str());
456-
printf(" -embd, --embeddings <0|1> (default: %s)\n",
457-
join(cmd_params_defaults.embeddings, ",").c_str());
458-
printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");
418+
printf(" -m, --model <filename> (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());
419+
printf(" -hf, -hfr, --hf-repo <user>/<model>[:quant] Hugging Face model repository; quant is optional, case-insensitive\n");
420+
printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n");
421+
printf(" example: unsloth/phi-4-GGUF:Q4_K_M\n");
422+
printf(" (default: unused)\n");
423+
printf(" -hff, --hf-file <file> Hugging Face model file. If specified, it will override the quant in --hf-repo\n");
424+
printf(" (default: unused)\n");
425+
printf(" -hft, --hf-token <token> Hugging Face access token\n");
426+
printf(" (default: value from HF_TOKEN environment variable)\n");
427+
printf(" -p, --n-prompt <n> (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());
428+
printf(" -n, --n-gen <n> (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());
429+
printf(" -pg <pp,tg> (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());
430+
printf(" -d, --n-depth <n> (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str());
431+
printf(" -b, --batch-size <n> (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());
432+
printf(" -ub, --ubatch-size <n> (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());
433+
printf(" -ctk, --cache-type-k <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());
434+
printf(" -ctv, --cache-type-v <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());
435+
printf(" -t, --threads <n> (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());
436+
printf(" -C, --cpu-mask <hex,hex> (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());
437+
printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());
438+
printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());
439+
printf(" -ngl, --n-gpu-layers <n> (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());
440+
printf(" -ncmoe, --n-cpu-moe <n> (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());
441+
printf(" -sm, --split-mode <none|layer|row> (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
442+
printf(" -mg, --main-gpu <i> (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());
443+
printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
444+
printf(" -fa, --flash-attn <0|1> (default: %s)\n", join(cmd_params_defaults.flash_attn, ",").c_str());
445+
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
446+
printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str());
447+
printf(" -dio, --direct-io <0|1> (default: %s)\n", join(cmd_params_defaults.use_direct_io, ",").c_str());
448+
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
449+
printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");
459450
printf(" -ot --override-tensor <tensor name pattern>=<buffer type>;...\n");
460-
printf(" (default: disabled)\n");
461-
printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");
462-
printf(" --no-host <0|1> (default: %s)\n",
463-
join(cmd_params_defaults.no_host, ",").c_str());
451+
printf(" (default: disabled)\n");
452+
printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");
453+
printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());
464454
printf("\n");
465455
printf(
466456
"Multiple values can be given for each parameter by separating them with ','\n"
@@ -514,6 +504,10 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
514504
params.progress = cmd_params_defaults.progress;
515505
params.no_warmup = cmd_params_defaults.no_warmup;
516506

507+
if (const char * env = getenv("HF_TOKEN")) {
508+
params.hf_token = env;
509+
}
510+
517511
for (int i = 1; i < argc; i++) {
518512
arg = argv[i];
519513
if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
@@ -531,6 +525,26 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
531525
}
532526
auto p = string_split<std::string>(argv[i], split_delim);
533527
params.model.insert(params.model.end(), p.begin(), p.end());
528+
} else if (arg == "-hf" || arg == "-hfr" || arg == "--hf-repo") {
529+
if (++i >= argc) {
530+
invalid_param = true;
531+
break;
532+
}
533+
auto p = string_split<std::string>(argv[i], split_delim);
534+
params.hf_repo.insert(params.hf_repo.end(), p.begin(), p.end());
535+
} else if (arg == "-hff" || arg == "--hf-file") {
536+
if (++i >= argc) {
537+
invalid_param = true;
538+
break;
539+
}
540+
auto p = string_split<std::string>(argv[i], split_delim);
541+
params.hf_file.insert(params.hf_file.end(), p.begin(), p.end());
542+
} else if (arg == "-hft" || arg == "--hf-token") {
543+
if (++i >= argc) {
544+
invalid_param = true;
545+
break;
546+
}
547+
params.hf_token = argv[i];
534548
} else if (arg == "-p" || arg == "--n-prompt") {
535549
if (++i >= argc) {
536550
invalid_param = true;
@@ -961,6 +975,44 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
961975
exit(1);
962976
}
963977

978+
if (!params.hf_repo.empty()) {
979+
for (size_t i = 0; i < params.hf_repo.size(); i++) {
980+
common_params_model model;
981+
982+
// step 1: no `-hff` provided, we auto-detect based on the `-hf` flag
983+
if (params.hf_file.empty() || params.hf_file[i].empty()) {
984+
auto auto_detected = common_get_hf_file(params.hf_repo[i], params.hf_token, false);
985+
if (auto_detected.repo.empty() || auto_detected.ggufFile.empty()) {
986+
exit(1);
987+
}
988+
989+
model.name = params.hf_repo[i];
990+
model.hf_repo = auto_detected.repo;
991+
model.hf_file = auto_detected.ggufFile;
992+
} else {
993+
model.hf_file = params.hf_file[i];
994+
}
995+
996+
// step 2: construct the model cache path
997+
std::string clean_fname = model.hf_repo + "_" + model.hf_file;
998+
string_replace_all(clean_fname, "\\", "_");
999+
string_replace_all(clean_fname, "/", "_");
1000+
model.path = fs_get_cache_file(clean_fname);
1001+
1002+
// step 3: download the model if not exists
1003+
std::string model_endpoint = get_model_endpoint();
1004+
model.url = model_endpoint + model.hf_repo + "/resolve/main/" + model.hf_file;
1005+
1006+
bool ok = common_download_model(model, params.hf_token, false);
1007+
if (!ok) {
1008+
fprintf(stderr, "error: failed to download model from %s\n", model.url.c_str());
1009+
exit(1);
1010+
}
1011+
1012+
params.model.push_back(model.path);
1013+
}
1014+
}
1015+
9641016
// set defaults
9651017
if (params.model.empty()) {
9661018
params.model = cmd_params_defaults.model;

0 commit comments

Comments
 (0)