首頁 探索 說明
登入
cturan
/
llama.cpp
镜像来自 https://github.com/cturan/llama.cpp
1
0
複刻 0
Files 問題管理 0 Wiki
目錄樹: ba69bbc84c
分支列表 標籤列表
llama.cpp
/
ci
/
run.sh

run.sh 27 KB
文件歷史 原始文件

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531 
  1. #/bin/bash
    2. 

  2. #
    3. 

  3. # sample usage:
    4. 

  4. #
    5. 

  5. # mkdir tmp
    6. 

  6. #
    7. 

  7. # # CPU-only build
    8. 

  8. # bash ./ci/run.sh ./tmp/results ./tmp/mnt
    9. 

  9. #
    10. 

  10. # # with CUDA support
    11. 

  11. # GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
    12. 

  12. #
    13. 

  13. 

  14. if [ -z "$2" ]; then
    15. 

  15.     echo "usage: $0 <output-dir> <mnt-dir>"
    16. 

  16.     exit 1
    17. 

  17. fi
    18. 

  18. 

  19. mkdir -p "$1"
    20. 

  20. mkdir -p "$2"
    21. 

  21. 

  22. OUT=$(realpath "$1")
    23. 

  23. MNT=$(realpath "$2")
    24. 

  24. 

  25. rm -v $OUT/*.log
    26. 

  26. rm -v $OUT/*.exit
    27. 

  27. rm -v $OUT/*.md
    28. 

  28. 

  29. sd=`dirname $0`
    30. 

  30. cd $sd/../
    31. 

  31. SRC=`pwd`
    32. 

  32. 

  33. CMAKE_EXTRA=""
    34. 

  34. 

  35. if [ ! -z ${GG_BUILD_METAL} ]; then
    36. 

  36.     CMAKE_EXTRA="${CMAKE_EXTRA} -DLLAMA_METAL_SHADER_DEBUG=ON"
    37. 

  37. fi
    38. 

  38. 

  39. ## helpers
    40. 

  40. 

  41. # download a file if it does not exist or if it is outdated
    42. 

  42. function gg_wget {
    43. 

  43.     local out=$1
    44. 

  44.     local url=$2
    45. 

  45. 

  46.     local cwd=`pwd`
    47. 

  47. 

  48.     mkdir -p $out
    49. 

  49.     cd $out
    50. 

  50. 

  51.     # should not re-download if file is the same
    52. 

  52.     wget -nv -N $url
    53. 

  53. 

  54.     cd $cwd
    55. 

  55. }
    56. 

  56. 

  57. function gg_printf {
    58. 

  58.     printf -- "$@" >> $OUT/README.md
    59. 

  59. }
    60. 

  60. 

  61. function gg_run {
    62. 

  62.     ci=$1
    63. 

  63. 

  64.     set -o pipefail
    65. 

  65.     set -x
    66. 

  66. 

  67.     gg_run_$ci | tee $OUT/$ci.log
    68. 

  68.     cur=$?
    69. 

  69.     echo "$cur" > $OUT/$ci.exit
    70. 

  70. 

  71.     set +x
    72. 

  72.     set +o pipefail
    73. 

  73. 

  74.     gg_sum_$ci
    75. 

  75. 

  76.     ret=$((ret | cur))
    77. 

  77. }
    78. 

  78. 

  79. ## ci
    80. 

  80. 

  81. # ctest_debug
    82. 

  82. 

  83. function gg_run_ctest_debug {
    84. 

  84.     cd ${SRC}
    85. 

  85. 

  86.     rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
    87. 

  87. 

  88.     set -e
    89. 

  89. 

  90.     (time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    91. 

  91.     (time make -j                                          ) 2>&1 | tee -a $OUT/${ci}-make.log
    92. 

  92. 

  93.     (time ctest --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    94. 

  94. 

  95.     set +e
    96. 

  96. }
    97. 

  97. 

  98. function gg_sum_ctest_debug {
    99. 

  99.     gg_printf '### %s\n\n' "${ci}"
    100. 

  100. 

  101.     gg_printf 'Runs ctest in debug mode\n'
    102. 

  102.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    103. 

  103.     gg_printf '```\n'
    104. 

  104.     gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
    105. 

  105.     gg_printf '```\n'
    106. 

  106.     gg_printf '\n'
    107. 

  107. }
    108. 

  108. 

  109. # ctest_release
    110. 

  110. 

  111. function gg_run_ctest_release {
    112. 

  112.     cd ${SRC}
    113. 

  113. 

  114.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    115. 

  115. 

  116.     set -e
    117. 

  117. 

  118.     (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    119. 

  119.     (time make -j                                            ) 2>&1 | tee -a $OUT/${ci}-make.log
    120. 

  120. 

  121.     if [ -z ${GG_BUILD_LOW_PERF} ]; then
    122. 

  122.         (time ctest --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    123. 

  123.     else
    124. 

  124.         (time ctest --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    125. 

  125.     fi
    126. 

  126. 

  127.     set +e
    128. 

  128. }
    129. 

  129. 

  130. function gg_sum_ctest_release {
    131. 

  131.     gg_printf '### %s\n\n' "${ci}"
    132. 

  132. 

  133.     gg_printf 'Runs ctest in release mode\n'
    134. 

  134.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    135. 

  135.     gg_printf '```\n'
    136. 

  136.     gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
    137. 

  137.     gg_printf '```\n'
    138. 

  138. }
    139. 

  139. 

  140. # open_llama_3b_v2
    141. 

  141. 

  142. function gg_run_open_llama_3b_v2 {
    143. 

  143.     cd ${SRC}
    144. 

  144. 

  145.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/config.json
    146. 

  146.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/tokenizer.model
    147. 

  147.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/tokenizer_config.json
    148. 

  148.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/special_tokens_map.json
    149. 

  149.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/pytorch_model.bin
    150. 

  150.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/generation_config.json
    151. 

  151. 

  152.     gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
    153. 

  153.     unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
    154. 

  154.     head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw
    155. 

  155. 

  156.     path_models="../models-mnt/open-llama/3B-v2"
    157. 

  157.     path_wiki="../models-mnt/wikitext/wikitext-2-raw"
    158. 

  158. 

  159.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    160. 

  160. 

  161.     set -e
    162. 

  162. 

  163.     (time cmake -DCMAKE_BUILD_TYPE=Release -DLLAMA_QKK_64=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    164. 

  164.     (time make -j                                              ) 2>&1 | tee -a $OUT/${ci}-make.log
    165. 

  165. 

  166.     python3 ../convert.py ${path_models}
    167. 

  167. 

  168.     model_f16="${path_models}/ggml-model-f16.gguf"
    169. 

  169.     model_q8_0="${path_models}/ggml-model-q8_0.gguf"
    170. 

  170.     model_q4_0="${path_models}/ggml-model-q4_0.gguf"
    171. 

  171.     model_q4_1="${path_models}/ggml-model-q4_1.gguf"
    172. 

  172.     model_q5_0="${path_models}/ggml-model-q5_0.gguf"
    173. 

  173.     model_q5_1="${path_models}/ggml-model-q5_1.gguf"
    174. 

  174.     model_q2_k="${path_models}/ggml-model-q2_k.gguf"
    175. 

  175.     model_q3_k="${path_models}/ggml-model-q3_k.gguf"
    176. 

  176.     model_q4_k="${path_models}/ggml-model-q4_k.gguf"
    177. 

  177.     model_q5_k="${path_models}/ggml-model-q5_k.gguf"
    178. 

  178.     model_q6_k="${path_models}/ggml-model-q6_k.gguf"
    179. 

  179. 

  180.     wiki_test_60="${path_wiki}/wiki.test-60.raw"
    181. 

  181. 

  182.     ./bin/test-autorelease ${model_f16}
    183. 

  183. 

  184.     ./bin/quantize ${model_f16} ${model_q8_0} q8_0
    185. 

  185.     ./bin/quantize ${model_f16} ${model_q4_0} q4_0
    186. 

  186.     ./bin/quantize ${model_f16} ${model_q4_1} q4_1
    187. 

  187.     ./bin/quantize ${model_f16} ${model_q5_0} q5_0
    188. 

  188.     ./bin/quantize ${model_f16} ${model_q5_1} q5_1
    189. 

  189.     ./bin/quantize ${model_f16} ${model_q2_k} q2_k
    190. 

  190.     ./bin/quantize ${model_f16} ${model_q3_k} q3_k
    191. 

  191.     ./bin/quantize ${model_f16} ${model_q4_k} q4_k
    192. 

  192.     ./bin/quantize ${model_f16} ${model_q5_k} q5_k
    193. 

  193.     ./bin/quantize ${model_f16} ${model_q6_k} q6_k
    194. 

  194. 

  195.     (time ./bin/main --model ${model_f16}  -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    196. 

  196.     (time ./bin/main --model ${model_q8_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    197. 

  197.     (time ./bin/main --model ${model_q4_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    198. 

  198.     (time ./bin/main --model ${model_q4_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    199. 

  199.     (time ./bin/main --model ${model_q5_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    200. 

  200.     (time ./bin/main --model ${model_q5_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    201. 

  201.     (time ./bin/main --model ${model_q2_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    202. 

  202.     (time ./bin/main --model ${model_q3_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    203. 

  203.     (time ./bin/main --model ${model_q4_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    204. 

  204.     (time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    205. 

  205.     (time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    206. 

  206. 

  207.     (time ./bin/perplexity --model ${model_f16}  -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    208. 

  208.     (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    209. 

  209.     (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    210. 

  210.     (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    211. 

  211.     (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    212. 

  212.     (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    213. 

  213.     (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    214. 

  214.     (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    215. 

  215.     (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    216. 

  216.     (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    217. 

  217.     (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    218. 

  218. 

  219.     (time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
    220. 

  220. 

  221.     (time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
    222. 

  222. 

  223.     function check_ppl {
    224. 

  224.         qnt="$1"
    225. 

  225.         ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    226. 

  226. 

  227.         if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
    228. 

  228.             printf '  - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
    229. 

  229.             return 20
    230. 

  230.         fi
    231. 

  231. 

  232.         printf '  - %s @ %s OK\n' "$qnt" "$ppl"
    233. 

  233.         return 0
    234. 

  234.     }
    235. 

  235. 

  236.     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    237. 

  237.     check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    238. 

  238.     check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    239. 

  239.     check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    240. 

  240.     check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    241. 

  241.     check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    242. 

  242.     check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    243. 

  243.     check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    244. 

  244.     check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    245. 

  245.     check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    246. 

  246.     check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    247. 

  247. 

  248.     cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
    249. 

  249. 

  250.     # lora
    251. 

  251.     function compare_ppl {
    252. 

  252.         qnt="$1"
    253. 

  253.         ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    254. 

  254.         ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    255. 

  255. 

  256.         if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
    257. 

  257.             printf '  - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
    258. 

  258.             return 20
    259. 

  259.         fi
    260. 

  260. 

  261.         printf '  - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
    262. 

  262.         return 0
    263. 

  263.     }
    264. 

  264. 

  265.     path_lora="../models-mnt/open-llama/3B-v2/lora"
    266. 

  266.     path_shakespeare="../models-mnt/shakespeare"
    267. 

  267. 

  268.     shakespeare="${path_shakespeare}/shakespeare.txt"
    269. 

  269.     lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
    270. 

  270. 

  271.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_config.json
    272. 

  272.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_model.bin
    273. 

  273.     gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/shakespeare.txt
    274. 

  274. 

  275.     python3 ../convert-lora-to-ggml.py ${path_lora}
    276. 

  276. 

  277.     # f16
    278. 

  278.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare}                            -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
    279. 

  279.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
    280. 

  280.     compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    281. 

  281. 

  282.     # q8_0
    283. 

  283.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare}                            -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
    284. 

  284.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
    285. 

  285.     compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    286. 

  286. 

  287.     # q8_0 + f16 lora-base
    288. 

  288.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
    289. 

  289.     compare_ppl "q8_0 / f16 base shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    290. 

  290. 

  291.     set +e
    292. 

  292. }
    293. 

  293. 

  294. function gg_sum_open_llama_3b_v2 {
    295. 

  295.     gg_printf '### %s\n\n' "${ci}"
    296. 

  296. 

  297.     gg_printf 'OpenLLaMA 3B-v2:\n'
    298. 

  298.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    299. 

  299.     gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
    300. 

  300.     gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
    301. 

  301.     gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
    302. 

  302.     gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
    303. 

  303.     gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
    304. 

  304.     gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
    305. 

  305.     gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
    306. 

  306.     gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
    307. 

  307.     gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
    308. 

  308.     gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
    309. 

  309.     gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
    310. 

  310.     gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
    311. 

  311.     gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
    312. 

  312.     gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
    313. 

  313.     gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
    314. 

  314.     gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
    315. 

  315.     gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
    316. 

  316.     gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
    317. 

  317.     gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
    318. 

  318.     gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
    319. 

  319. }
    320. 

  320. 

  321. # open_llama_7b_v2
    322. 

  322. # requires: GG_BUILD_CUDA
    323. 

  323. 

  324. function gg_run_open_llama_7b_v2 {
    325. 

  325.     cd ${SRC}
    326. 

  326. 

  327.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/config.json
    328. 

  328.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/tokenizer.model
    329. 

  329.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/tokenizer_config.json
    330. 

  330.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/special_tokens_map.json
    331. 

  331.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/pytorch_model.bin.index.json
    332. 

  332.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00001-of-00002.bin
    333. 

  333.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00002-of-00002.bin
    334. 

  334.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/generation_config.json
    335. 

  335. 

  336.     gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
    337. 

  337.     unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
    338. 

  338. 

  339.     path_models="../models-mnt/open-llama/7B-v2"
    340. 

  340.     path_wiki="../models-mnt/wikitext/wikitext-2-raw"
    341. 

  341. 

  342.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    343. 

  343. 

  344.     set -e
    345. 

  345. 

  346.     (time cmake -DCMAKE_BUILD_TYPE=Release -DLLAMA_CUBLAS=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    347. 

  347.     (time make -j                                              ) 2>&1 | tee -a $OUT/${ci}-make.log
    348. 

  348. 

  349.     python3 ../convert.py ${path_models}
    350. 

  350. 

  351.     model_f16="${path_models}/ggml-model-f16.gguf"
    352. 

  352.     model_q8_0="${path_models}/ggml-model-q8_0.gguf"
    353. 

  353.     model_q4_0="${path_models}/ggml-model-q4_0.gguf"
    354. 

  354.     model_q4_1="${path_models}/ggml-model-q4_1.gguf"
    355. 

  355.     model_q5_0="${path_models}/ggml-model-q5_0.gguf"
    356. 

  356.     model_q5_1="${path_models}/ggml-model-q5_1.gguf"
    357. 

  357.     model_q2_k="${path_models}/ggml-model-q2_k.gguf"
    358. 

  358.     model_q3_k="${path_models}/ggml-model-q3_k.gguf"
    359. 

  359.     model_q4_k="${path_models}/ggml-model-q4_k.gguf"
    360. 

  360.     model_q5_k="${path_models}/ggml-model-q5_k.gguf"
    361. 

  361.     model_q6_k="${path_models}/ggml-model-q6_k.gguf"
    362. 

  362. 

  363.     wiki_test="${path_wiki}/wiki.test.raw"
    364. 

  364. 

  365.     ./bin/quantize ${model_f16} ${model_q8_0} q8_0
    366. 

  366.     ./bin/quantize ${model_f16} ${model_q4_0} q4_0
    367. 

  367.     ./bin/quantize ${model_f16} ${model_q4_1} q4_1
    368. 

  368.     ./bin/quantize ${model_f16} ${model_q5_0} q5_0
    369. 

  369.     ./bin/quantize ${model_f16} ${model_q5_1} q5_1
    370. 

  370.     ./bin/quantize ${model_f16} ${model_q2_k} q2_k
    371. 

  371.     ./bin/quantize ${model_f16} ${model_q3_k} q3_k
    372. 

  372.     ./bin/quantize ${model_f16} ${model_q4_k} q4_k
    373. 

  373.     ./bin/quantize ${model_f16} ${model_q5_k} q5_k
    374. 

  374.     ./bin/quantize ${model_f16} ${model_q6_k} q6_k
    375. 

  375. 

  376.     (time ./bin/main --model ${model_f16}  -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    377. 

  377.     (time ./bin/main --model ${model_q8_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    378. 

  378.     (time ./bin/main --model ${model_q4_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    379. 

  379.     (time ./bin/main --model ${model_q4_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    380. 

  380.     (time ./bin/main --model ${model_q5_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    381. 

  381.     (time ./bin/main --model ${model_q5_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    382. 

  382.     (time ./bin/main --model ${model_q2_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    383. 

  383.     (time ./bin/main --model ${model_q3_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    384. 

  384.     (time ./bin/main --model ${model_q4_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    385. 

  385.     (time ./bin/main --model ${model_q5_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    386. 

  386.     (time ./bin/main --model ${model_q6_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    387. 

  387. 

  388.     (time ./bin/perplexity --model ${model_f16}  -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    389. 

  389.     (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    390. 

  390.     (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    391. 

  391.     (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    392. 

  392.     (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    393. 

  393.     (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    394. 

  394.     (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    395. 

  395.     (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    396. 

  396.     (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    397. 

  397.     (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    398. 

  398.     (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    399. 

  399. 

  400.     (time ./bin/imatrix --model ${model_f16} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
    401. 

  401. 

  402.     (time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
    403. 

  403. 

  404.     function check_ppl {
    405. 

  405.         qnt="$1"
    406. 

  406.         ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    407. 

  407. 

  408.         if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
    409. 

  409.             printf '  - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
    410. 

  410.             return 20
    411. 

  411.         fi
    412. 

  412. 

  413.         printf '  - %s @ %s OK\n' "$qnt" "$ppl"
    414. 

  414.         return 0
    415. 

  415.     }
    416. 

  416. 

  417.     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    418. 

  418.     check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    419. 

  419.     check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    420. 

  420.     check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    421. 

  421.     check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    422. 

  422.     check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    423. 

  423.     check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    424. 

  424.     check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    425. 

  425.     check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    426. 

  426.     check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    427. 

  427.     check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    428. 

  428. 

  429.     cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
    430. 

  430. 

  431.     # lora
    432. 

  432.     function compare_ppl {
    433. 

  433.         qnt="$1"
    434. 

  434.         ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    435. 

  435.         ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    436. 

  436. 

  437.         if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
    438. 

  438.             printf '  - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
    439. 

  439.             return 20
    440. 

  440.         fi
    441. 

  441. 

  442.         printf '  - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
    443. 

  443.         return 0
    444. 

  444.     }
    445. 

  445. 

  446.     path_lora="../models-mnt/open-llama/7B-v2/lora"
    447. 

  447.     path_shakespeare="../models-mnt/shakespeare"
    448. 

  448. 

  449.     shakespeare="${path_shakespeare}/shakespeare.txt"
    450. 

  450.     lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
    451. 

  451. 

  452.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_config.json
    453. 

  453.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_model.bin
    454. 

  454.     gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/shakespeare.txt
    455. 

  455. 

  456.     python3 ../convert-lora-to-ggml.py ${path_lora}
    457. 

  457. 

  458.     # f16
    459. 

  459.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare}                            -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
    460. 

  460.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
    461. 

  461.     compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    462. 

  462. 

  463.     # currently not supported by the CUDA backend
    464. 

  464.     # q8_0
    465. 

  465.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare}                            -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
    466. 

  466.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
    467. 

  467.     #compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    468. 

  468. 

  469.     # q8_0 + f16 lora-base
    470. 

  470.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
    471. 

  471.     #compare_ppl "q8_0 / f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    472. 

  472. 

  473.     set +e
    474. 

  474. }
    475. 

  475. 

  476. function gg_sum_open_llama_7b_v2 {
    477. 

  477.     gg_printf '### %s\n\n' "${ci}"
    478. 

  478. 

  479.     gg_printf 'OpenLLaMA 7B-v2:\n'
    480. 

  480.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    481. 

  481.     gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
    482. 

  482.     gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
    483. 

  483.     gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
    484. 

  484.     gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
    485. 

  485.     gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
    486. 

  486.     gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
    487. 

  487.     gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
    488. 

  488.     gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
    489. 

  489.     gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
    490. 

  490.     gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
    491. 

  491.     gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
    492. 

  492.     gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
    493. 

  493.     gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
    494. 

  494.     gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
    495. 

  495.     gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
    496. 

  496.     gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
    497. 

  497.     gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
    498. 

  498.     #gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
    499. 

  499.     #gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
    500. 

  500.     #gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
    501. 

  501. }
    502. 

  502. 

  503. ## main
    504. 

  504. 

  505. if [ -z ${GG_BUILD_LOW_PERF} ]; then
    506. 

  506.     rm -rf ${SRC}/models-mnt
    507. 

  507. 

  508.     mnt_models=${MNT}/models
    509. 

  509.     mkdir -p ${mnt_models}
    510. 

  510.     ln -sfn ${mnt_models} ${SRC}/models-mnt
    511. 

  511. 

  512.     python3 -m pip install -r ${SRC}/requirements.txt
    513. 

  513.     python3 -m pip install --editable gguf-py
    514. 

  514. fi
    515. 

  515. 

  516. ret=0
    517. 

  517. 

  518. test $ret -eq 0 && gg_run ctest_debug
    519. 

  519. test $ret -eq 0 && gg_run ctest_release
    520. 

  520. 

  521. if [ -z ${GG_BUILD_LOW_PERF} ]; then
    522. 

  522.     if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 8 ]; then
    523. 

  523.         if [ -z ${GG_BUILD_CUDA} ]; then
    524. 

  524.             test $ret -eq 0 && gg_run open_llama_3b_v2
    525. 

  525.         else
    526. 

  526.             test $ret -eq 0 && gg_run open_llama_7b_v2
    527. 

  527.         fi
    528. 

  528.     fi
    529. 

  529. fi
    530. 

  530. 

  531. exit $ret
    532.