Home Explore Help
Sign In
cturan
/
llama.cpp
mirror of https://github.com/cturan/llama.cpp
1
0
Fork 0
Files Issues 0 Wiki
Tree: 70c29da118
Branches Tags
llama.cpp
/
ci
/
run.sh

run.sh 26 KB
History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506 
  1. #/bin/bash
    2. 

  2. #
    3. 

  3. # sample usage:
    4. 

  4. #
    5. 

  5. # mkdir tmp
    6. 

  6. #
    7. 

  7. # # CPU-only build
    8. 

  8. # bash ./ci/run.sh ./tmp/results ./tmp/mnt
    9. 

  9. #
    10. 

  10. # # with CUDA support
    11. 

  11. # GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
    12. 

  12. #
    13. 

  13. 

  14. if [ -z "$2" ]; then
    15. 

  15.     echo "usage: $0 <output-dir> <mnt-dir>"
    16. 

  16.     exit 1
    17. 

  17. fi
    18. 

  18. 

  19. mkdir -p "$1"
    20. 

  20. mkdir -p "$2"
    21. 

  21. 

  22. OUT=$(realpath "$1")
    23. 

  23. MNT=$(realpath "$2")
    24. 

  24. 

  25. rm -v $OUT/*.log
    26. 

  26. rm -v $OUT/*.exit
    27. 

  27. rm -v $OUT/*.md
    28. 

  28. 

  29. sd=`dirname $0`
    30. 

  30. cd $sd/../
    31. 

  31. SRC=`pwd`
    32. 

  32. 

  33. ## helpers
    34. 

  34. 

  35. # download a file if it does not exist or if it is outdated
    36. 

  36. function gg_wget {
    37. 

  37.     local out=$1
    38. 

  38.     local url=$2
    39. 

  39. 

  40.     local cwd=`pwd`
    41. 

  41. 

  42.     mkdir -p $out
    43. 

  43.     cd $out
    44. 

  44. 

  45.     # should not re-download if file is the same
    46. 

  46.     wget -nv -N $url
    47. 

  47. 

  48.     cd $cwd
    49. 

  49. }
    50. 

  50. 

  51. function gg_printf {
    52. 

  52.     printf -- "$@" >> $OUT/README.md
    53. 

  53. }
    54. 

  54. 

  55. function gg_run {
    56. 

  56.     ci=$1
    57. 

  57. 

  58.     set -o pipefail
    59. 

  59.     set -x
    60. 

  60. 

  61.     gg_run_$ci | tee $OUT/$ci.log
    62. 

  62.     cur=$?
    63. 

  63.     echo "$cur" > $OUT/$ci.exit
    64. 

  64. 

  65.     set +x
    66. 

  66.     set +o pipefail
    67. 

  67. 

  68.     gg_sum_$ci
    69. 

  69. 

  70.     ret=$((ret | cur))
    71. 

  71. }
    72. 

  72. 

  73. ## ci
    74. 

  74. 

  75. # ctest_debug
    76. 

  76. 

  77. function gg_run_ctest_debug {
    78. 

  78.     cd ${SRC}
    79. 

  79. 

  80.     rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
    81. 

  81. 

  82.     set -e
    83. 

  83. 

  84.     (time cmake -DCMAKE_BUILD_TYPE=Debug ..     ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    85. 

  85.     (time make -j                               ) 2>&1 | tee -a $OUT/${ci}-make.log
    86. 

  86. 

  87.     (time ctest --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    88. 

  88. 

  89.     set +e
    90. 

  90. }
    91. 

  91. 

  92. function gg_sum_ctest_debug {
    93. 

  93.     gg_printf '### %s\n\n' "${ci}"
    94. 

  94. 

  95.     gg_printf 'Runs ctest in debug mode\n'
    96. 

  96.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    97. 

  97.     gg_printf '```\n'
    98. 

  98.     gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
    99. 

  99.     gg_printf '```\n'
    100. 

  100.     gg_printf '\n'
    101. 

  101. }
    102. 

  102. 

  103. # ctest_release
    104. 

  104. 

  105. function gg_run_ctest_release {
    106. 

  106.     cd ${SRC}
    107. 

  107. 

  108.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    109. 

  109. 

  110.     set -e
    111. 

  111. 

  112.     (time cmake -DCMAKE_BUILD_TYPE=Release ..   ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    113. 

  113.     (time make -j                               ) 2>&1 | tee -a $OUT/${ci}-make.log
    114. 

  114. 

  115.     if [ -z ${GG_BUILD_LOW_PERF} ]; then
    116. 

  116.         (time ctest --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    117. 

  117.     else
    118. 

  118.         (time ctest --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
    119. 

  119.     fi
    120. 

  120. 

  121.     set +e
    122. 

  122. }
    123. 

  123. 

  124. function gg_sum_ctest_release {
    125. 

  125.     gg_printf '### %s\n\n' "${ci}"
    126. 

  126. 

  127.     gg_printf 'Runs ctest in release mode\n'
    128. 

  128.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    129. 

  129.     gg_printf '```\n'
    130. 

  130.     gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
    131. 

  131.     gg_printf '```\n'
    132. 

  132. }
    133. 

  133. 

  134. # open_llama_3b_v2
    135. 

  135. 

  136. function gg_run_open_llama_3b_v2 {
    137. 

  137.     cd ${SRC}
    138. 

  138. 

  139.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/config.json
    140. 

  140.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/tokenizer.model
    141. 

  141.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/tokenizer_config.json
    142. 

  142.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/special_tokens_map.json
    143. 

  143.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/pytorch_model.bin
    144. 

  144.     gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/generation_config.json
    145. 

  145. 

  146.     gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
    147. 

  147.     unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
    148. 

  148.     head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw
    149. 

  149. 

  150.     path_models="../models-mnt/open-llama/3B-v2"
    151. 

  151.     path_wiki="../models-mnt/wikitext/wikitext-2-raw"
    152. 

  152. 

  153.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    154. 

  154. 

  155.     set -e
    156. 

  156. 

  157.     (time cmake -DCMAKE_BUILD_TYPE=Release -DLLAMA_QKK_64=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    158. 

  158.     (time make -j                                              ) 2>&1 | tee -a $OUT/${ci}-make.log
    159. 

  159. 

  160.     python3 ../convert.py ${path_models}
    161. 

  161. 

  162.     model_f16="${path_models}/ggml-model-f16.gguf"
    163. 

  163.     model_q8_0="${path_models}/ggml-model-q8_0.gguf"
    164. 

  164.     model_q4_0="${path_models}/ggml-model-q4_0.gguf"
    165. 

  165.     model_q4_1="${path_models}/ggml-model-q4_1.gguf"
    166. 

  166.     model_q5_0="${path_models}/ggml-model-q5_0.gguf"
    167. 

  167.     model_q5_1="${path_models}/ggml-model-q5_1.gguf"
    168. 

  168.     model_q2_k="${path_models}/ggml-model-q2_k.gguf"
    169. 

  169.     model_q3_k="${path_models}/ggml-model-q3_k.gguf"
    170. 

  170.     model_q4_k="${path_models}/ggml-model-q4_k.gguf"
    171. 

  171.     model_q5_k="${path_models}/ggml-model-q5_k.gguf"
    172. 

  172.     model_q6_k="${path_models}/ggml-model-q6_k.gguf"
    173. 

  173. 

  174.     wiki_test_60="${path_wiki}/wiki.test-60.raw"
    175. 

  175. 

  176.     ./bin/quantize ${model_f16} ${model_q8_0} q8_0
    177. 

  177.     ./bin/quantize ${model_f16} ${model_q4_0} q4_0
    178. 

  178.     ./bin/quantize ${model_f16} ${model_q4_1} q4_1
    179. 

  179.     ./bin/quantize ${model_f16} ${model_q5_0} q5_0
    180. 

  180.     ./bin/quantize ${model_f16} ${model_q5_1} q5_1
    181. 

  181.     ./bin/quantize ${model_f16} ${model_q2_k} q2_k
    182. 

  182.     ./bin/quantize ${model_f16} ${model_q3_k} q3_k
    183. 

  183.     ./bin/quantize ${model_f16} ${model_q4_k} q4_k
    184. 

  184.     ./bin/quantize ${model_f16} ${model_q5_k} q5_k
    185. 

  185.     ./bin/quantize ${model_f16} ${model_q6_k} q6_k
    186. 

  186. 

  187.     (time ./bin/main --model ${model_f16}  -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    188. 

  188.     (time ./bin/main --model ${model_q8_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    189. 

  189.     (time ./bin/main --model ${model_q4_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    190. 

  190.     (time ./bin/main --model ${model_q4_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    191. 

  191.     (time ./bin/main --model ${model_q5_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    192. 

  192.     (time ./bin/main --model ${model_q5_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    193. 

  193.     (time ./bin/main --model ${model_q2_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    194. 

  194.     (time ./bin/main --model ${model_q3_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    195. 

  195.     (time ./bin/main --model ${model_q4_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    196. 

  196.     (time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    197. 

  197.     (time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    198. 

  198. 

  199.     (time ./bin/perplexity --model ${model_f16}  -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    200. 

  200.     (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    201. 

  201.     (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    202. 

  202.     (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    203. 

  203.     (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    204. 

  204.     (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    205. 

  205.     (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    206. 

  206.     (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    207. 

  207.     (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    208. 

  208.     (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    209. 

  209.     (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    210. 

  210. 

  211.     function check_ppl {
    212. 

  212.         qnt="$1"
    213. 

  213.         ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    214. 

  214. 

  215.         if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
    216. 

  216.             printf '  - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
    217. 

  217.             return 20
    218. 

  218.         fi
    219. 

  219. 

  220.         printf '  - %s @ %s OK\n' "$qnt" "$ppl"
    221. 

  221.         return 0
    222. 

  222.     }
    223. 

  223. 

  224.     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    225. 

  225.     check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    226. 

  226.     check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    227. 

  227.     check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    228. 

  228.     check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    229. 

  229.     check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    230. 

  230.     check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    231. 

  231.     check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    232. 

  232.     check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    233. 

  233.     check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    234. 

  234.     check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    235. 

  235. 

  236.     # lora
    237. 

  237.     function compare_ppl {
    238. 

  238.         qnt="$1"
    239. 

  239.         ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    240. 

  240.         ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    241. 

  241. 

  242.         if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
    243. 

  243.             printf '  - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
    244. 

  244.             return 20
    245. 

  245.         fi
    246. 

  246. 

  247.         printf '  - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
    248. 

  248.         return 0
    249. 

  249.     }
    250. 

  250. 

  251.     path_lora="../models-mnt/open-llama/3B-v2/lora"
    252. 

  252.     path_shakespeare="../models-mnt/shakespeare"
    253. 

  253. 

  254.     shakespeare="${path_shakespeare}/shakespeare.txt"
    255. 

  255.     lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
    256. 

  256. 

  257.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_config.json
    258. 

  258.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_model.bin
    259. 

  259.     gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/shakespeare.txt
    260. 

  260. 

  261.     python3 ../convert-lora-to-ggml.py ${path_lora}
    262. 

  262. 

  263.     # f16
    264. 

  264.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare}                            -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
    265. 

  265.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
    266. 

  266.     compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    267. 

  267. 

  268.     # q8_0
    269. 

  269.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare}                            -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
    270. 

  270.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
    271. 

  271.     compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    272. 

  272. 

  273.     # q8_0 + f16 lora-base
    274. 

  274.     (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
    275. 

  275.     compare_ppl "q8_0 / f16 base shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    276. 

  276. 

  277. 

  278.     set +e
    279. 

  279. }
    280. 

  280. 

  281. function gg_sum_open_llama_3b_v2 {
    282. 

  282.     gg_printf '### %s\n\n' "${ci}"
    283. 

  283. 

  284.     gg_printf 'OpenLLaMA 3B-v2:\n'
    285. 

  285.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    286. 

  286.     gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
    287. 

  287.     gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
    288. 

  288.     gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
    289. 

  289.     gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
    290. 

  290.     gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
    291. 

  291.     gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
    292. 

  292.     gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
    293. 

  293.     gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
    294. 

  294.     gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
    295. 

  295.     gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
    296. 

  296.     gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
    297. 

  297.     gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
    298. 

  298.     gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
    299. 

  299.     gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
    300. 

  300.     gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
    301. 

  301.     gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
    302. 

  302.     gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
    303. 

  303.     gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
    304. 

  304. }
    305. 

  305. 

  306. # open_llama_7b_v2
    307. 

  307. # requires: GG_BUILD_CUDA
    308. 

  308. 

  309. function gg_run_open_llama_7b_v2 {
    310. 

  310.     cd ${SRC}
    311. 

  311. 

  312.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/config.json
    313. 

  313.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/tokenizer.model
    314. 

  314.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/tokenizer_config.json
    315. 

  315.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/special_tokens_map.json
    316. 

  316.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/pytorch_model.bin.index.json
    317. 

  317.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00001-of-00002.bin
    318. 

  318.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00002-of-00002.bin
    319. 

  319.     gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/generation_config.json
    320. 

  320. 

  321.     gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
    322. 

  322.     unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
    323. 

  323. 

  324.     path_models="../models-mnt/open-llama/7B-v2"
    325. 

  325.     path_wiki="../models-mnt/wikitext/wikitext-2-raw"
    326. 

  326. 

  327.     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
    328. 

  328. 

  329.     set -e
    330. 

  330. 

  331.     (time cmake -DCMAKE_BUILD_TYPE=Release -DLLAMA_CUBLAS=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
    332. 

  332.     (time make -j                                              ) 2>&1 | tee -a $OUT/${ci}-make.log
    333. 

  333. 

  334.     python3 ../convert.py ${path_models}
    335. 

  335. 

  336.     model_f16="${path_models}/ggml-model-f16.gguf"
    337. 

  337.     model_q8_0="${path_models}/ggml-model-q8_0.gguf"
    338. 

  338.     model_q4_0="${path_models}/ggml-model-q4_0.gguf"
    339. 

  339.     model_q4_1="${path_models}/ggml-model-q4_1.gguf"
    340. 

  340.     model_q5_0="${path_models}/ggml-model-q5_0.gguf"
    341. 

  341.     model_q5_1="${path_models}/ggml-model-q5_1.gguf"
    342. 

  342.     model_q2_k="${path_models}/ggml-model-q2_k.gguf"
    343. 

  343.     model_q3_k="${path_models}/ggml-model-q3_k.gguf"
    344. 

  344.     model_q4_k="${path_models}/ggml-model-q4_k.gguf"
    345. 

  345.     model_q5_k="${path_models}/ggml-model-q5_k.gguf"
    346. 

  346.     model_q6_k="${path_models}/ggml-model-q6_k.gguf"
    347. 

  347. 

  348.     wiki_test="${path_wiki}/wiki.test.raw"
    349. 

  349. 

  350.     ./bin/quantize ${model_f16} ${model_q8_0} q8_0
    351. 

  351.     ./bin/quantize ${model_f16} ${model_q4_0} q4_0
    352. 

  352.     ./bin/quantize ${model_f16} ${model_q4_1} q4_1
    353. 

  353.     ./bin/quantize ${model_f16} ${model_q5_0} q5_0
    354. 

  354.     ./bin/quantize ${model_f16} ${model_q5_1} q5_1
    355. 

  355.     ./bin/quantize ${model_f16} ${model_q2_k} q2_k
    356. 

  356.     ./bin/quantize ${model_f16} ${model_q3_k} q3_k
    357. 

  357.     ./bin/quantize ${model_f16} ${model_q4_k} q4_k
    358. 

  358.     ./bin/quantize ${model_f16} ${model_q5_k} q5_k
    359. 

  359.     ./bin/quantize ${model_f16} ${model_q6_k} q6_k
    360. 

  360. 

  361.     (time ./bin/main --model ${model_f16}  -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    362. 

  362.     (time ./bin/main --model ${model_q8_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    363. 

  363.     (time ./bin/main --model ${model_q4_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    364. 

  364.     (time ./bin/main --model ${model_q4_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    365. 

  365.     (time ./bin/main --model ${model_q5_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    366. 

  366.     (time ./bin/main --model ${model_q5_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    367. 

  367.     (time ./bin/main --model ${model_q2_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    368. 

  368.     (time ./bin/main --model ${model_q3_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    369. 

  369.     (time ./bin/main --model ${model_q4_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    370. 

  370.     (time ./bin/main --model ${model_q5_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    371. 

  371.     (time ./bin/main --model ${model_q6_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    372. 

  372. 

  373.     (time ./bin/perplexity --model ${model_f16}  -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
    374. 

  374.     (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
    375. 

  375.     (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
    376. 

  376.     (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
    377. 

  377.     (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
    378. 

  378.     (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
    379. 

  379.     (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
    380. 

  380.     (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
    381. 

  381.     (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
    382. 

  382.     (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
    383. 

  383.     (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
    384. 

  384. 

  385.     function check_ppl {
    386. 

  386.         qnt="$1"
    387. 

  387.         ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    388. 

  388. 

  389.         if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
    390. 

  390.             printf '  - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
    391. 

  391.             return 20
    392. 

  392.         fi
    393. 

  393. 

  394.         printf '  - %s @ %s OK\n' "$qnt" "$ppl"
    395. 

  395.         return 0
    396. 

  396.     }
    397. 

  397. 

  398.     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    399. 

  399.     check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    400. 

  400.     check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    401. 

  401.     check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    402. 

  402.     check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    403. 

  403.     check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    404. 

  404.     check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    405. 

  405.     check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    406. 

  406.     check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    407. 

  407.     check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    408. 

  408.     check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
    409. 

  409. 

  410.     # lora
    411. 

  411.     function compare_ppl {
    412. 

  412.         qnt="$1"
    413. 

  413.         ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    414. 

  414.         ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
    415. 

  415. 

  416.         if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
    417. 

  417.             printf '  - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
    418. 

  418.             return 20
    419. 

  419.         fi
    420. 

  420. 

  421.         printf '  - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
    422. 

  422.         return 0
    423. 

  423.     }
    424. 

  424. 

  425.     path_lora="../models-mnt/open-llama/7B-v2/lora"
    426. 

  426.     path_shakespeare="../models-mnt/shakespeare"
    427. 

  427. 

  428.     shakespeare="${path_shakespeare}/shakespeare.txt"
    429. 

  429.     lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
    430. 

  430. 

  431.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_config.json
    432. 

  432.     gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_model.bin
    433. 

  433.     gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/shakespeare.txt
    434. 

  434. 

  435.     python3 ../convert-lora-to-ggml.py ${path_lora}
    436. 

  436. 

  437.     # f16
    438. 

  438.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare}                            -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
    439. 

  439.     (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
    440. 

  440.     compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    441. 

  441. 

  442.     # currently not supported by the CUDA backend
    443. 

  443.     # q8_0
    444. 

  444.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare}                            -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
    445. 

  445.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
    446. 

  446.     #compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    447. 

  447. 

  448.     # q8_0 + f16 lora-base
    449. 

  449.     #(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
    450. 

  450.     #compare_ppl "q8_0 / f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
    451. 

  451. 

  452.     set +e
    453. 

  453. }
    454. 

  454. 

  455. function gg_sum_open_llama_7b_v2 {
    456. 

  456.     gg_printf '### %s\n\n' "${ci}"
    457. 

  457. 

  458.     gg_printf 'OpenLLaMA 7B-v2:\n'
    459. 

  459.     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
    460. 

  460.     gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
    461. 

  461.     gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
    462. 

  462.     gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
    463. 

  463.     gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
    464. 

  464.     gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
    465. 

  465.     gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
    466. 

  466.     gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
    467. 

  467.     gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
    468. 

  468.     gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
    469. 

  469.     gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
    470. 

  470.     gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
    471. 

  471.     gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
    472. 

  472.     gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
    473. 

  473.     gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
    474. 

  474.     gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
    475. 

  475.     #gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
    476. 

  476.     #gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
    477. 

  477.     #gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
    478. 

  478. }
    479. 

  479. 

  480. ## main
    481. 

  481. 

  482. if [ -z ${GG_BUILD_LOW_PERF} ]; then
    483. 

  483.     rm -rf ${SRC}/models-mnt
    484. 

  484. 

  485.     mnt_models=${MNT}/models
    486. 

  486.     mkdir -p ${mnt_models}
    487. 

  487.     ln -sfn ${mnt_models} ${SRC}/models-mnt
    488. 

  488. 

  489.     python3 -m pip install -r ${SRC}/requirements.txt
    490. 

  490.     python3 -m pip install --editable gguf-py
    491. 

  491. fi
    492. 

  492. 

  493. ret=0
    494. 

  494. 

  495. test $ret -eq 0 && gg_run ctest_debug
    496. 

  496. test $ret -eq 0 && gg_run ctest_release
    497. 

  497. 

  498. if [ -z ${GG_BUILD_LOW_PERF} ]; then
    499. 

  499.     if [ -z ${GG_BUILD_CUDA} ]; then
    500. 

  500.         test $ret -eq 0 && gg_run open_llama_3b_v2
    501. 

  501.     else
    502. 

  502.         test $ret -eq 0 && gg_run open_llama_7b_v2
    503. 

  503.     fi
    504. 

  504. fi
    505. 

  505. 

  506. exit $ret
    507.