|
@@ -557,6 +557,10 @@ python3 convert.py models/7B/
|
|
|
# quantize the model to 4-bits (using q4_0 method)
|
|
# quantize the model to 4-bits (using q4_0 method)
|
|
|
./quantize ./models/7B/ggml-model-f16.gguf ./models/7B/ggml-model-q4_0.gguf q4_0
|
|
./quantize ./models/7B/ggml-model-f16.gguf ./models/7B/ggml-model-q4_0.gguf q4_0
|
|
|
|
|
|
|
|
|
|
+# update the gguf filetype to current if older version is unsupported by another application
|
|
|
|
|
+./quantize ./models/7B/ggml-model-q4_0.gguf ./models/7B/ggml-model-q4_0-v2.gguf COPY
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
# run the inference
|
|
# run the inference
|
|
|
./main -m ./models/7B/ggml-model-q4_0.gguf -n 128
|
|
./main -m ./models/7B/ggml-model-q4_0.gguf -n 128
|
|
|
```
|
|
```
|