|
|
@@ -144,6 +144,7 @@ def main() -> None:
|
|
|
parser.add_argument("--general-description", type=str, help="The models general.description", metavar='"Description ..."')
|
|
|
parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)", metavar='"{% ... %} ..."')
|
|
|
parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat template(s)", metavar='tokenizer_config.json')
|
|
|
+ parser.add_argument("--pre-tokenizer", type=str, help="The models tokenizer.ggml.pre", metavar='"pre tokenizer"')
|
|
|
parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model", metavar='general.url')
|
|
|
parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '"<token>"'))
|
|
|
parser.add_argument("--special-token-by-id", action="append", type=str, help="Special token by id", nargs=2, metavar=(' | '.join(token_names.keys()), '0'))
|
|
|
@@ -172,6 +173,9 @@ def main() -> None:
|
|
|
if template:
|
|
|
new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, template)
|
|
|
|
|
|
+ if args.pre_tokenizer:
|
|
|
+ new_metadata[gguf.Keys.Tokenizer.PRE] = MetadataDetails(gguf.GGUFValueType.STRING, args.pre_tokenizer)
|
|
|
+
|
|
|
if remove_metadata:
|
|
|
logger.warning('*** Warning *** Warning *** Warning **')
|
|
|
logger.warning('* Most metadata is required for a fully functional GGUF file,')
|
Galunid