Signed-off-by: Jiri Podivin <jpodivin@gmail.com> Co-authored-by: Jiri Podivin <jpodivin@redhat.com>
@@ -6,11 +6,9 @@ from __future__ import annotations
import argparse
import json
import os
-import struct
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Any
-import itertools
import numpy as np
import torch
from sentencepiece import SentencePieceProcessor # type: ignore[import]
@@ -2,7 +2,6 @@
from __future__ import annotations
-import math
import struct
from enum import IntEnum
@@ -3,9 +3,7 @@
import gguf
-import os
-import sys
@@ -1,7 +1,5 @@
# tests with BPE tokenizer
from transformers import AutoTokenizer
# tests with SPM tokenizer
from sentencepiece import SentencePieceProcessor