|
@@ -110,7 +110,6 @@ class SafetensorRemote:
|
|
|
"""
|
|
"""
|
|
|
|
|
|
|
|
BASE_DOMAIN = "https://huggingface.co"
|
|
BASE_DOMAIN = "https://huggingface.co"
|
|
|
- ALIGNMENT = 8 # bytes
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
|
def get_list_tensors_hf_model(cls, model_id: str) -> dict[str, RemoteTensor]:
|
|
def get_list_tensors_hf_model(cls, model_id: str) -> dict[str, RemoteTensor]:
|
|
@@ -204,9 +203,6 @@ class SafetensorRemote:
|
|
|
|
|
|
|
|
# Calculate the data start offset
|
|
# Calculate the data start offset
|
|
|
data_start_offset = 8 + metadata_length
|
|
data_start_offset = 8 + metadata_length
|
|
|
- alignment = SafetensorRemote.ALIGNMENT
|
|
|
|
|
- if data_start_offset % alignment != 0:
|
|
|
|
|
- data_start_offset += alignment - (data_start_offset % alignment)
|
|
|
|
|
|
|
|
|
|
# Check if we have enough data to read the metadata
|
|
# Check if we have enough data to read the metadata
|
|
|
if len(raw_data) < 8 + metadata_length:
|
|
if len(raw_data) < 8 + metadata_length:
|
|
@@ -298,7 +294,6 @@ class SafetensorsLocal:
|
|
|
Custom parsing gives a bit more control over the memory usage.
|
|
Custom parsing gives a bit more control over the memory usage.
|
|
|
The official safetensors library doesn't expose file ranges.
|
|
The official safetensors library doesn't expose file ranges.
|
|
|
"""
|
|
"""
|
|
|
- ALIGNMENT = 8 # bytes
|
|
|
|
|
|
|
|
|
|
tensors: dict[str, LocalTensor]
|
|
tensors: dict[str, LocalTensor]
|
|
|
|
|
|
|
@@ -316,9 +311,6 @@ class SafetensorsLocal:
|
|
|
raise ValueError(f"Failed to parse safetensors metadata as JSON: {e}")
|
|
raise ValueError(f"Failed to parse safetensors metadata as JSON: {e}")
|
|
|
|
|
|
|
|
data_start_offset = f.tell()
|
|
data_start_offset = f.tell()
|
|
|
- alignment = self.ALIGNMENT
|
|
|
|
|
- if data_start_offset % alignment != 0:
|
|
|
|
|
- data_start_offset += alignment - (data_start_offset % alignment)
|
|
|
|
|
|
|
|
|
|
tensors: dict[str, LocalTensor] = {}
|
|
tensors: dict[str, LocalTensor] = {}
|
|
|
for name, meta in metadata.items():
|
|
for name, meta in metadata.items():
|