|
|
@@ -1942,7 +1942,7 @@ class Phi3MiniModel(Model):
|
|
|
if len(rope_scaling_type) == 0:
|
|
|
raise KeyError('Missing the required key rope_scaling.type')
|
|
|
|
|
|
- if rope_scaling_type == 'su':
|
|
|
+ if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':
|
|
|
attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0
|
|
|
elif rope_scaling_type == 'yarn':
|
|
|
attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0
|