Update modeling_nano.py
Browse files- modeling_nano.py +1 -1
modeling_nano.py
CHANGED
|
@@ -94,7 +94,7 @@ class NanoAttention(nn.Module):
|
|
| 94 |
elif scaling_type == "dynamic":
|
| 95 |
self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
|
| 96 |
self.head_dim,
|
| 97 |
-
max_position_embeddings=self.max_position_embeddings,
|
| 98 |
scaling_factor=scaling_factor,
|
| 99 |
base=self.config.rope_theta,
|
| 100 |
)
|
|
|
|
| 94 |
elif scaling_type == "dynamic":
|
| 95 |
self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
|
| 96 |
self.head_dim,
|
| 97 |
+
max_position_embeddings=self.config.max_position_embeddings,
|
| 98 |
scaling_factor=scaling_factor,
|
| 99 |
base=self.config.rope_theta,
|
| 100 |
)
|