Upload configuration_llama.py with huggingface_hub

configuration_llama.py

@@ -143,8 +143,9 @@ class LlamaConfig(PretrainedConfig):
         beacon_attend_previous=True,
         beacon_ratio=[8],
         beacon_ratio_mix="step-random",
-        beacon_seed=42,
-        beacon_layers=None,
+        beacon_param=["q","k","v","o"],
+        retrieval_method=None,
+        retrieval_topk=None,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -177,9 +178,9 @@ class LlamaConfig(PretrainedConfig):
         self.beacon_ratio = beacon_ratio
         self.beacon_stride_mix = beacon_stride_mix
         self.beacon_ratio_mix = beacon_ratio_mix
-        self.beacon_seed = beacon_seed
-        self.beacon_layers = beacon_layers
-        self._beacon_validation()
+        self.beacon_param = beacon_param
+        self.retrieval_method = retrieval_method
+        self.retrieval_topk = retrieval_topk
 
         super().__init__(
             pad_token_id=pad_token_id,
@@ -210,10 +211,3 @@ class LlamaConfig(PretrainedConfig):
             )
         if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
             raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")
-
-    def _beacon_validation(self):
-        for stride in self.beacon_stride:
-            assert self.beacon_window >= stride, f"Make sure the beacon_window {self.beacon_window} >= beacon_stride {stride}!"
-        assert self.beacon_attn in ["segmentation", "step-expansion", "full-coverage"], f"beacon_attn {self.beacon_attn} not implemented!"
-        assert self.beacon_stride_mix in ["instance-random", "step-random", "mix-random"], f"beacon_stride_mix {self.beacon_stride_mix} not implemented!"
-        assert self.beacon_ratio_mix in ["instance-random", "step-random", "mix-random"] or "adapt-" in self.beacon_ratio_mix, f"beacon_ratio_mix {self.beacon_ratio_mix} not implemented!"