namespace-Pt
/

activation-beacon-llama2-7b-chat

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

namespace-Pt commited on Jan 28

Commit

b6abf69

•

1 Parent(s): c06f207

Upload config.json with huggingface_hub

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -11,9 +11,9 @@
   "attention_dropout": 0.0,
   "beacon_attn": "step-expansion",
   "beacon_layers": null,
   "beacon_ratio": [2,4,8,16,32,64,128],
   "beacon_ratio_mix": "adapt-1024",
-  "beacon_seed": 42,
   "beacon_stride": [1024],
   "beacon_stride_mix": "step-random",
   "beacon_window": 1024,
@@ -23,7 +23,7 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 11008,
-  "max_position_embeddings": 400000,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,

   "attention_dropout": 0.0,
   "beacon_attn": "step-expansion",
   "beacon_layers": null,
+  "beacon_param": ["q", "k", "v", "o"],
   "beacon_ratio": [2,4,8,16,32,64,128],
   "beacon_ratio_mix": "adapt-1024",
   "beacon_stride": [1024],
   "beacon_stride_mix": "step-random",
   "beacon_window": 1024,
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,