Hugging Face's logo

nisten
/
lobotollama-368b-base

Text Generation
Transformers
Safetensors
llama
mergekit
Merge
text-generation-inference
Inference Endpoints
Model card Files Community
nisten commited on
Commit
0db0ad9
•
1 Parent(s): 066c7f2

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +70 -0
  2. config.json +35 -0
  3. mergekit_config.yml +42 -0
  4. model-00001-of-00173.safetensors +3 -0
  5. model-00002-of-00173.safetensors +3 -0
  6. model-00003-of-00173.safetensors +3 -0
  7. model-00004-of-00173.safetensors +3 -0
  8. model-00005-of-00173.safetensors +3 -0
  9. model-00006-of-00173.safetensors +3 -0
  10. model-00007-of-00173.safetensors +3 -0
  11. model-00008-of-00173.safetensors +3 -0
  12. model-00009-of-00173.safetensors +3 -0
  13. model-00010-of-00173.safetensors +3 -0
  14. model-00011-of-00173.safetensors +3 -0
  15. model-00012-of-00173.safetensors +3 -0
  16. model-00013-of-00173.safetensors +3 -0
  17. model-00014-of-00173.safetensors +3 -0
  18. model-00015-of-00173.safetensors +3 -0
  19. model-00016-of-00173.safetensors +3 -0
  20. model-00017-of-00173.safetensors +3 -0
  21. model-00018-of-00173.safetensors +3 -0
  22. model-00019-of-00173.safetensors +3 -0
  23. model-00020-of-00173.safetensors +3 -0
  24. model-00021-of-00173.safetensors +3 -0
  25. model-00022-of-00173.safetensors +3 -0
  26. model-00023-of-00173.safetensors +3 -0
  27. model-00024-of-00173.safetensors +3 -0
  28. model-00025-of-00173.safetensors +3 -0
  29. model-00026-of-00173.safetensors +3 -0
  30. model-00027-of-00173.safetensors +3 -0
  31. model-00028-of-00173.safetensors +3 -0
  32. model-00029-of-00173.safetensors +3 -0
  33. model-00030-of-00173.safetensors +3 -0
  34. model-00031-of-00173.safetensors +3 -0
  35. model-00032-of-00173.safetensors +3 -0
  36. model-00033-of-00173.safetensors +3 -0
  37. model-00034-of-00173.safetensors +3 -0
  38. model-00035-of-00173.safetensors +3 -0
  39. model-00036-of-00173.safetensors +3 -0
  40. model-00037-of-00173.safetensors +3 -0
  41. model-00038-of-00173.safetensors +3 -0
  42. model-00039-of-00173.safetensors +3 -0
  43. model-00040-of-00173.safetensors +3 -0
  44. model-00041-of-00173.safetensors +3 -0
  45. model-00042-of-00173.safetensors +3 -0
  46. model-00043-of-00173.safetensors +3 -0
  47. model-00044-of-00173.safetensors +3 -0
  48. model-00045-of-00173.safetensors +3 -0
  49. model-00046-of-00173.safetensors +3 -0
  50. model-00047-of-00173.safetensors +3 -0
README.md ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: []
3
+ library_name: transformers
4
+ tags:
5
+ - mergekit
6
+ - merge
7
+
8
+ ---
9
+ # lobotollama369
10
+
11
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
12
+
13
+ ## Merge Details
14
+ ### Merge Method
15
+
16
+ This model was merged using the passthrough merge method.
17
+
18
+ ### Models Merged
19
+
20
+ The following models were included in the merge:
21
+ * /scratch-4
22
+
23
+ ### Configuration
24
+
25
+ The following YAML configuration was used to produce this model:
26
+
27
+ ```yaml
28
+ dtype: bfloat16
29
+ merge_method: passthrough
30
+ slices:
31
+ - sources:
32
+ - layer_range: [0, 29]
33
+ model: /scratch-4
34
+ - sources:
35
+ - layer_range: [30, 35]
36
+ model: /scratch-4
37
+ - sources:
38
+ - layer_range: [36, 40]
39
+ model: /scratch-4
40
+ - sources:
41
+ - layer_range: [41, 45]
42
+ model: /scratch-4
43
+ - sources:
44
+ - layer_range: [46, 49]
45
+ model: /scratch-4
46
+ - sources:
47
+ - layer_range: [50, 54]
48
+ model: /scratch-4
49
+ - sources:
50
+ - layer_range: [55, 59]
51
+ model: /scratch-4
52
+ - sources:
53
+ - layer_range: [60, 64]
54
+ model: /scratch-4
55
+ - sources:
56
+ - layer_range: [65, 69]
57
+ model: /scratch-4
58
+ - sources:
59
+ - layer_range: [70, 74]
60
+ model: /scratch-4
61
+ - sources:
62
+ - layer_range: [75, 79]
63
+ model: /scratch-4
64
+ - sources:
65
+ - layer_range: [80, 84]
66
+ model: /scratch-4
67
+ - sources:
68
+ - layer_range: [85, 126]
69
+ model: /scratch-4
70
+ ```
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/scratch-4",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128001,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 16384,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 53248,
14
+ "max_position_embeddings": 131072,
15
+ "mlp_bias": false,
16
+ "model_type": "llama",
17
+ "num_attention_heads": 128,
18
+ "num_hidden_layers": 114,
19
+ "num_key_value_heads": 8,
20
+ "pretraining_tp": 1,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_scaling": {
23
+ "factor": 8.0,
24
+ "high_freq_factor": 4.0,
25
+ "low_freq_factor": 1.0,
26
+ "original_max_position_embeddings": 8192,
27
+ "rope_type": "llama3"
28
+ },
29
+ "rope_theta": 500000.0,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "bfloat16",
32
+ "transformers_version": "4.44.1",
33
+ "use_cache": true,
34
+ "vocab_size": 128256
35
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dtype: bfloat16
2
+ merge_method: passthrough
3
+ slices:
4
+ - sources:
5
+ - layer_range: [0, 29]
6
+ model: /scratch-4
7
+ - sources:
8
+ - layer_range: [30, 35]
9
+ model: /scratch-4
10
+ - sources:
11
+ - layer_range: [36, 40]
12
+ model: /scratch-4
13
+ - sources:
14
+ - layer_range: [41, 45]
15
+ model: /scratch-4
16
+ - sources:
17
+ - layer_range: [46, 49]
18
+ model: /scratch-4
19
+ - sources:
20
+ - layer_range: [50, 54]
21
+ model: /scratch-4
22
+ - sources:
23
+ - layer_range: [55, 59]
24
+ model: /scratch-4
25
+ - sources:
26
+ - layer_range: [60, 64]
27
+ model: /scratch-4
28
+ - sources:
29
+ - layer_range: [65, 69]
30
+ model: /scratch-4
31
+ - sources:
32
+ - layer_range: [70, 74]
33
+ model: /scratch-4
34
+ - sources:
35
+ - layer_range: [75, 79]
36
+ model: /scratch-4
37
+ - sources:
38
+ - layer_range: [80, 84]
39
+ model: /scratch-4
40
+ - sources:
41
+ - layer_range: [85, 126]
42
+ model: /scratch-4
model-00001-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c19b417dcc7fd337f97811ce0cd49f33750240019dc873802e7e15b6ee63b428
3
+ size 4202692736
model-00002-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77065ad35c40719fa9761102ccd68820d9406639e15a5ae507892da160a383c4
3
+ size 4202725632
model-00003-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c5a6c11269864c010c9bf4000aa693796baf20f9e4b9c93889413b02ea6a4be
3
+ size 3489661192
model-00004-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:660751b36bef946ef437b53af958fa83a658a5f7692d8a8af906b3c476ba63d6
3
+ size 4630578120
model-00005-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfe7acb4e64cf4f9390595e02d192be248aa1ca6032818594879310e9818b411
3
+ size 4630578128
model-00006-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04712b55ece1385b5e139b382e76e74ca21bf70e5499b2a7d45fa028870f5717
3
+ size 3489661200
model-00007-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bace405bd8ca6a758fbaa84c2a3bdbe058cc039e8d0ab4987485c4af829d4689
3
+ size 4630578128
model-00008-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7feedba9992087976730bcef6c8e863aff75cd59a2d57b433d6342d04cac9db8
3
+ size 4630578128
model-00009-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70d718fcb8260f3ce6e03cc308e95027e17c86a0bbc1dac04ddb31db417a7d40
3
+ size 3489661200
model-00010-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:899c4400650af3acfc9e5745fb7205d0c99d5aad133e1d5fb96010dfd3942544
3
+ size 4630578128
model-00011-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a49159d24dbb8bbe22e15d237bdb55f13ae0c94b57acf5cdec40d5567d74240
3
+ size 4630578128
model-00012-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0a9935b1a3796576a7ede1293814dce15c0c4a5ca90740b88dd6324d3c84760
3
+ size 3489661200
model-00013-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b9861d694b2817287d0d83d72320dedfbd92614561b9ab321c09c026115ad99
3
+ size 4630578128
model-00014-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52d47bb08a2c3b486f9370d42d733f02e126f98152e76fae3ca640a84aec119e
3
+ size 4630578128
model-00015-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a78345095223d6858dcecc31db28f7c940c812095a9bc6aa99bec8010f43e5a
3
+ size 3489661200
model-00016-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7e8e267580f68a53c20ff2b077566b001279f4ff18fad0faead83f3cb64e835
3
+ size 4630578128
model-00017-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:900e1a12ad3e070333825694cafdfa58727e6d135a7c219d236a16d654a3432e
3
+ size 4630578128
model-00018-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c08bfd7149e89a428b9bbc3d35fdb4637bc54ebdd69971e89669a9dc09a6719
3
+ size 3489661200
model-00019-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa8d4aec528dccdba03c4f817fcaf151b1564e445e6d86392a853d8581c43eb4
3
+ size 4630578128
model-00020-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf57ef1c5e10ca21979cab1faca6babea55ab8e13c0b812183612def7aede4a2
3
+ size 4630578128
model-00021-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9b46d36c03454c5966618c1ee3a235389d0190dc5144a3fada7db9301c0db46
3
+ size 3489661200
model-00022-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3e35ae1d51bb04aeb7893cfe10296352856fb58eeebc3ad9a90ff14080ff8d7
3
+ size 4630578120
model-00023-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08b81be492b98cf302ef660a3dc4720f90ef5e6adee0a94fc39709a983dc3e14
3
+ size 4630578128
model-00024-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ac8ed72b0744c04b0e4be6eee2527c758dc094457aed5128c5a40193d852e32
3
+ size 3489661200
model-00025-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5689c5820f219d451a69c5375013f5380577c32015b94816f58b2c2d19d241d0
3
+ size 4630578128
model-00026-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a49303e8003505b03ec7c866828f635ab3f1101c61a05b20bdf6e8840274f36
3
+ size 4630578120
model-00027-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c0149681479bf187411677577ad3d7918f0136b62cd21c182dbe192d92c0495
3
+ size 3489661200
model-00028-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:413b02bf32c06027ba1618d2452f39dec0b059009c9d7d3e569a8e6517590daa
3
+ size 4630578136
model-00029-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb1e484102d702d9b49eda51dacd9a8480d35fd9a7b16f67b5e9b314549733f5
3
+ size 4630578136
model-00030-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:004f389c16d5491bee5859b03ae121661cddf80d549e1c3f0eb757c7428e8cfb
3
+ size 3489661200
model-00031-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3423fc9484d1f897c991a110e4ed5ec03592d6fd8b243671e3f7f237e5e576d
3
+ size 4630578136
model-00032-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86b39e15d83f3ca3cf8ff5c6abbd2d9e9c84af5b10322be36a9e147bb34573fe
3
+ size 4630578136
model-00033-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2550189748de6a3d65fe080ad52e3bdda6b9c2a1c72a9ef0a735174a9f924468
3
+ size 3489661200
model-00034-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7099383767b47d2581d509d5769a5f8f83ffbc082483578721883f825d53043d
3
+ size 4630578136
model-00035-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddd65c78e6b031ae6e38bd156f9e22dc8f86931a20b55deaa0df30e59fb32e80
3
+ size 4630578136
model-00036-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61dda38a1ef1282adfa70a2ecd78051c325369078bda709daf906df4d477ac03
3
+ size 3489661200
model-00037-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a853ef1ff5ce65762e9cd1c2051f363821d0ad90746b8388861f6dfa7e83dc3a
3
+ size 4630578136
model-00038-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b114383de70bb8e4789be0a1fae7078d1a3a9811edb425d1e524614d62aa3d02
3
+ size 4630578136
model-00039-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbccea6a88ce9b4bb9358b6b98860b5bd50577f388fc1ef91f05119cf42352dd
3
+ size 3489661200
model-00040-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:068535f1ba9bfebbce5a7f503d55cc5ac3a74257c482a183aa0df5cd4ad24612
3
+ size 4630578120
model-00041-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a1c8e48e28b46bdb57e9722ba866a72e84d4b8ce9d840a3f352f133b0be386e
3
+ size 4630578136
model-00042-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd7b8b6bde410026060232e856de777eb61afa594530a0a34d9b2dc9c7c7e924
3
+ size 3489661200
model-00043-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c38e8476795d685f30b3dc19696da359e3e53c7cd5f60dc41fbdb372efbab22
3
+ size 4630578136
model-00044-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67eea5b22aa34b98a2e7d032fa653ea2a2c85d11bc3811d747389bd800187053
3
+ size 4630578136
model-00045-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:312ca327532c0ef22d2401ee4df9a53c8d614b056a702f10786cc05eb5d59116
3
+ size 3489661200
model-00046-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c22998a5bc2c4bb39da49c92840f9ebb1263e924005d0ef983fd3db622672de
3
+ size 4630578136
model-00047-of-00173.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16b62043b4a611b6a7fa15d2c43759cb2ff0686ea490646e0184e3365b16a21c
3
+ size 4630578136