# Copyright Lightning AI. Licensed under the Apache License 2.0, see LICENSE file. from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path from typing import Any, Literal import yaml from typing_extensions import Self def find_multiple(n: int, k: int) -> int: """Utility function for finding the nearest value to n which is a multiple of k. NOTE: We define this function in this module rather than `litgpt.utils` so that users can import this file to do configuration manipulations in Python environments which do not include all the dependencies demanded by `litgpt.utils`. """ assert k > 0 if n % k == 0: return n return n + k - (n % k) @dataclass class Config: name: str = "" hf_config: dict = field(default_factory=dict) # General size parameters block_size: int = 4096 n_layer: int = 16 n_embd: int = 4096 vocab_size: int = 50254 padding_multiple: int = 512 padded_vocab_size: int | None = None # Transformer block (structure, normalizations) norm_class_name: Literal["LayerNorm", "RMSNorm"] = "LayerNorm" norm_eps: float = 1e-5 norm_qk: bool = False norm_qk_type: Literal["default", "olmo2"] = "default" post_attention_norm: bool = False post_mlp_norm: bool = False parallel_residual: bool = True shared_attention_norm: bool = False # Transformer block (self-attention) n_head: int = 32 head_size: int | None = None # to use multi-head attention (MHA), set this to `n_head` (default) # to use multi-query attention (MQA), set this to 1 # to use grouped-query attention (GQA), set this to a value in between # Example with `n_head=4` # ┌───┐┌───┐┌───┐┌───┐ ┌───┐ ┌───┐ ┌───┐ # │ v ││ v ││ v ││ v │ │ v │ │ v │ │ v │ # └───┘└───┘└───┘└───┘ └───┘ └───┘ └───┘ # │ │ │ │ │ │ │ # ┌───┐┌───┐┌───┐┌───┐ ┌───┐ ┌───┐ ┌───┐ # │ k ││ k ││ k ││ k │ │ k │ │ k │ │ k │ # └───┘└───┘└───┘└───┘ └───┘ └───┘ └───┘ # │ │ │ │ ┌──┴──┐ ┌──┴──┐ ┌────┬──┴─┬────┐ # ┌───┐┌───┐┌───┐┌───┐ ┌───┐┌───┐┌───┐┌───┐ ┌───┐┌───┐┌───┐┌───┐ # │ q ││ q ││ q ││ q │ │ q ││ q ││ q ││ q │ │ q ││ q ││ q ││ q │ # └───┘└───┘└───┘└───┘ └───┘└───┘└───┘└───┘ └───┘└───┘└───┘└───┘ # ◀──────────────────▶ ◀──────────────────▶ ◀──────────────────▶ # MHA GQA MQA # n_query_groups=4 n_query_groups=2 n_query_groups=1 # # credit https://arxiv.org/pdf/2305.13245.pdf n_query_groups: int | None = None attn_bias: bool = False attention_scores_scalar: int | None = None # If `sliding_window_size` is given, sliding window attention with this # size is used in layers where `sliding_window_indices` has a 1. The # default is all 1, so that sliding window attention is used in all # layers. If `len(sliding_window_indices) > n_layer`, we only use the # initial part. sliding_window_size: int | None = None sliding_window_indices: list[int] | None = None # if `attention_logit_softcapping` is used, cannot use optimized # `torch.nn.functional.scaled_dot_product_attention` (which implements # Flash attention), may result in higher memory and runtime footprint. attention_logit_softcapping: float | None = None # Rotary position embedding (RoPE) rope_base: int = 10000 rotary_percentage: float = 0.25 rope_condense_ratio: int = 1 rope_adjustments: dict | None = None rope_interleave: bool = False # Transformer block (MLP) intermediate_size: int | None = None moe_intermediate_size: int | None = None bias: bool = True mlp_class_name: Literal["GptNeoxMLP", "LLaMAMLP", "GemmaMLP", "LLaMAMoE"] = "GptNeoxMLP" gelu_approximate: str = "none" n_expert: int = 0 n_shared_expert: int | None = None n_expert_groups: int | None = None n_topk_groups: int | None = None n_topk_scores_per_group: int | None = None n_expert_per_token: int = 0 first_k_dense_replace: int | None = None routed_scaling_factor: float = 1.0 norm_topk_prob: bool = False # GPT before/after blocks scale_embeddings: bool = False lm_head_bias: bool = False final_logit_softcapping: float | None = None norm_1: bool = True norm_2: bool = True latent_attention: dict | None = None # The base period of the RoPE embeddings for local attention. # If not provided, `rope_base` will be used for both local and global attention. rope_local_base_freq: float | None = None # If provided, must have `>= n_layer` entries, either 0 or 1. For 0, # `rope_base` is used, for 1 `rope_local_base_freq` is used. If # `len(rope_indices) > n_layer`, we only use the initial part. rope_indices: list[int] | None = None def __post_init__(self): if not self.name: self.name = self.hf_config.get("name", self.name) if self.head_size is None: assert self.n_embd % self.n_head == 0 self.head_size = self.n_embd // self.n_head # vocab size should be a power of 2 to be optimal on hardware. compute the closest value if self.padded_vocab_size is None: self.padded_vocab_size = find_multiple(self.vocab_size, self.padding_multiple) else: # vocab size shouldn't be larger than padded vocab size self.vocab_size = min(self.vocab_size, self.padded_vocab_size) # compute the number of query groups if self.n_query_groups is not None: assert self.n_head % self.n_query_groups == 0 else: self.n_query_groups = self.n_head # compute the intermediate size for MLP if not set if self.intermediate_size is None: if self.mlp_class_name == "LLaMAMLP": raise ValueError(f"The config {self.name!r}, needs to set the `intermediate_size`") self.intermediate_size = 4 * self.n_embd self.rope_n_elem = int(self.rotary_percentage * self.head_size) if self.sliding_window_size is not None: self.sliding_window_indices = check_indicator_and_length( self.sliding_window_indices, name="sliding_window_indices", required_length=self.n_layer, ) if self.rope_local_base_freq is not None: self.rope_indices = check_indicator_and_length( self.rope_indices, name="rope_indices", required_length=self.n_layer, ) if self.latent_attention is not None: self.q_lora_rank = self.latent_attention.get("q_lora_rank") self.kv_lora_rank = self.latent_attention.get("kv_lora_rank") self.qk_rope_head_dim = self.latent_attention.get("qk_rope_head_dim") self.qk_nope_head_dim = self.latent_attention.get("qk_nope_head_dim") self.v_head_dim = self.latent_attention.get("v_head_dim") assert ( self.q_lora_rank and self.kv_lora_rank and self.qk_rope_head_dim and self.qk_nope_head_dim and self.v_head_dim ) is not None assert self.n_head == self.n_query_groups, "Latent attention does not support MQA/GQA" self.qk_head_dim = self.qk_rope_head_dim + self.qk_nope_head_dim self.rope_n_elem = self.qk_rope_head_dim if self.first_k_dense_replace is not None: assert self.mlp_class_name == "LLaMAMoE" if self.n_expert_groups is not None: assert self.n_expert % self.n_expert_groups == 0 and self.n_expert_groups > 1 assert self.n_topk_groups is not None experts_per_group = self.n_expert // self.n_expert_groups assert self.n_topk_scores_per_group is not None and self.n_topk_scores_per_group <= experts_per_group @classmethod def from_name(cls, name: str, **kwargs: Any) -> Self | None: if name not in name_to_config: # search through all `config['hf_config']['name']` try: conf_dict = next( config for config in configs if name == config["hf_config"]["name"] or config["hf_config"]["org"] + "/" + config["hf_config"]["name"] == name ) except StopIteration: raise ValueError(f"{name!r} is not a supported config name") else: conf_dict = name_to_config[name] conf_dict = conf_dict.copy() conf_dict.update(kwargs) return cls(**conf_dict) @classmethod def from_file(cls, path: str | Path, **kwargs: Any) -> Self: with open(path, encoding="utf-8") as fp: file_kwargs = yaml.safe_load(fp) if file_kwargs is None: raise ValueError(f"{path} is empty which is likely unexpected.") file_kwargs.update(kwargs) return cls(**file_kwargs) @classmethod def from_checkpoint(cls, path: Path, **kwargs: Any) -> Self: """Automatically load `model_config.yaml` and if it doesn't exist - a matching config from `litgpt/config.py`.""" if (config_path := path / "model_config.yaml").is_file(): return cls.from_file(config_path, **kwargs) if (model_name := path.name) in name_to_config: return cls.from_name(model_name, **kwargs) raise FileNotFoundError(f"For {str(path)!r} neither 'model_config.yaml' nor matching config exists.") @property def mlp_class(self) -> type: # `self.mlp_class_name` cannot be the type to keep the config serializable import litgpt.model return getattr(litgpt.model, self.mlp_class_name) @property def norm_class(self) -> type: # `self.norm_class_name` cannot be the type to keep the config serializable from functools import partial import torch # Torch import is lazy to make config loading faster if self.norm_class_name == "RMSNorm": from litgpt.model import RMSNorm return partial(RMSNorm, add_unit_offset="Gemma" in self.name) if self.norm_class_name == "LayerNorm" and "OLMo" in self.name: # this makes it equivalent to `torch.nn.functional.layer_norm` # that is used by OLMo # Table 5 caption in the OLMo paper shows this - https://aclanthology.org/2024.acl-long.841 return partial(torch.nn.LayerNorm, elementwise_affine=False) return getattr(torch.nn, self.norm_class_name) def check_indicator_and_length( params: list[int] | None, name: str, required_length: int, use_initial_part: bool = True, def_val: int = 1, ) -> list[int]: if params is None: return [def_val] * required_length if len(params) != required_length: if use_initial_part and len(params) > required_length: params = params[:required_length] else: raise ValueError(f"{name} = {params}, must have length {required_length}") if not set(params).issubset({0, 1}): raise ValueError(f"{name} = {params}, must only contain 0 and 1") return params ######################## # Stability AI StableLM ######################## configs = [ # https://huggingface.co/stabilityai/stablelm-base-alpha-3b/blob/main/config.json dict(name="stablelm-base-alpha-3b", hf_config=dict(org="stabilityai", name="stablelm-base-alpha-3b")), # https://huggingface.co/stabilityai/stablelm-base-alpha-7b/blob/main/config.json dict( name="stablelm-base-alpha-7b", hf_config=dict(org="stabilityai", name="stablelm-base-alpha-7b"), n_head=48, n_embd=6144, padding_multiple=256, ), # https://huggingface.co/stabilityai/stablelm-tuned-alpha-3b/blob/main/config.json dict(name="stablelm-tuned-alpha-3b", hf_config=dict(org="stabilityai", name="stablelm-tuned-alpha-3b"), n_head=32), # https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b/blob/main/config.json dict( name="stablelm-tuned-alpha-7b", hf_config=dict(org="stabilityai", name="stablelm-tuned-alpha-7b"), n_head=48, n_embd=6144, padding_multiple=256, ), # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json dict( name="stablelm-3b-4e1t", hf_config=dict(org="stabilityai", name="stablelm-3b-4e1t"), padded_vocab_size=50304, n_layer=32, n_head=32, n_embd=2560, parallel_residual=False, bias=False, mlp_class_name="LLaMAMLP", intermediate_size=6912, ), # https://huggingface.co/stabilityai/stablelm-zephyr-3b/blob/main/config.json dict( name="stablelm-zephyr-3b", hf_config=dict(org="stabilityai", name="stablelm-zephyr-3b"), padded_vocab_size=50304, n_layer=32, n_head=32, n_embd=2560, parallel_residual=False, bias=False, mlp_class_name="LLaMAMLP", intermediate_size=6912, ), ] ########################## # Stability AI StableCode ########################## stablecode = [ # https://huggingface.co/stabilityai/stablecode-completion-alpha-3b/blob/main/config.json dict( name="stablecode-completion-alpha-3b", hf_config=dict(org="stabilityai", name="stablecode-completion-alpha-3b"), block_size=16384, vocab_size=49152, n_layer=32, n_embd=2560, ), # https://huggingface.co/stabilityai/stablecode-completion-alpha-3b-4k/blob/main/config.json dict( name="stablecode-completion-alpha-3b-4k", hf_config=dict(org="stabilityai", name="stablecode-completion-alpha-3b-4k"), vocab_size=49152, n_layer=32, n_embd=2560, ), # https://huggingface.co/stabilityai/stablecode-instruct-alpha-3b/blob/main/config.json dict( name="stablecode-instruct-alpha-3b", hf_config=dict(org="stabilityai", name="stablecode-instruct-alpha-3b"), vocab_size=49152, n_layer=32, n_embd=2560, ), # https://huggingface.co/stabilityai/stable-code-3b/blob/main/config.json dict( name="stable-code-3b", hf_config=dict(org="stabilityai", name="stable-code-3b"), padded_vocab_size=50304, n_layer=32, n_embd=2560, block_size=16384, parallel_residual=False, bias=False, mlp_class_name="LLaMAMLP", intermediate_size=6912, ), ] configs.extend(stablecode) #################### # EleutherAI Pythia #################### pythia = [ # https://huggingface.co/EleutherAI/pythia-14m/blob/main/config.json dict( name="pythia-14m", hf_config=dict(org="EleutherAI", name="pythia-14m"), block_size=512, n_layer=6, n_embd=128, n_head=4, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-31m/blob/main/config.json dict( name="pythia-31m", hf_config=dict(org="EleutherAI", name="pythia-31m"), block_size=1024, n_layer=6, n_embd=256, n_head=8, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-70m/blob/main/config.json dict( name="pythia-70m", hf_config=dict(org="EleutherAI", name="pythia-70m"), block_size=2048, n_layer=6, n_embd=512, n_head=8, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-160m/blob/main/config.json dict( name="pythia-160m", hf_config=dict(org="EleutherAI", name="pythia-160m"), block_size=2048, n_layer=12, n_embd=768, n_head=12, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-410m/blob/main/config.json dict( name="pythia-410m", hf_config=dict(org="EleutherAI", name="pythia-410m"), block_size=2048, n_layer=24, n_embd=1024, n_head=16, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-1b/blob/main/config.json dict( name="pythia-1b", hf_config=dict(org="EleutherAI", name="pythia-1b"), block_size=2048, n_embd=2048, n_head=8, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-1.4b/blob/main/config.json dict( name="pythia-1.4b", hf_config=dict(org="EleutherAI", name="pythia-1.4b"), block_size=2048, n_layer=24, n_embd=2048, n_head=16, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-2.8b/blob/main/config.json dict( name="pythia-2.8b", hf_config=dict(org="EleutherAI", name="pythia-2.8b"), block_size=2048, n_layer=32, n_embd=2560, padding_multiple=128, ), # https://huggingface.co/EleutherAI/pythia-6.9b/blob/main/config.json dict( name="pythia-6.9b", hf_config=dict(org="EleutherAI", name="pythia-6.9b"), block_size=2048, n_layer=32, padding_multiple=256, ), # https://huggingface.co/EleutherAI/pythia-12b/blob/main/config.json dict( name="pythia-12b", hf_config=dict(org="EleutherAI", name="pythia-12b"), block_size=2048, n_layer=36, n_embd=5120, n_head=40, ), ] configs.extend(pythia) for c in pythia: # "pythia-14m" and "pythia-31m" don't have deduped version if c["name"] in ("pythia-14m", "pythia-31m"): continue copy = deepcopy(c) copy["name"] = f"{c['name']}-deduped" copy["hf_config"]["name"] = f"{c['hf_config']['name']}-deduped" configs.append(copy) ################# # TII UAE Falcon ################# falcon = [ # https://huggingface.co/tiiuae/falcon-7b/blob/main/config.json dict( name="falcon-7b{}", hf_config=dict(org="tiiuae", name="falcon-7b{}"), block_size=2048, vocab_size=65024, padded_vocab_size=65024, n_layer=32, n_head=71, n_embd=4544, rotary_percentage=1.0, n_query_groups=1, bias=False, # this is not in the config, but in the original model implementation, only for this config shared_attention_norm=True, ), # https://huggingface.co/tiiuae/falcon-40b/blob/main/config.json dict( name="falcon-40b{}", hf_config=dict(org="tiiuae", name="falcon-40b{}"), block_size=2048, vocab_size=65024, padded_vocab_size=65024, n_layer=60, n_head=128, n_embd=8192, rotary_percentage=1.0, n_query_groups=8, bias=False, ), ] for c in falcon: for kind in ("", "-instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) # https://huggingface.co/tiiuae/falcon-180b/blob/main/config.json falcon180b = dict( name="falcon-180B{}", hf_config=dict(org="tiiuae", name="falcon-180B{}"), block_size=2048, vocab_size=65024, padded_vocab_size=65024, n_layer=80, n_head=232, n_embd=14848, rotary_percentage=1.0, n_query_groups=8, bias=False, ) for kind in ("", "-chat"): copy = deepcopy(falcon180b) copy["name"] = falcon180b["name"].format(kind) copy["hf_config"]["name"] = falcon180b["hf_config"]["name"].format(kind) configs.append(copy) falcon3 = [ # https://huggingface.co/tiiuae/Falcon3-1B-Base/blob/main/config.json dict( name="Falcon3-1B{}", hf_config=dict(org="tiiuae", name="Falcon3-1B{}"), block_size=4096, vocab_size=131072, padded_vocab_size=131072, n_layer=18, n_head=8, n_query_groups=4, n_embd=2048, rotary_percentage=1.0, parallel_residual=False, rope_base=1000042, norm_eps=1e-6, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8192, ), # https://huggingface.co/tiiuae/Falcon3-3B-Base/blob/main/config.json dict( name="Falcon3-3B{}", hf_config=dict(org="tiiuae", name="Falcon3-3B{}"), block_size=32768, vocab_size=131072, padded_vocab_size=131072, n_layer=22, n_head=12, n_query_groups=4, n_embd=3072, rotary_percentage=1.0, parallel_residual=False, rope_base=1000042, norm_eps=1e-6, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=9216, ), # https://huggingface.co/tiiuae/Falcon3-7B-Base/blob/main/config.json dict( name="Falcon3-7B{}", hf_config=dict(org="tiiuae", name="Falcon3-7B{}"), block_size=32768, vocab_size=131072, padded_vocab_size=131072, n_layer=28, n_head=12, n_query_groups=4, n_embd=3072, rotary_percentage=1.0, parallel_residual=False, rope_base=1000042, norm_eps=1e-6, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=23040, ), # https://huggingface.co/tiiuae/Falcon3-10B-Base/blob/main/config.json dict( name="Falcon3-10B{}", hf_config=dict(org="tiiuae", name="Falcon3-10B{}"), block_size=32768, vocab_size=131072, padded_vocab_size=131072, n_layer=40, n_head=12, n_query_groups=4, n_embd=3072, rotary_percentage=1.0, parallel_residual=False, rope_base=1000042, norm_eps=1e-6, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=23040, ), ] for c in falcon3: for kind in ("-Base", "-Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############################# # OpenLM Research Open LLaMA ############################# open_LLaMA = [ # https://huggingface.co/openlm-research/open_llama_3b/blob/main/config.json dict( name="open_llama_3b", hf_config=dict(org="openlm-research", name="open_llama_3b"), block_size=2048, vocab_size=32000, padding_multiple=64, n_layer=26, n_embd=3200, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-6, mlp_class_name="LLaMAMLP", intermediate_size=8640, ), # https://huggingface.co/openlm-research/open_llama_7b/blob/main/config.json dict( name="open_llama_7b", hf_config=dict(org="openlm-research", name="open_llama_7b"), block_size=2048, vocab_size=32000, padding_multiple=64, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-6, mlp_class_name="LLaMAMLP", intermediate_size=11008, ), # https://huggingface.co/openlm-research/open_llama_13b/blob/main/config.json dict( name="open_llama_13b", hf_config=dict(org="openlm-research", name="open_llama_13b"), block_size=2048, vocab_size=32000, padding_multiple=64, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-6, mlp_class_name="LLaMAMLP", intermediate_size=13824, ), ] configs.extend(open_LLaMA) ############### # Meta LLaMA 2 ############### llama_2 = [ # https://huggingface.co/meta-llama/Llama-2-7b-hf/blob/main/config.json dict( name="Llama-2-7b{}-hf", hf_config=dict(org="meta-llama", name="Llama-2-7b{}-hf"), vocab_size=32000, padding_multiple=64, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, ), # https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json dict( name="Llama-2-13b{}-hf", hf_config=dict(org="meta-llama", name="Llama-2-13b{}-hf"), vocab_size=32000, padding_multiple=64, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, ), # https://huggingface.co/meta-llama/Llama-2-70b-hf/blob/main/config.json dict( name="Llama-2-70b{}-hf", hf_config=dict(org="meta-llama", name="Llama-2-70b{}-hf"), vocab_size=32000, padding_multiple=64, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, ), ] for c in llama_2: for kind in ("", "-chat"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############### # Meta LLaMA 3 ############### llama_3 = [ # https://huggingface.co/meta-llama/Meta-Llama-3-8B/blob/main/config.json dict( name="Llama-3-8B{}", hf_config=dict(org="meta-llama", name="Meta-Llama-3-8B{}"), block_size=8192, vocab_size=128000, padded_vocab_size=128256, n_layer=32, n_head=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=500000, ), # https://huggingface.co/meta-llama/Meta-Llama-3.1-8B/blob/main/config.json dict( name="Llama-3.1-8B{}", hf_config=dict(org="meta-llama", name="Meta-Llama-3.1-8B{}"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=32, n_head=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/meta-llama/Meta-Llama-3-70B/blob/main/config.json dict( name="Llama-3-70B{}", hf_config=dict(org="meta-llama", name="Meta-Llama-3-70B{}"), block_size=8192, vocab_size=128000, padded_vocab_size=128256, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=500000, ), # https://huggingface.co/meta-llama/Meta-Llama-3.1-70B/blob/main/config.json dict( name="Llama-3.1-70B{}", hf_config=dict(org="meta-llama", name="Meta-Llama-3.1-70B{}"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/meta-llama/Meta-Llama-3.1-405B/blob/main/config.json dict( name="Llama-3.1-405B{}", hf_config=dict(org="meta-llama", name="Meta-Llama-3.1-405B{}"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=126, n_head=128, n_embd=16384, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=53248, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/config.json dict( name="Llama-3.2-1B{}", hf_config=dict(org="meta-llama", name="Llama-3.2-1B{}"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=16, n_embd=2048, n_head=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8192, rope_base=500000, rope_adjustments=dict(factor=32.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/meta-llama/Llama-3.2-3B/blob/main/config.json dict( name="Llama-3.2-3B{}", hf_config=dict(org="meta-llama", name="Llama-3.2-3B{}"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=28, n_embd=3072, n_head=24, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8192, rope_base=500000, rope_adjustments=dict(factor=32.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/blob/main/config.json dict( name="Llama-3.3-70B-Instruct", hf_config=dict(org="meta-llama", name="Llama-3.3-70B-Instruct"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), ] for c in llama_3: if c["name"] == "Llama-3.3-70B-Instruct": configs.append(c) continue for kind in ("", "-Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ######################### # NVIDIA Llama Nemotron ######################### configs.append( dict( name="Llama-3.1-Nemotron-70B-Instruct-HF", hf_config=dict(org="nvidia", name="Llama-3.1-Nemotron-70B-Instruct-HF"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), ) ################# # Allen AI OLMo ################# olmo = [ # https://huggingface.co/allenai/OLMo-1B-hf/blob/main/config.json dict( name="OLMo-1B-hf", hf_config=dict(org="allenai", name="OLMo-1B-hf"), vocab_size=50280, padded_vocab_size=50304, block_size=2048, n_embd=2048, n_layer=16, n_head=16, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="LayerNorm", mlp_class_name="LLaMAMLP", intermediate_size=8192, ), # https://huggingface.co/allenai/OLMo-7B-hf/blob/main/config.json dict( name="OLMo-7B-hf", hf_config=dict(org="allenai", name="OLMo-7B-hf"), vocab_size=50280, padded_vocab_size=50304, block_size=2048, n_layer=32, n_head=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="LayerNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, ), # https://huggingface.co/allenai/OLMo-7B-Instruct-hf/blob/main/config.json dict( name="OLMo-7B-Instruct-hf", hf_config=dict(org="allenai", name="OLMo-7B-Instruct-hf"), vocab_size=50280, padded_vocab_size=50304, block_size=2048, n_layer=32, n_head=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="LayerNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, ), ] configs.extend(olmo) olmo2 = [ # https://huggingface.co/allenai/OLMo-2-1124-7B/blob/main/config.json dict( name="OLMo-2-1124-7B{}", hf_config=dict(org="allenai", name="OLMo-2-1124-7B{}"), vocab_size=100278, padded_vocab_size=100352, block_size=4096, n_embd=4096, n_layer=32, n_head=32, n_query_groups=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", norm_eps=1e-06, intermediate_size=11008, rope_base=500000, norm_qk=True, post_mlp_norm=True, norm_1=False, norm_2=False, norm_qk_type="olmo2", post_attention_norm=True, ), # https://huggingface.co/allenai/OLMo-2-1124-13B/blob/main/config.json dict( name="OLMo-2-1124-13B{}", hf_config=dict(org="allenai", name="OLMo-2-1124-13B{}"), vocab_size=100278, padded_vocab_size=100352, block_size=4096, n_embd=5120, n_layer=40, n_head=40, n_query_groups=40, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", norm_eps=1e-06, intermediate_size=13824, rope_base=500000, norm_qk=True, post_mlp_norm=True, norm_1=False, norm_2=False, norm_qk_type="olmo2", post_attention_norm=True, ), ] for c in olmo2: for kind in ("", "-SFT", "-DPO", "-Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############### # Google Gemma ############### gemma = [ # https://huggingface.co/google/gemma-2b/blob/main/config.json dict( name="Gemma-2b", hf_config=dict(org="google", name="gemma-2b"), scale_embeddings=True, vocab_size=256000, padding_multiple=64, n_embd=2048, n_layer=18, n_head=8, n_query_groups=1, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", intermediate_size=16384, ), # https://huggingface.co/google/gemma-7b/blob/main/config.json dict( name="Gemma-7b", hf_config=dict(org="google", name="gemma-7b"), scale_embeddings=True, vocab_size=256000, padding_multiple=64, n_embd=3072, n_layer=28, n_head=16, head_size=256, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", intermediate_size=24576, ), # https://huggingface.co/google/gemma-2-2b/blob/main/config.json dict( name="Gemma-2-2b", hf_config=dict(org="google", name="gemma-2-2b"), scale_embeddings=True, attention_scores_scalar=256, vocab_size=256000, block_size=8192, sliding_window_size=4096, # only layer with idx 0, 2, 4, ... have sliding window attention sliding_window_indices=[1 if i % 2 == 0 else 0 for i in range(26)], intermediate_size=9216, n_embd=2304, n_layer=26, n_head=8, n_query_groups=4, head_size=256, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, attention_logit_softcapping=50.0, final_logit_softcapping=30.0, ), # https://huggingface.co/google/gemma-2-9b/blob/main/config.json dict( name="Gemma-2-9b", hf_config=dict(org="google", name="gemma-2-9b"), scale_embeddings=True, attention_scores_scalar=256, vocab_size=256000, block_size=8192, sliding_window_size=4096, # only layer with idx 0, 2, 4, ... have sliding window attention sliding_window_indices=[1 if i % 2 == 0 else 0 for i in range(42)], intermediate_size=14336, n_embd=3584, n_layer=42, n_head=16, n_query_groups=8, head_size=256, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, attention_logit_softcapping=50.0, final_logit_softcapping=30.0, ), # https://huggingface.co/google/gemma-2-27b/blob/main/config.json dict( name="Gemma-2-27b", hf_config=dict(org="google", name="gemma-2-27b"), scale_embeddings=True, # In Gemma 2 27B attention scores are scaled not by `sqrt(head_size)` (11.31), # but by `sqrt(n_emb // n_head)` = sqrt(4608 // 32) = 12 attention_scores_scalar=144, vocab_size=256000, block_size=8192, sliding_window_size=4096, # only layer with idx 0, 2, 4, ... have sliding window attention sliding_window_indices=[1 if i % 2 == 0 else 0 for i in range(46)], intermediate_size=36864, n_embd=4608, n_layer=46, n_head=32, n_query_groups=16, head_size=128, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, attention_logit_softcapping=50.0, final_logit_softcapping=30.0, ), ] configs.extend(gemma) for c in gemma: copy = deepcopy(c) copy["name"] = f"{c['name']}-it" copy["hf_config"]["name"] = f"{c['hf_config']['name']}-it" configs.append(copy) ################## # Google Gemma 3 ################## gemma3 = [ # https://huggingface.co/google/gemma-3-1b-it/blob/main/config.json dict( name="Gemma-3-1b-it", hf_config=dict(org="google", name="gemma-3-1b-it"), scale_embeddings=True, attention_scores_scalar=256, vocab_size=262144, block_size=131072, sliding_window_size=512, # 5 local layers for every global layer sliding_window_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(26)], intermediate_size=6912, n_embd=1152, n_layer=26, n_head=4, n_query_groups=1, head_size=256, rotary_percentage=1.0, rope_adjustments=None, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, norm_qk=True, rope_base=1000000, rope_local_base_freq=10000, # 5 local layers for every global layer rope_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(26)], ), # https://huggingface.co/google/gemma-3-4b-it/blob/main/config.json dict( name="Gemma-3-4b-it", hf_config=dict(org="google", name="gemma-3-4b-it"), scale_embeddings=True, attention_scores_scalar=256, vocab_size=262144, block_size=131072, sliding_window_size=1024, # 5 local layers for every global layer sliding_window_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(34)], intermediate_size=10240, n_embd=2560, n_layer=34, n_head=8, n_query_groups=4, head_size=256, rotary_percentage=1.0, rope_adjustments=dict(factor=8.0), parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, norm_qk=True, rope_base=1000000, rope_local_base_freq=10000, # 5 local layers for every global layer rope_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(34)], ), # https://huggingface.co/google/gemma-3-12b-it/blob/main/config.json dict( name="Gemma-3-12b-it", hf_config=dict(org="google", name="gemma-3-12b-it"), scale_embeddings=True, attention_scores_scalar=256, vocab_size=262144, block_size=131072, sliding_window_size=1024, # 5 local layers for every global layer sliding_window_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(48)], intermediate_size=15360, n_embd=3840, n_layer=48, n_head=16, n_query_groups=8, head_size=256, rotary_percentage=1.0, rope_adjustments=dict(factor=8.0), parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, norm_qk=True, rope_base=1000000, rope_local_base_freq=10000, # 5 local layers for every global layer rope_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(48)], ), # https://huggingface.co/google/gemma-3-27b-it/blob/main/config.json dict( name="Gemma-3-27b-it", hf_config=dict(org="google", name="gemma-3-27b-it"), scale_embeddings=True, attention_scores_scalar=168, vocab_size=262144, block_size=131072, sliding_window_size=1024, # 5 local layers for every global layer sliding_window_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(62)], intermediate_size=21504, n_embd=5376, n_layer=62, n_head=32, n_query_groups=16, head_size=128, rotary_percentage=1.0, rope_adjustments=dict(factor=8.0), parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", post_attention_norm=True, post_mlp_norm=True, norm_qk=True, rope_base=1000000, rope_local_base_freq=10000, # 5 local layers for every global layer rope_indices=[0 if (i + 1) % 6 == 0 else 1 for i in range(62)], ), ] configs.extend(gemma3) ################## # Google CodeGemma ################## codegemma = [ # https://huggingface.co/google/codegemma-7b-it/blob/main/config.json dict( name="CodeGemma-7b-it", hf_config=dict(org="google", name="codegemma-7b-it"), scale_embeddings=True, vocab_size=256000, padding_multiple=64, n_embd=3072, n_layer=28, n_head=16, head_size=256, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="GemmaMLP", gelu_approximate="tanh", intermediate_size=24576, ), ] configs.extend(codegemma) ########################## # Stability AI FreeWilly2 ########################## freewilly_2 = [ # https://huggingface.co/stabilityai/FreeWilly2/blob/main/config.json dict( name="FreeWilly2", hf_config=dict(org="stabilityai", name="FreeWilly2"), vocab_size=32000, padding_multiple=64, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, ) ] configs.extend(freewilly_2) ################## # Meta Code Llama ################## code_llama = [ # https://huggingface.co/codellama/CodeLlama-7b-hf/blob/main/config.json dict( name="CodeLlama-7b-hf", hf_config=dict(org="codellama", name="CodeLlama-7b-hf"), block_size=16384, vocab_size=32016, padding_multiple=16, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=11008, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-13b-hf/blob/main/config.json dict( name="CodeLlama-13b-hf", hf_config=dict(org="codellama", name="CodeLlama-13b-hf"), block_size=16384, vocab_size=32016, padding_multiple=16, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=13824, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-34b-hf/blob/main/config.json dict( name="CodeLlama-34b-hf", hf_config=dict(org="codellama", name="CodeLlama-34b-hf"), block_size=16384, vocab_size=32000, padded_vocab_size=32000, n_layer=48, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=22016, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-70b-hf/blob/main/config.json dict( name="CodeLlama-70b-hf", hf_config=dict(org="codellama", name="CodeLlama-70b-hf"), block_size=16384, vocab_size=32016, padding_multiple=16, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/config.json dict( name="CodeLlama-7b-Python-hf", hf_config=dict(org="codellama", name="CodeLlama-7b-Python-hf"), block_size=16384, vocab_size=32000, padded_vocab_size=32000, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=11008, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-13b-Python-hf/blob/main/config.json dict( name="CodeLlama-13b-Python-hf", hf_config=dict(org="codellama", name="CodeLlama-13b-Python-hf"), block_size=16384, vocab_size=32000, padded_vocab_size=32000, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=13824, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-34b-Python-hf/blob/main/config.json dict( name="CodeLlama-34b-Python-hf", hf_config=dict(org="codellama", name="CodeLlama-34b-Python-hf"), block_size=16384, vocab_size=32000, padded_vocab_size=32000, n_layer=48, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=22016, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-70b-Python-hf/blob/main/config.json dict( name="CodeLlama-70b-Python-hf", hf_config=dict(org="codellama", name="CodeLlama-70b-Python-hf"), block_size=16384, vocab_size=32016, padding_multiple=16, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/config.json dict( name="CodeLlama-7b-Instruct-hf", hf_config=dict(org="codellama", name="CodeLlama-7b-Instruct-hf"), block_size=16384, vocab_size=32016, padding_multiple=16, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=11008, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf/blob/main/config.json dict( name="CodeLlama-13b-Instruct-hf", hf_config=dict(org="codellama", name="CodeLlama-13b-Instruct-hf"), block_size=2048, vocab_size=32016, padding_multiple=16, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=13824, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf/blob/main/config.json dict( name="CodeLlama-34b-Instruct-hf", hf_config=dict(org="codellama", name="CodeLlama-34b-Instruct-hf"), block_size=16384, vocab_size=32000, padded_vocab_size=32000, n_layer=48, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=22016, rope_base=1000000, ), # https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf/blob/main/config.json dict( name="CodeLlama-70b-Instruct-hf", hf_config=dict(org="codellama", name="CodeLlama-70b-Instruct-hf"), block_size=16384, # 32016 is an added token, so not reported in vocab_size # https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf/blob/main/tokenizer_config.json vocab_size=32015, padding_multiple=16, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=1000000, ), ] configs.extend(code_llama) ######################## # garage-bAInd Platypus ######################## platypus = [ # https://huggingface.co/garage-bAInd/Platypus-30B/blob/main/config.json dict( name="Platypus-30B", hf_config=dict(org="garage-bAInd", name="Platypus-30B"), block_size=2048, padded_vocab_size=32000, n_layer=60, n_head=52, n_embd=6656, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-06, mlp_class_name="LLaMAMLP", intermediate_size=17920, ), # https://huggingface.co/garage-bAInd/Platypus2-7B/blob/main/config.json dict( name="Platypus2-7B", hf_config=dict(org="garage-bAInd", name="Platypus2-7B"), padded_vocab_size=32000, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=11008, ), # https://huggingface.co/garage-bAInd/Platypus2-13B/blob/main/config.json dict( name="Platypus2-13B", hf_config=dict(org="garage-bAInd", name="Platypus2-13B"), padded_vocab_size=32000, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=13824, ), # https://huggingface.co/garage-bAInd/Platypus2-70B/blob/main/config.json dict( name="Platypus2-70B", hf_config=dict(org="garage-bAInd", name="Platypus2-70B"), padded_vocab_size=32000, n_layer=80, n_head=64, n_embd=8192, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, ), # https://huggingface.co/garage-bAInd/Camel-Platypus2-13B/blob/main/config.json dict( name="Camel-Platypus2-13B", hf_config=dict(org="garage-bAInd", name="Camel-Platypus2-13B"), padded_vocab_size=32000, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, ), # https://huggingface.co/garage-bAInd/Camel-Platypus2-70B/blob/main/config.json dict( name="Camel-Platypus2-70B", hf_config=dict(org="garage-bAInd", name="Camel-Platypus2-70B"), padded_vocab_size=32000, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, ), # https://huggingface.co/garage-bAInd/Stable-Platypus2-13B/blob/main/config.json dict( name="Stable-Platypus2-13B", hf_config=dict(org="garage-bAInd", name="Stable-Platypus2-13B"), padded_vocab_size=32000, n_layer=40, n_head=40, n_embd=5120, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, ), # https://huggingface.co/garage-bAInd/Platypus2-70B-instruct/blob/main/config.json dict( name="Platypus2-70B-instruct", hf_config=dict(org="garage-bAInd", name="Platypus2-70B-instruct"), padded_vocab_size=32000, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, ), ] configs.extend(platypus) ################################## # togethercomputer LLaMA-2-7B-32K ################################## together_llama2_32k = [ # https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/blob/main/config.json dict( name="LLaMA-2-7B-32K", hf_config=dict(org="togethercomputer", name="LLaMA-2-7B-32K"), vocab_size=32000, padding_multiple=64, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, rope_condense_ratio=8, ) ] configs.extend(together_llama2_32k) ################ # Microsoft Phi ################ phi = [ # https://huggingface.co/microsoft/phi-1_5/blob/main/config.json dict( name="phi-1_5", hf_config=dict(org="microsoft", name="phi-1_5"), vocab_size=50257, padded_vocab_size=51200, block_size=2048, n_embd=2048, n_layer=24, rotary_percentage=0.5, # 32 / (n_embd / n_head) = 32 / 64 shared_attention_norm=True, lm_head_bias=True, gelu_approximate="tanh", ), # https://huggingface.co/microsoft/phi-2/blob/main/config.json dict( name="phi-2", hf_config=dict(org="microsoft", name="phi-2"), vocab_size=50257, padded_vocab_size=51200, block_size=2048, n_embd=2560, n_layer=32, rotary_percentage=0.4, # 32 / (n_embd / n_head) = 32 / 80 shared_attention_norm=True, lm_head_bias=True, gelu_approximate="tanh", ), # https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/config.json dict( name="Phi-3-mini-4k-instruct", hf_config=dict(org="microsoft", name="Phi-3-mini-4k-instruct"), vocab_size=32000, padded_vocab_size=32064, block_size=4096, n_embd=3072, n_layer=32, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=8192, mlp_class_name="LLaMAMLP", parallel_residual=False, sliding_window_size=2048, ), # https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/blob/main/config.json dict( name="Phi-3-mini-128k-instruct", hf_config=dict(org="microsoft", name="Phi-3-mini-128k-instruct"), vocab_size=32000, padded_vocab_size=32064, block_size=131072, n_embd=3072, n_layer=32, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=8192, mlp_class_name="LLaMAMLP", parallel_residual=False, sliding_window_size=262145, ), # https://huggingface.co/microsoft/Phi-3.5-mini-instruct/blob/main/config.json dict( name="Phi-3.5-mini-instruct", hf_config=dict(org="microsoft", name="Phi-3.5-mini-instruct"), vocab_size=32000, padded_vocab_size=32064, block_size=4096, n_embd=3072, n_layer=32, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=8192, mlp_class_name="LLaMAMLP", parallel_residual=False, ), # https://huggingface.co/microsoft/phi-4/blob/main/config.json dict( name="phi-4", hf_config=dict(org="microsoft", name="phi-4"), vocab_size=100352, padded_vocab_size=100352, block_size=16384, n_embd=5120, n_layer=40, n_head=40, n_query_groups=10, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=17920, rope_base=250000, mlp_class_name="LLaMAMLP", parallel_residual=False, ), # https://huggingface.co/microsoft/Phi-4-reasoning/blob/main/config.json dict( name="Phi-4-reasoning", hf_config=dict(org="microsoft", name="Phi-4-reasoning"), vocab_size=100352, padded_vocab_size=100352, block_size=32768, n_embd=5120, n_layer=40, n_head=40, n_query_groups=10, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=17920, rope_base=500000, mlp_class_name="LLaMAMLP", parallel_residual=False, ), # https://huggingface.co/microsoft/Phi-4-reasoning-plus/blob/main/config.json dict( name="Phi-4-reasoning-plus", hf_config=dict(org="microsoft", name="Phi-4-reasoning-plus"), vocab_size=100352, padded_vocab_size=100352, block_size=32768, n_embd=5120, n_layer=40, n_head=40, n_query_groups=10, rotary_percentage=1.0, bias=False, norm_class_name="RMSNorm", intermediate_size=17920, rope_base=500000, mlp_class_name="LLaMAMLP", parallel_residual=False, ), # https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/main/config.json dict( name="Phi-4-mini-instruct", hf_config=dict(org="microsoft", name="Phi-4-mini-instruct"), vocab_size=200019, padded_vocab_size=200064, block_size=131072, n_embd=3072, n_layer=32, n_head=24, n_query_groups=8, rotary_percentage=0.75, bias=False, norm_class_name="RMSNorm", intermediate_size=8192, mlp_class_name="LLaMAMLP", parallel_residual=False, sliding_window_size=262145, ), # https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/main/config.json dict( name="Phi-4-mini-reasoning", hf_config=dict(org="microsoft", name="Phi-4-mini-reasoning"), vocab_size=200019, padded_vocab_size=200064, block_size=131072, n_embd=3072, n_layer=32, n_head=24, n_query_groups=8, rotary_percentage=0.75, bias=False, norm_class_name="RMSNorm", intermediate_size=8192, mlp_class_name="LLaMAMLP", parallel_residual=False, sliding_window_size=262145, ), ] configs.extend(phi) ############# # Mistral AI ############# configs.append( # https://huggingface.co/mistralai/mathstral-7B-v0.1/blob/main/config.json dict( name="Mathstral-7B-v0.1", hf_config=dict(org="mistralai", name="mathstral-7B-v0.1"), padded_vocab_size=32768, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=1000000, sliding_window_size=4096, ) ) mistral = [ # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json dict( name="Mistral-7B-{}v0.1", hf_config=dict(org="mistralai", name="Mistral-7B-{}v0.1"), padded_vocab_size=32000, block_size=4096, # should be 32768 but sliding window attention is not implemented n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, sliding_window_size=4096, ), # https://huggingface.co/mistralai/Mixtral-8x7B-v0.1/blob/main/config.json dict( name="Mixtral-8x7B-{}v0.1", hf_config=dict(org="mistralai", name="Mixtral-8x7B-{}v0.1"), padded_vocab_size=32000, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMoE", intermediate_size=14336, rope_base=1000000, n_expert=8, n_expert_per_token=2, ), # https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1/blob/main/config.json dict( name="Mixtral-8x22B-{}v0.1", hf_config=dict(org="mistralai", name="Mixtral-8x22B-{}v0.1"), padded_vocab_size=32768, block_size=65536, n_layer=56, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMoE", intermediate_size=16384, n_head=48, n_embd=6144, rope_base=1000000, n_expert=8, n_expert_per_token=2, ), ] for c in mistral: for kind in ("", "Instruct-"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) configs.append( # https://huggingface.co/unsloth/mistral-7b-v0.2/blob/main/config.json dict( name="Mistral-7B-v0.2", hf_config=dict(org="unsloth", name="Mistral-7B-v0.2"), padded_vocab_size=32000, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=1000000, ) ) configs.append( # https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2/blob/main/config.json dict( name="Mistral-7B-Instruct-v0.2", hf_config=dict(org="mistralai", name="Mistral-7B-Instruct-v0.2"), padded_vocab_size=32000, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=1000000, ) ) configs.append( # https://huggingface.co/mistralai/Mistral-7B-v0.3/blob/main/config.json dict( name="Mistral-7B-v0.3", hf_config=dict(org="mistralai", name="Mistral-7B-v0.3"), padded_vocab_size=32768, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=1000000, ) ) configs.append( # https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3/blob/main/config.json dict( name="Mistral-7B-Instruct-v0.3", hf_config=dict(org="mistralai", name="Mistral-7B-Instruct-v0.3"), padded_vocab_size=32768, block_size=32768, n_layer=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=1000000, ) ) configs.append( # https://huggingface.co/mistralai/Mistral-Large-Instruct-2407/blob/main/config.json dict( name="Mistral-Large-Instruct-2407", hf_config=dict(org="mistralai", name="Mistral-Large-Instruct-2407"), padded_vocab_size=32768, block_size=32768, n_layer=88, n_head=96, n_embd=12288, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=1000000, ) ) configs.append( # https://huggingface.co/mistralai/Mistral-Large-Instruct-2411/blob/main/config.json dict( name="Mistral-Large-Instruct-2411", hf_config=dict(org="mistralai", name="Mistral-Large-Instruct-2411"), padded_vocab_size=32768, block_size=32768, n_layer=88, n_head=96, n_embd=12288, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", norm_eps=1e-05, mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=1000000, ) ) ############ # TinyLlama ############ tiny_llama = [ dict( name="tiny-llama-1.1b{}", hf_config=dict(org="TinyLlama", name="TinyLlama-1.1B{}"), block_size=2048, vocab_size=32000, padding_multiple=64, n_layer=22, n_head=32, n_embd=2048, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", # original TinyLlama use FusedRMSNorm norm_eps=1e-5, mlp_class_name="LLaMAMLP", intermediate_size=5632, n_query_groups=4, ) ] for c in tiny_llama: for kind, hf_postfix in (("", "-intermediate-step-1431k-3T"), ("-chat", "-Chat-v1.0")): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(hf_postfix) configs.append(copy) ############ # MicroLlama ############ micro_llama = [ dict( name="micro-llama-300M", hf_config=dict(org="keeeeenw", name="MicroLlama"), block_size=2048, vocab_size=32000, padding_multiple=64, n_layer=12, n_head=16, n_embd=1024, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", # original TinyLlama and MicroLlama use FusedRMSNorm norm_eps=1e-5, mlp_class_name="LLaMAMLP", intermediate_size=5632, n_query_groups=4, ) ] configs.extend(micro_llama) ########################## # Trelis Function Calling ########################## llama_2_function_calling = [ # https://huggingface.co/Trelis/Llama-2-7b-chat-hf-function-calling-v2/blob/main/config.json dict( name="Llama-2-7b-chat-hf-function-calling-v2", hf_config=dict(org="Trelis", name="Llama-2-7b-chat-hf-function-calling-v2"), padding_multiple=64, n_layer=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, norm_eps=1e-6, block_size=4096, vocab_size=32000, n_head=32, n_embd=4096, rope_base=10000, ) ] configs.extend(llama_2_function_calling) ########## # Qwen2.5 ########## qwen_2_5 = [ # https://huggingface.co/Qwen/Qwen2.5-0.5B/blob/main/config.json dict( name="Qwen2.5-0.5B{}", hf_config=dict(org="Qwen", name="Qwen2.5-0.5B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=151936, n_layer=24, n_head=14, n_embd=896, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=4864, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-1.5B/blob/main/config.json dict( name="Qwen2.5-1.5B{}", hf_config=dict(org="Qwen", name="Qwen2.5-1.5B{}"), block_size=131072, vocab_size=151643, padded_vocab_size=151936, n_layer=28, n_head=12, n_embd=1536, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8960, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-3B/blob/main/config.json dict( name="Qwen2.5-3B{}", hf_config=dict(org="Qwen", name="Qwen2.5-3B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=151936, n_layer=36, n_head=16, n_embd=2048, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-7B/blob/main/config.json dict( name="Qwen2.5-7B{}", hf_config=dict(org="Qwen", name="Qwen2.5-7B{}"), block_size=131072, vocab_size=151643, padded_vocab_size=152064, n_layer=28, n_head=28, n_embd=3584, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=18944, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-14B/blob/main/config.json dict( name="Qwen2.5-14B{}", hf_config=dict(org="Qwen", name="Qwen2.5-14B{}"), block_size=131072, vocab_size=151643, padded_vocab_size=152064, n_layer=48, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, norm_eps=1e-5, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-32B/blob/main/config.json dict( name="Qwen2.5-32B{}", hf_config=dict(org="Qwen", name="Qwen2.5-32B{}"), block_size=131072, vocab_size=151643, padded_vocab_size=152064, n_layer=64, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=27648, norm_eps=1e-5, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-72B/blob/main/config.json dict( name="Qwen2.5-72B{}", hf_config=dict(org="Qwen", name="Qwen2.5-72B{}"), block_size=131072, vocab_size=151643, padded_vocab_size=152064, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=29568, norm_eps=1e-5, rope_base=1000000, ), ] qwen_2_5_coder = [ # https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B/blob/main/config.json dict( name="Qwen2.5-Coder-0.5B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-0.5B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=151936, n_layer=24, n_head=14, n_embd=896, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=4864, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B/blob/main/config.json dict( name="Qwen2.5-Coder-1.5B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-1.5B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=151936, n_layer=28, n_head=12, n_embd=1536, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8960, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-Coder-3B/blob/main/config.json dict( name="Qwen2.5-Coder-3B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-3B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=151936, n_layer=36, n_head=16, n_embd=2048, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-Coder-7B/blob/main/config.json dict( name="Qwen2.5-Coder-7B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-7B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=152064, n_layer=28, n_head=28, n_embd=3584, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=18944, norm_eps=1e-6, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-Coder-14B/blob/main/config.json dict( name="Qwen2.5-Coder-14B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-14B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=152064, n_layer=48, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, norm_eps=1e-5, rope_base=1000000, ), # https://huggingface.co/Qwen/Qwen2.5-Coder-32B/blob/main/config.json dict( name="Qwen2.5-Coder-32B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Coder-32B{}"), block_size=32768, vocab_size=151643, padded_vocab_size=152064, n_layer=64, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=27648, norm_eps=1e-5, rope_base=1000000, ), ] qwen_2_5.extend(qwen_2_5_coder) qwen_2_5_math = [ # https://huggingface.co/Qwen/Qwen2.5-Math-1.5B/blob/main/config.json dict( name="Qwen2.5-Math-1.5B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Math-1.5B{}"), block_size=4096, vocab_size=151643, padded_vocab_size=151936, n_layer=28, n_head=12, n_embd=1536, n_query_groups=2, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8960, norm_eps=1e-6, rope_base=10000, ), # https://huggingface.co/Qwen/Qwen2.5-Math-7B/blob/main/config.json dict( name="Qwen2.5-Math-7B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Math-7B{}"), block_size=4096, vocab_size=151643, padded_vocab_size=152064, n_layer=28, n_head=28, n_embd=3584, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=18944, norm_eps=1e-6, rope_base=10000, ), # https://huggingface.co/Qwen/Qwen2.5-Math-72B/blob/main/config.json dict( name="Qwen2.5-Math-72B{}", hf_config=dict(org="Qwen", name="Qwen2.5-Math-72B{}"), block_size=4096, vocab_size=151643, padded_vocab_size=152064, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=29568, norm_eps=1e-5, rope_base=10000, ), ] qwen_2_5.extend(qwen_2_5_math) for c in qwen_2_5: for kind in ("", "-Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) qwen_2_5_1m = [ # https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M/blob/main/config.json dict( name="Qwen2.5-7B-Instruct-1M", hf_config=dict(org="Qwen", name="Qwen2.5-7B-Instruct-1M"), block_size=1010000, vocab_size=151643, padded_vocab_size=152064, n_layer=28, n_head=28, n_embd=3584, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=18944, norm_eps=1e-5, rope_base=10000000, ), # https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-1M/blob/main/config.json dict( name="Qwen2.5-14B-Instruct-1M", hf_config=dict(org="Qwen", name="Qwen2.5-14B-Instruct-1M"), block_size=1010000, vocab_size=151643, padded_vocab_size=152064, n_layer=48, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=13824, norm_eps=1e-5, rope_base=10000000, ), ] configs.extend(qwen_2_5_1m) ########## # QwQ ########## qwq = [ # https://huggingface.co/Qwen/QwQ-32B/blob/main/config.json dict( name="QwQ-32B", hf_config=dict(org="Qwen", name="QwQ-32B"), block_size=131072, vocab_size=151643, padded_vocab_size=152064, n_layer=64, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=27648, norm_eps=1e-5, rope_base=1000000, ), # https://huggingface.co/Qwen/QwQ-32B-Preview/blob/main/config.json dict( name="QwQ-32B-Preview", hf_config=dict(org="Qwen", name="QwQ-32B-Preview"), block_size=32768, vocab_size=151643, padded_vocab_size=152064, n_layer=64, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, attn_bias=True, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=27648, norm_eps=1e-5, rope_base=1000000, ), ] configs.extend(qwq) ########## # Qwen3 ########## qwen_3 = [ # https://huggingface.co/Qwen/Qwen3-0.6B/blob/main/config.json dict( name="Qwen3-0.6B{}", hf_config=dict(org="Qwen", name="Qwen3-0.6B{}"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=28, n_head=16, n_embd=1024, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=3072, norm_eps=1e-6, rope_base=1000000, head_size=128, norm_qk=True, ), # https://huggingface.co/Qwen/Qwen3-1.7B/blob/main/config.json dict( name="Qwen3-1.7B{}", hf_config=dict(org="Qwen", name="Qwen3-1.7B{}"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=28, n_head=16, n_embd=2048, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=6144, norm_eps=1e-6, rope_base=1000000, norm_qk=True, ), # https://huggingface.co/Qwen/Qwen3-4B/blob/main/config.json dict( name="Qwen3-4B{}", hf_config=dict(org="Qwen", name="Qwen3-4B{}"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=36, n_head=32, n_embd=2560, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=9728, norm_eps=1e-6, rope_base=1000000, head_size=128, norm_qk=True, ), # https://huggingface.co/Qwen/Qwen3-8B/blob/main/config.json dict( name="Qwen3-8B{}", hf_config=dict(org="Qwen", name="Qwen3-8B{}"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=36, n_head=32, n_embd=4096, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=12288, norm_eps=1e-6, rope_base=1000000, norm_qk=True, ), # https://huggingface.co/Qwen/Qwen3-14B/blob/main/config.json dict( name="Qwen3-14B{}", hf_config=dict(org="Qwen", name="Qwen3-14B{}"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=40, n_head=40, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=17408, norm_eps=1e-6, rope_base=1000000, norm_qk=True, ), ] for c in qwen_3: for kind in ("", "-Base"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) qwen_3_32b = [ # https://huggingface.co/Qwen/Qwen3-32B/blob/main/config.json dict( name="Qwen3-32B", hf_config=dict(org="Qwen", name="Qwen3-32B"), block_size=40960, vocab_size=151643, padded_vocab_size=151936, n_layer=64, n_head=64, n_embd=5120, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=25600, norm_eps=1e-6, rope_base=1000000, head_size=128, norm_qk=True, ), ] configs.extend(qwen_3_32b) qwen_3_moe = [ # https://huggingface.co/Qwen/Qwen3-30B-A3B/blob/main/config.json dict( name="Qwen3-30B-A3B", hf_config=dict(org="Qwen", name="Qwen3-30B-A3B"), block_size=40960, head_size=128, vocab_size=151643, padded_vocab_size=151936, n_layer=48, n_head=32, n_embd=2048, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMoE", intermediate_size=6144, moe_intermediate_size=768, norm_eps=1e-6, rope_base=1000000, norm_qk=True, n_expert=128, n_expert_per_token=8, ), # https://huggingface.co/Qwen/Qwen3-30B-A3B-Base/blob/main/config.json dict( name="Qwen3-30B-A3B-Base", hf_config=dict(org="Qwen", name="Qwen3-30B-A3B-Base"), block_size=40960, head_size=128, vocab_size=151643, padded_vocab_size=151936, n_layer=48, n_head=32, n_embd=2048, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMoE", intermediate_size=6144, moe_intermediate_size=768, norm_eps=1e-6, rope_base=1000000, norm_qk=True, n_expert=128, n_expert_per_token=8, ), # https://huggingface.co/Qwen/Qwen3-235B-A22B/blob/main/config.json dict( name="Qwen3-235B-A22B", hf_config=dict(org="Qwen", name="Qwen3-235B-A22B"), block_size=40960, head_size=128, vocab_size=151643, padded_vocab_size=151936, n_layer=94, n_head=64, n_embd=4096, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMoE", intermediate_size=12288, moe_intermediate_size=1536, norm_eps=1e-6, rope_base=1000000, norm_qk=True, n_expert=128, n_expert_per_token=8, ), ] configs.extend(qwen_3_moe) qwen_3_2507_thinking_instruct = [ # https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507/blob/main/config.json dict( name="Qwen3-235B-A22B-{}-2507", hf_config=dict(org="Qwen", name="Qwen3-235B-A22B-{}-2507"), block_size=262144, head_size=128, vocab_size=151643, padded_vocab_size=151936, n_layer=94, n_head=64, n_embd=4096, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMoE", intermediate_size=12288, moe_intermediate_size=1536, norm_eps=1e-6, rope_base=5000000, norm_qk=True, n_expert=128, n_expert_per_token=8, ), # https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507/blob/main/config.json dict( name="Qwen3-30B-A3B-{}-2507", hf_config=dict(org="Qwen", name="Qwen3-30B-A3B-{}-2507"), block_size=262144, head_size=128, vocab_size=151643, padded_vocab_size=151936, n_layer=48, n_head=32, n_embd=2048, n_query_groups=4, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMoE", intermediate_size=6144, moe_intermediate_size=768, norm_eps=1e-6, rope_base=10000000, norm_qk=True, n_expert=128, n_expert_per_token=8, ), # https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507/blob/main/config.json dict( name="Qwen3-4B-{}-2507", hf_config=dict(org="Qwen", name="Qwen3-4B-{}-2507"), block_size=262144, vocab_size=151643, padded_vocab_size=151936, n_layer=36, n_head=32, n_embd=2560, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=9728, norm_eps=1e-6, rope_base=5000000, head_size=128, norm_qk=True, ), ] for c in qwen_3_2507_thinking_instruct: for kind in ("Thinking", "Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############# # Salamandra ############# salamandra = [ # https://huggingface.co/BSC-LT/salamandra-2b-instruct/blob/main/config.json dict( name="salamandra-2b{}", hf_config=dict(org="BSC-LT", name="salamandra-2b{}"), block_size=8192, vocab_size=256000, padded_vocab_size=256000, n_layer=24, n_head=16, n_embd=2048, n_query_groups=16, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=5440, norm_eps=1e-5, rope_base=10000, ), # https://huggingface.co/BSC-LT/salamandra-7b-instruct/blob/main/config.json dict( name="salamandra-7b{}", hf_config=dict(org="BSC-LT", name="salamandra-7b{}"), block_size=8192, vocab_size=256000, padded_vocab_size=256000, n_layer=32, n_head=32, n_embd=4096, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=11008, norm_eps=1e-6, rope_base=10000, ), ] for c in salamandra: for kind in ("", "-instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############### # SmolLM2 ############### smollm2 = [ # https://huggingface.co/HuggingFaceTB/SmolLM2-135M/blob/main/config.json dict( name="SmolLM2-135M{}", hf_config=dict(org="HuggingFaceTB", name="SmolLM2-135M{}"), block_size=8192, vocab_size=49152, padded_vocab_size=49152, n_layer=30, n_head=9, n_embd=576, n_query_groups=3, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=1536, rope_base=100000, norm_eps=1e-5, ), # https://huggingface.co/HuggingFaceTB/SmolLM2-360M/blob/main/config.json dict( name="SmolLM2-360M{}", hf_config=dict(org="HuggingFaceTB", name="SmolLM2-360M{}"), block_size=8192, vocab_size=49152, padded_vocab_size=49152, n_layer=32, n_head=15, n_embd=960, n_query_groups=5, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=2560, rope_base=100000, norm_eps=1e-5, ), # https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B/blob/main/config.json dict( name="SmolLM2-1.7B{}", hf_config=dict(org="HuggingFaceTB", name="SmolLM2-1.7B{}"), block_size=8192, vocab_size=49152, padded_vocab_size=49152, n_layer=24, n_head=32, n_embd=2048, n_query_groups=32, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=8192, rope_base=130000, norm_eps=1e-5, ), ] for c in smollm2: for kind in ("", "-Instruct"): copy = deepcopy(c) copy["name"] = c["name"].format(kind) copy["hf_config"]["name"] = c["hf_config"]["name"].format(kind) configs.append(copy) ############### # DeepSeek R1 Distill ############### r1_distill_llama = [ # https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B/blob/main/config.json dict( name="R1-Distill-Llama-8B", hf_config=dict(org="deepseek-ai", name="DeepSeek-R1-Distill-Llama-8B"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=32, n_head=32, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=14336, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), # https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/blob/main/config.json dict( name="R1-Distill-Llama-70B", hf_config=dict(org="deepseek-ai", name="DeepSeek-R1-Distill-Llama-70B"), block_size=131072, vocab_size=128000, padded_vocab_size=128256, n_layer=80, n_head=64, n_embd=8192, n_query_groups=8, rotary_percentage=1.0, parallel_residual=False, bias=False, norm_class_name="RMSNorm", mlp_class_name="LLaMAMLP", intermediate_size=28672, rope_base=500000, rope_adjustments=dict(factor=8.0, low_freq_factor=1.0, high_freq_factor=4.0, original_max_seq_len=8192), ), ] configs.extend(r1_distill_llama) name_to_config = {config["name"]: config for config in configs}