{"groups":{"meta":"Release","params":"Parameters","shape":"Shape","attn":"Attention","ffn":"Feed-forward","moe":"Mixture of experts","mtp":"Multi-token prediction","norm":"Normalization and stability","ctx":"Context length","train":"Pretraining recipe","scaling":"Scaling & HP transfer","kv":"KV cache","eval":"Evaluations"},"fields":{"meta.released":{"type":"date","required":"always","label":"Released","definition":"Date the weights (or, if earlier, the technical report) were first made public, YYYY-MM-DD, as dated by the publisher. A Hugging Face repository creation date is used only when no dated publisher page exists, with a note."},"meta.license":{"type":"str","required":"open_weights","label":"License","definition":"License of the released weights as declared by the publisher (model card metadata or license file), e.g. apache-2.0, mit, or the name of a custom license."},"meta.modality":{"type":"str","label":"Input modalities","aliases":["multimodal","vision","VLM","omni","audio"],"definition":"Input modalities of the released checkpoint as the publisher states them, from: text, image, video, audio, joined with ' + ' in that order (\"text\", \"text + image + video\")."},"params.total":{"type":"float","required":"always","unit":"params","label":"Total params","definition":"Total parameter count as reported by the publisher. See params.note for whether embeddings, MTP heads or vision towers are included."},"params.active":{"type":"float","required":"always","unit":"params","label":"Active params","definition":"Parameters used per token in the forward pass, as reported by the publisher."},"params.active_prefill":{"type":"float","unit":"params","label":"Active params (prefill)","aliases":["YOCO","prefill"],"definition":"Parameters used per prompt token during prefill when the publisher states it differs from decode (e.g. YOCO-style designs that skip the upper half of the stack)."},"params.active_non_embedding":{"type":"float","unit":"params","required":"always","label":"Active non-embedding (stated)","aliases":["non-embedding","Kaplan N"],"definition":"Active parameters excluding the input embedding and the output head (unembedding). Stated by the publisher, or computed from params.active by subtracting vocab × d_model for each of the embedding and output head that params.active is stated to include (basis inferred, note with the arithmetic). Not disclosed when the scope of params.active is not stated."},"params.counted_total":{"type":"float","unit":"params","label":"Counted total params","aliases":["checkpoint","safetensors"],"derived":true,"definition":"Parameters of the text backbone counted from the released checkpoint's safetensors headers (embedding, output head, all experts and other weights; excluding MTP modules, vision/audio towers and n-gram lookup tables). Shown only when the counts reproduce the publisher's stated total and active parameters within 5% under some scope."},"params.counted_embedding":{"type":"float","unit":"params","label":"Counted embedding + head","derived":true,"definition":"Input embedding plus output head (once if tied), counted from the checkpoint."},"params.counted_lookup":{"type":"float","unit":"params","label":"Counted lookup tables","aliases":["ngram","engram","n-gram","per-layer embedding"],"derived":true,"definition":"Parameters in n-gram / Engram / per-layer embedding tables counted from the checkpoint. Like the token embedding, they are lookups and are excluded from params.counted_total and params.counted_active_non_embedding."},"params.counted_non_embedding":{"type":"float","unit":"params","label":"Non-emb. params (counted)","aliases":["non-embedding","checkpoint"],"derived":true,"definition":"Language backbone weights excluding input embeddings, output head, lookup/Engram tables, MTP modules and vision/audio towers. Equals params.counted_total minus params.counted_embedding. Same validation as params.counted_total."},"params.counted_max_layer":{"type":"float","unit":"params","label":"Largest layer (counted)","aliases":["per-layer","all-gather","FSDP","checkpoint"],"derived":true,"definition":"Largest group of language-backbone weights sharing a full layers.<index> tensor path, including every routed expert. Excludes lookup/Engram tables, embeddings, output head, MTP and other modalities. Hybrid checkpoints may store mixers and FFNs as separate layers. All-gather volume also depends on wrapping, sharding and dtype."},"params.counted_active_non_embedding":{"type":"float","unit":"params","label":"Active non-emb. (counted)","aliases":["non-embedding","Kaplan N","checkpoint"],"derived":true,"definition":"Active parameters per token excluding the input embedding and output head, counted from the checkpoint as all non-expert weights plus routed-expert weights × top_k / experts. Same validation as params.counted_total."},"params.note":{"type":"str","label":"Counting scope","definition":"What the publisher's counts include or exclude (MTP modules, embeddings, vision encoder, n-gram tables)."},"params.embedding":{"type":"float","unit":"params","label":"Embedding params","definition":"Parameters in input embedding and output head, as reported."},"params.non_embedding":{"type":"float","unit":"params","label":"Non-embedding params","definition":"Parameters excluding embedding and output head, as reported."},"shape.layers":{"type":"int","required":"always","unit":"layers","label":"Layers","definition":"Number of transformer blocks in the main backbone (attention or token mixer plus FFN), excluding MTP modules."},"shape.d_model":{"type":"int","required":"always","unit":"elements","label":"d_model","definition":"Residual stream width."},"shape.vocab":{"type":"int","required":"always","unit":"count","label":"Vocab","definition":"Embedding table rows (vocab_size in config, may include padding)."},"shape.tied_embeddings":{"type":"bool","required":"always","label":"Tied embeddings","definition":"Input embedding and output head share weights."},"shape.memory":{"type":"str","label":"Lookup memory","definition":"Hashed n-gram or conditional-memory tables outside the dense backbone (Engram, n-gram embeddings), with size and placement."},"shape.tokenizer":{"type":"str","label":"Tokenizer","definition":"Tokenizer type and lineage as described by the publisher (e.g. byte-level BPE shared with an earlier model)."},"shape.aspect":{"type":"float","unit":"ratio","label":"d_model / layers","aliases":["aspect ratio","width depth"],"derived":true,"definition":"Width-to-depth ratio, d_model divided by the number of blocks."},"attn.kind":{"type":"str","required":"always","label":"Token mixers","definition":"Token-mixing layer types in the backbone, e.g. GQA, MLA, SWA, Gated DeltaNet, Mamba-2, DSA (DeepSeek sparse attention)."},"attn.layout":{"type":"str","required":"hybrid","label":"Layer layout","definition":"Token-mixer order, summarized as a pattern or layer positions. The fact's optional sequence lists each mixer in order; a ratio alone does not establish the order. Auxiliary prediction modules and standalone FFNs are excluded."},"attn.global_layers":{"type":"int","required":"always","unit":"layers","label":"Global-attention layers","definition":"Layers whose KV cache grows with full context (dense or sparse global attention, MLA). Excludes sliding-window and linear/recurrent layers."},"attn.swa_ratio":{"type":"float","unit":"ratio","label":"SWA : global","aliases":["interleave","pattern","local global ratio"],"derived":true,"definition":"attn.local_layers divided by attn.global_layers, the sliding-window to global interleave ratio (5 for '5 SWA : 1 global')."},"attn.linear_ratio":{"type":"float","unit":"ratio","label":"Linear : global","aliases":["hybrid ratio","interleave","pattern"],"derived":true,"definition":"attn.linear_layers divided by attn.global_layers (3 for '3 Gated DeltaNet : 1 GQA')."},"attn.local_layers":{"type":"int","required":"swa_kind","unit":"layers","label":"Sliding-window layers","definition":"Layers with a fixed attention window."},"attn.linear_layers":{"type":"int","required":"linear_kind","unit":"layers","label":"Linear / SSM layers","definition":"Layers with a fixed-size recurrent state (linear attention, DeltaNet variants, Mamba)."},"attn.heads":{"type":"int","required":"always","unit":"count","label":"Query heads","definition":"Query heads in the global-attention layers."},"attn.kv_heads":{"type":"int","required":"gqa","unit":"count","label":"KV heads","definition":"Key/value heads in the global-attention layers. Not applicable to MLA, whose cache is a shared latent."},"attn.qk_width":{"type":"int","unit":"elements","label":"Heads × head dim","derived":true,"definition":"attn.heads × attn.head_dim, the query projection width of global layers."},"attn.group_size":{"type":"float","unit":"ratio","label":"Q heads / KV head","aliases":["GQA ratio","grouping"],"derived":true,"definition":"attn.heads divided by attn.kv_heads in global layers."},"attn.head_dim":{"type":"int","required":"always","unit":"elements","label":"QK head dim","definition":"Query/key dimension per head in global-attention layers (for MLA, nope + rope dims)."},"attn.v_head_dim":{"type":"int","required":"mla","unit":"elements","label":"V head dim","definition":"Value dimension per head in global layers (MLA v_head_dim; stated for MLA even when equal to the QK head dim)."},"attn.window":{"type":"int","required":"swa","unit":"tokens","label":"Window","definition":"Sliding-window size of the local layers."},"attn.long_window_layers":{"type":"int","unit":"layers","label":"Long-window SWA layers","definition":"Sliding-window layers (counted in attn.local_layers) that use attn.long_window instead of attn.window."},"attn.long_window":{"type":"int","unit":"tokens","label":"Long window","definition":"Window of the attn.long_window_layers."},"attn.local_kv_heads":{"type":"int","required":"swa","unit":"count","label":"SWA KV heads","definition":"KV heads in sliding-window layers (stated even when equal to attn.kv_heads)."},"attn.local_heads":{"type":"int","required":"swa","unit":"count","label":"SWA query heads","definition":"Query heads in sliding-window layers (stated even when equal to attn.heads)."},"attn.local_head_dim":{"type":"int","required":"swa","unit":"elements","label":"SWA QK head dim","definition":"Query/key head dimension in sliding-window layers. Value head dimensions may differ; see attn.local_v_head_dim."},"attn.local_v_head_dim":{"type":"int","unit":"elements","label":"SWA value head dim","definition":"Value head dimension in sliding-window layers, when separately specified. KV formulas use this value when present, otherwise the explicit global value dimension, or the local QK dimension when no separate value dimension is given."},"attn.kv_lora_rank":{"type":"int","required":"mla","unit":"elements","label":"MLA KV latent","definition":"Width of the compressed KV latent that MLA caches per token per layer."},"attn.q_lora_rank":{"type":"int","required":"mla","unit":"elements","label":"MLA Q latent","definition":"Width of the low-rank query projection in MLA; 0 when queries use a full-rank projection."},"attn.rope_dim":{"type":"int","required":"mla","unit":"elements","label":"Decoupled RoPE dim","definition":"Width of the shared positional key that MLA caches next to the latent."},"attn.position":{"type":"str","required":"always","label":"Positional encoding","definition":"Positional scheme in global layers: RoPE, partial RoPE, NoPE, or mixed per layer type."},"attn.rope_theta":{"type":"float","required":"rope","label":"RoPE base","definition":"RoPE base frequency of the global layers at release."},"attn.local_rope_theta":{"type":"float","required":"swa_rope","label":"SWA RoPE base","definition":"RoPE base frequency of the sliding-window layers (stated even when equal to attn.rope_theta)."},"attn.rope_fraction":{"type":"float","unit":"ratio","label":"Rotary fraction","definition":"Fraction of each head's QK dimensions that are rotated."},"attn.qk_norm":{"type":"bool","required":"always","label":"QK norm","definition":"Queries and keys are normalized per head before the dot product."},"attn.gate":{"type":"str","label":"Output gate","definition":"Gate applied to the attention output before the output projection, with its placement."},"attn.softcap":{"type":"float","label":"Attention logit softcap","definition":"tanh soft-capping value applied to attention logits."},"attn.sparse":{"type":"str","label":"Sparse attention","definition":"Token-selection mechanism inside global layers, e.g. DSA lightning indexer with top-k."},"attn.scale":{"type":"str","label":"Attention scaling","definition":"Query-key scaling when it is not 1/sqrt(head_dim), e.g. query_pre_attn_scalar, length-dependent (SSMax-style) scaling, or value scaling."},"attn.sinks":{"type":"str","label":"Attention sinks","definition":"Learned sink logits or sink bias in the softmax, with the layer types that use them."},"attn.bias":{"type":"bool","required":"always","label":"QKV bias","definition":"Query/key/value projections have bias terms."},"attn.linear_kind":{"type":"str","required":"linear","label":"Linear-mixer details","definition":"Configuration of the linear / SSM layers (e.g. Gated DeltaNet heads and dims, Mamba-2 state size), as one string."},"attn.linear_heads":{"type":"int","required":"linear","unit":"count","label":"Linear-mixer heads","definition":"Value heads (or SSM heads) in the linear / SSM layers."},"attn.linear_head_dim":{"type":"int","required":"linear","unit":"elements","label":"Linear-mixer head dim","definition":"Head dimension of the linear / SSM layers (value head dim for DeltaNet variants, head_dim for Mamba-2)."},"attn.indexer_dim":{"type":"int","required":"dsa","unit":"elements","label":"Indexer key dim","definition":"Width of the sparse-attention indexer key cached per token in each global layer (DSA lightning indexer), counted in the KV cache."},"attn.kv_sharing":{"type":"str","label":"Cross-layer KV sharing","definition":"Layers that reuse another layer's KV cache instead of computing their own (YOCO-style, cross-layer attention), and which layers own caches."},"attn.kv_groups":{"type":"list","items":["name","layers","elements","tokens_per_entry","window","source","at","note","quote"],"label":"KV cache groups","definition":"Cache owners and entry shapes for compressed or shared attention. Each item specifies layers owning the cache, scalar elements per entry (K and V together), tokens_per_entry (compression ratio, default 1), and optionally window (maximum entries retained). Completed compressed entries are floor(context / tokens_per_entry); incomplete compressor state is excluded. KV sizes use these groups when present."},"attn.kv_precision":{"type":"list","items":["name","group","elements","bits","format","scale_count","scale_bits","padding_bytes","source","at","note","quote"],"label":"KV storage formats","aliases":["FP4 cache","FP8 cache","quantization scales","mixed precision","sourced storage"],"definition":"Sourced storage components of attn.kv_groups. Each component names its group and gives elements, bits per element, format, scale_count, scale_bits and padding_bytes per entry. Different parts of one entry can have different formats. Sourced-storage sizes are available only when every group's elements are covered. These are inference-cache formats; they do not establish pretraining precision."},"attn.conv":{"type":"int","unit":"elements","label":"Short conv kernel","definition":"Kernel size of the causal short convolution in linear / SSM layers or attention projections."},"ffn.activation":{"type":"str","required":"always","label":"Activation","aliases":["SwiGLU","GeLU","nonlinearity"],"definition":"FFN activation (SwiGLU, GeGLU, ...)."},"ffn.dim":{"type":"int","required":"dense_ffn","unit":"elements","label":"Dense FFN dim","definition":"Intermediate width of the dense FFN layers that are actually executed."},"ffn.clamp":{"type":"str","label":"Activation clamp","definition":"Clamping of GLU gate / up activations (e.g. SwiGLU limit 7), with the layers it applies to."},"ffn.bias":{"type":"bool","label":"FFN bias","definition":"FFN projections have bias terms."},"ffn.mult":{"type":"float","unit":"ratio","label":"Dense FFN / d_model","derived":true,"definition":"ffn.dim divided by shape.d_model."},"moe.experts":{"type":"int","required":"moe","unit":"count","label":"Routed experts","definition":"Routed experts per MoE layer."},"moe.top_k":{"type":"int","required":"moe","unit":"count","label":"Experts per token","definition":"Routed experts selected per token."},"moe.shared":{"type":"int","required":"moe","unit":"count","label":"Shared experts","definition":"Always-on experts per MoE layer."},"moe.expert_dim":{"type":"int","required":"moe","unit":"elements","label":"Expert dim","definition":"Intermediate width of one routed expert."},"moe.shared_dim":{"type":"int","required":"shared_experts","unit":"elements","label":"Shared expert dim","definition":"Total intermediate width of the shared expert(s)."},"moe.dense_layers":{"type":"int","required":"moe","unit":"layers","label":"Leading dense layers","definition":"Number of initial blocks that use a dense FFN instead of MoE."},"moe.router":{"type":"str","required":"moe","label":"Router score","definition":"Gating function over expert logits (softmax, sigmoid)."},"moe.balancing":{"type":"str","required":"moe","label":"Load balancing","definition":"Load-balancing method, with coefficients where disclosed."},"moe.routed_scale":{"type":"float","label":"Routed scaling factor","definition":"Multiplier applied to the combined routed-expert output (routed_scaling_factor)."},"moe.norm_topk":{"type":"bool","required":"moe","label":"Renormalize top-k","definition":"Selected expert weights are renormalized to sum to 1."},"moe.groups":{"type":"str","label":"Group-limited routing","definition":"Expert groups and groups selected per token, or node-limited routing, when used."},"moe.latent_dim":{"type":"int","unit":"elements","label":"Expert latent dim","definition":"Width of the projected space routed experts operate in (LatentMoE), when smaller than d_model."},"moe.latent_ratio":{"type":"float","unit":"ratio","label":"Latent compression α (d_model / latent)","aliases":["alpha","α","LatentMoE"],"derived":true,"definition":"shape.d_model divided by moe.latent_dim, how much LatentMoE compresses the hidden state before routed experts."},"moe.granularity":{"type":"float","unit":"ratio","label":"Expert dim / d_model","derived":true,"definition":"moe.expert_dim divided by shape.d_model."},"moe.sparsity":{"type":"float","unit":"ratio","label":"Sparsity (experts / top-k)","aliases":["sparsity","expert ratio"],"derived":true,"definition":"Routed experts divided by experts selected per token."},"moe.active_fraction":{"type":"float","unit":"ratio","label":"Active / total","derived":true,"definition":"params.active divided by params.total."},"mtp.depth":{"type":"int","required":"always","unit":"count","label":"MTP depth","aliases":["multi-token prediction","nextn"],"definition":"Additional future tokens predicted during pretraining (0 when the model was trained without MTP)."},"mtp.modules":{"type":"int","required":"mtp","unit":"count","label":"MTP modules","definition":"MTP modules with distinct weights. 1 with mtp.depth > 1 means one module is applied recursively."},"mtp.block":{"type":"str","label":"MTP module","definition":"Contents of one MTP module (mixer and FFN type, norms, how the previous hidden state and next-token embedding are combined)."},"mtp.ffn":{"type":"str","required":"mtp","label":"MTP FFN","definition":"FFN in the MTP module: 'dense' or 'MoE', with its width when it differs from the backbone."},"mtp.shares_embedding":{"type":"bool","required":"mtp","label":"MTP shares embedding/head","definition":"MTP modules use the main model's input embedding and output head."},"mtp.params":{"type":"float","unit":"params","label":"MTP params","definition":"Parameters of all MTP modules as reported by the publisher."},"mtp.loss_weight":{"type":"str","required":"mtp","label":"MTP loss weight","definition":"Weight of the MTP loss, including its schedule."},"mtp.released":{"type":"bool","required":"mtp","label":"MTP weights released","definition":"The released checkpoint contains the MTP module weights."},"norm.type":{"type":"str","required":"always","label":"Norm","definition":"Normalization layer type in the blocks."},"norm.placement":{"type":"str","required":"always","label":"Norm placement","aliases":["pre-norm","post-norm","sandwich"],"definition":"Pre-norm, post-norm, or sandwich (pre and post)."},"norm.logit_softcap":{"type":"float","label":"Final logit softcap","definition":"tanh soft-capping value on output logits."},"norm.eps":{"type":"float","required":"always","label":"Norm epsilon","definition":"Epsilon of the block normalization layers."},"norm.residual":{"type":"str","label":"Residual stream","definition":"Residual design when it is not a plain sum, e.g. hyper-connections with n streams, gated residuals, depth-scaled residuals, FP32 residual accumulation."},"norm.embedding":{"type":"str","label":"Embedding scaling","definition":"Scaling or normalization applied to input embeddings (e.g. × sqrt(d_model), RMSNorm after embedding)."},"norm.output_scale":{"type":"str","label":"Output logit scaling","definition":"Scaling applied to output logits other than a softcap (e.g. muP-style width multiplier)."},"ctx.pretrain":{"type":"int","required":"always","unit":"tokens","label":"Pretraining context","definition":"Sequence length of the main pretraining stage."},"ctx.final":{"type":"int","required":"always","unit":"tokens","label":"Trained context","definition":"Longest sequence length trained on after context extension."},"ctx.max":{"type":"int","required":"always","unit":"tokens","label":"Advertised context","definition":"Longest context the publisher says the model supports, possibly beyond trained length via RoPE scaling."},"ctx.extension":{"type":"str","label":"Extension method","definition":"How context was extended (stages, RoPE base change, YaRN factor)."},"train.tokens":{"type":"float","required":"always","unit":"tokens","label":"Pretraining tokens","definition":"Tokens seen in pretraining including annealing and context extension, as reported."},"train.optimizer":{"type":"str","required":"always","label":"Optimizer","definition":"Optimizer used for the bulk of parameters."},"train.lr_peak":{"type":"float","required":"always","unit":"lr","label":"Peak LR","aliases":["learning rate"],"definition":"Peak learning rate of the main pretraining stage."},"train.lr_min":{"type":"float","required":"always","unit":"lr","label":"Final LR","aliases":["learning rate"],"definition":"Learning rate at the end of the decay."},"train.schedule":{"type":"str","required":"always","label":"LR schedule","definition":"Learning-rate schedule shape with phase boundaries."},"train.warmup":{"type":"str","required":"always","label":"Warmup","definition":"Warmup length in steps or tokens."},"train.batch":{"type":"str","required":"always","label":"Batch size","definition":"Global batch size, including any batch-size ramp: sequences × sequence length (token total), or the token total alone when only that is stated."},"train.batch_tokens":{"type":"float","unit":"tokens","label":"Batch (tokens)","aliases":["batch size","global batch"],"definition":"Global batch size in tokens during the main pretraining stage, after any ramp (sequences × sequence length when stated in sequences, marked inferred)."},"train.weight_decay":{"type":"float","required":"always","label":"Weight decay","definition":"Decoupled weight decay coefficient."},"train.betas":{"type":"str","required":"always","label":"Adam betas","definition":"(beta1, beta2) of the Adam-family optimizer or its AdamW fallback."},"train.grad_clip":{"type":"float","required":"always","label":"Grad clip","definition":"Global gradient-norm clipping threshold."},"train.dropout":{"type":"float","label":"Dropout","definition":"Dropout rate applied during pretraining, with placement in the note. Recorded only when the publisher states it."},"train.precision":{"type":"str","required":"always","label":"Training precision","definition":"Matmul precision of pretraining (BF16, FP8 recipe, NVFP4 recipe) with the scaling granularity where disclosed. Never inferred from the released checkpoint dtype."},"train.init":{"type":"str","label":"Initialization","definition":"Weight initialization (std or scheme) as stated by the publisher; config initializer_range alone is not evidence."},"train.stability":{"type":"str","label":"Stability measures","definition":"Interventions used during pretraining to keep it stable (QK-clip, z-loss, logit caps, spike handling), with coefficients."},"train.masking":{"type":"str","label":"Document masking","definition":"Whether attention crosses document boundaries within a packed sequence."},"train.objective":{"type":"str","label":"Extra objectives","definition":"Objectives beyond next-token prediction and MTP (fill-in-the-middle rate, distillation, multimodal tokens from the start)."},"train.data":{"type":"str","label":"Data mix","definition":"Pretraining data composition as disclosed (domain shares, synthetic fraction, repetition)."},"train.cutoff":{"type":"str","label":"Data cutoff","definition":"Pretraining data cutoff date."},"train.hardware":{"type":"str","label":"Hardware","definition":"Accelerators used for pretraining, count and duration where stated."},"train.accelerators":{"type":"str","label":"Accelerators","aliases":["GPUs","TPUs","hardware","cluster"],"definition":"Accelerator count and type for the main pretraining run, as the count with thousands separators and the chip's short name (\"2,048 H800\", \"2,048 GH200\"). A stated range or bound stays as stated (\"2,048–4,096 GH200\")."},"train.parallelism":{"type":"str","label":"Parallelism","aliases":["TP","PP","EP","DP","CP","FSDP","ZeRO","pipeline","tensor parallel","expert parallel"],"definition":"Parallelism layout of the main pretraining run as stated, as degree-annotated terms in the order TP, PP, VPP, EP, CP, DP (\"PP16 · EP64 · ZeRO-1 DP\"); a term without a stated degree has none (\"FSDP\")."},"train.compute":{"type":"str","label":"Compute","definition":"Pretraining compute as stated (GPU-hours or FLOPs)."},"train.averaging":{"type":"str","label":"Checkpoint averaging","definition":"EMA or checkpoint merging applied to produce the released base weights."},"train.stages":{"type":"list","items":["name","tokens","seq_len","lr","batch_tokens","batch_sequences","steps","note","quote","source"],"label":"Stages","definition":"Ordered pretraining stages. Each item: name, tokens, seq_len, lr, batch_tokens, batch_sequences, steps, note, quote, and source when it differs from the fact's source."},"scaling.parametrization":{"type":"str","label":"Parametrization","aliases":["muP","μP","CompleteP","maximal update","standard parametrization"],"definition":"Width/depth parametrization used to transfer hyperparameters: SP (standard), μP, CompleteP, u-μP, depth-μP, with the multipliers that differ from SP. Only when stated."},"scaling.hp_rule":{"type":"str","label":"HP selection rule","aliases":["scaling law","hyperparameter transfer","optimal learning rate","optimal batch"],"definition":"How peak LR, batch size or other hyperparameters were chosen for the released run (fitted scaling law with its formula, μP transfer from a proxy width, sweep), as stated."},"scaling.ladder":{"type":"str","label":"Scaling ladder","aliases":["proxy models","ablation ladder"],"definition":"Smaller models trained to choose the architecture or recipe (sizes, token budgets), as disclosed."},"scaling.tpp":{"type":"float","unit":"ratio","label":"Tokens / param","aliases":["TPP","chinchilla","overtraining"],"derived":true,"definition":"train.tokens divided by params.total."},"scaling.tpp_active":{"type":"float","unit":"ratio","label":"Tokens / active param (non-embedding)","aliases":["TPP","chinchilla","overtraining"],"derived":true,"definition":"train.tokens divided by params.counted_active_non_embedding, or by the stated params.active_non_embedding when there is no validated checkpoint count."},"scaling.adamw_tau":{"type":"float","unit":"ratio","label":"AdamW timescale τ","aliases":["power lines","EMA timescale","weight decay"],"derived":true,"definition":"B / (η λ D) with B = train.batch_tokens, η = train.lr_peak, λ = train.weight_decay and D = train.tokens: the AdamW EMA timescale as a fraction of training (Bergsma et al. 2025, Power Lines, arXiv 2505.13738). Computed only when the optimizer is AdamW; uses the peak LR."},"kv.bytes_per_token":{"type":"float","unit":"bytes","label":"KV bytes / token","aliases":["kv cache","memory"],"derived":true,"definition":"Retained KV cache divided by context length, including bounded SWA caches and indexer keys. Exports use uniform BF16 at 131,072 tokens. The website can select uniform payload precision (no quantization overhead) or sourced component formats (including scales and documented padding). This is cache payload, not peak prefill memory or persistent prefix storage."},"kv.growth_per_token":{"type":"float","unit":"bytes","label":"Global KV bytes / token","aliases":["kv cache","marginal","asymptotic","growth"],"derived":true,"definition":"Long-context global KV bytes per token, including indexer keys; excludes bounded SWA caches. Compression rates and cross-layer cache reuse are included. Exports use uniform BF16; the website's storage selector also applies here. This is the field to compare with global-KV figures in reports, using the matching storage recipe."},"kv.bytes_per_token_per_active":{"type":"float","unit":"bytes","label":"KV bytes / token / B active","aliases":["kv cache","efficiency"],"derived":true,"definition":"kv.bytes_per_token divided by active parameters in billions (131,072 tokens in exports)."},"kv.bytes_at_context_per_active":{"type":"float","unit":"bytes","label":"KV at context / B active","aliases":["kv cache","efficiency","memory per parameter"],"derived":true,"definition":"kv.bytes_at_context divided by active parameters in billions (131,072 tokens in exports)."},"kv.bytes_at_context":{"type":"float","unit":"bytes","label":"KV at context","aliases":["kv cache","memory"],"derived":true,"definition":"Retained KV cache for one sequence at the selected context, including SWA windows and indexer keys. Exports use uniform BF16 at 131,072 tokens. Excludes recurrent state, compressor tail state, draft/MTP caches, temporary prefill tensors, allocator metadata and persistent prefix snapshots. Does not estimate peak prefill or total serving memory."},"eval.aa_index":{"type":"float","unit":"score","label":"AA Intelligence Index","aliases":["artificial analysis","benchmark"],"definition":"Artificial Analysis Intelligence Index v4.3.2 of the post-trained model, from a dated Wayback snapshot of its Artificial Analysis page. The note names the variant (reasoning setting)."},"eval.deepswe":{"type":"float","unit":"score","label":"DeepSWE pass@1","aliases":["swe","coding","agentic","benchmark"],"definition":"DeepSWE v1.1 pass@1 (Datacurve; 113 tasks, same mini-swe-agent scaffold for every model), as a fraction."}},"orgs":{"Ai2":{"country":"United States","url":"https://web.archive.org/web/20261002212725/https://allenai.org/about","quote":"we are a seattle based non-profit ai research institute"},"Arcee AI":{"country":"United States","url":"https://web.archive.org/web/20261010083502/https://www.arcee.ai/","quote":"arcee is an open intelligence lab based in the u.s."},"Cohere":{"country":"Canada","url":"https://en.wikipedia.org/w/index.php?title=Cohere&oldid=1375422810","quote":"is headquartered in toronto"},"DeepSeek":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=DeepSeek&oldid=1380939138","quote":"based in hangzhou , zhejiang"},"Google DeepMind":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Google_DeepMind&oldid=1380412362","quote":"is a british-american artificial intelligence (ai) research laboratory which is a subsidiary of alphabet inc."},"IBM":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=IBM&oldid=1379231841","quote":"is an american multinational technology company headquartered in armonk, new york"},"inclusionAI":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Ant_Group&oldid=1380505693","quote":"headquarters xihu district , hangzhou , china"},"LG AI Research":{"country":"South Korea","url":"https://en.wikipedia.org/w/index.php?title=LG&oldid=1376104107","quote":"is a south korean multinational conglomerate"},"MBZUAI IFM":{"country":"United Arab Emirates","url":"https://en.wikipedia.org/w/index.php?title=Mohamed_bin_Zayed_University_of_Artificial_Intelligence&oldid=1374891836","quote":"public university in abu dhabi, uae"},"Meituan":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Meituan&oldid=1379460130","quote":"is a chinese technology company"},"Meta":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Meta_Platforms&oldid=1380291192","quote":"headquartered in menlo park, california , is an american multinational technology company"},"Microsoft AI":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Microsoft_AI&oldid=1380003212","quote":"headquarters in redmond, washington"},"MiniMax":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=MiniMax_Group&oldid=1377609224","quote":"is an artificial intelligence (ai) company based in shanghai , china"},"Mistral AI":{"country":"France","url":"https://en.wikipedia.org/w/index.php?title=Mistral_AI&oldid=1379950095","quote":"is a french artificial intelligence (ai) company headquartered in paris"},"Moonshot AI":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Moonshot_AI&oldid=1380036373","quote":"is an artificial intelligence (ai) company based in beijing , china"},"Motif Technologies":{"country":"South Korea","url":"https://web.archive.org/web/20260611074323/https://en.wowtale.net/2026/05/28/234131/","quote":"a south korean ai company building large-scale foundation models from the ground up"},"NVIDIA":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Nvidia&oldid=1380904349","quote":"is an american multinational technology company headquartered in santa clara, california"},"OpenAI":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=OpenAI&oldid=1380903030","quote":"headquartered in san francisco , california, led by ceo sam altman"},"Poolside":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Poolside_AI&oldid=1375163829","quote":"it is headquartered in san francisco"},"Qwen":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Alibaba_Cloud&oldid=1376856898","quote":"alibaba cloud also develops the qwen series of open weights large language models . it is the largest cloud computing company in china"},"Reflection AI":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Reflection_AI&oldid=1379939634","quote":"reflection is headquartered in new york city"},"SK Telecom":{"country":"South Korea","url":"https://en.wikipedia.org/w/index.php?title=SK_Telecom&oldid=1378301817","quote":"is a south korean wireless telecommunications operator"},"StepFun":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=StepFun&oldid=1380325424","quote":"is an artificial intelligence (ai) company based in shanghai , china"},"Swiss AI":{"country":"Switzerland","url":"https://web.archive.org/web/20260924045626/https://www.swiss-ai.org/","quote":"over 10 academic institutions across switzerland"},"Tencent":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Tencent&oldid=1380562568","quote":"is a chinese multinational technology conglomerate and holding company headquartered in shenzhen"},"Thinking Machines Lab":{"country":"United States","url":"https://en.wikipedia.org/w/index.php?title=Thinking_Machines_Lab&oldid=1380003250","quote":"the company is based in san francisco"},"Upstage":{"country":"South Korea","url":"https://en.wikipedia.org/w/index.php?title=Upstage_%28company%29&oldid=1369298162","quote":"is a south korean artificial intelligence company founded in 2020"},"Xiaomi":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Xiaomi&oldid=1380612502","quote":"headquartered in beijing , china"},"Z.ai":{"country":"China","url":"https://en.wikipedia.org/w/index.php?title=Z.ai&oldid=1376085152","quote":"is a chinese artificial intelligence company"}},"models":[{"id":"apertus-1-5-70b","name":"Apertus-v1.5-70B-base","org":"Swiss AI","country":"Switzerland","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/swiss-ai/Apertus-v1.5-70B-base/resolve/0f3fbdd07bad839bb0c71b23d5adad0433dfcd16/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/swiss-ai/Apertus-v1.5-70B-base/resolve/0f3fbdd07bad839bb0c71b23d5adad0433dfcd16/config.json","kind":"config"},"index":{"url":"https://huggingface.co/swiss-ai/Apertus-v1.5-70B-base/resolve/0f3fbdd07bad839bb0c71b23d5adad0433dfcd16/model.safetensors.index.json","kind":"config"},"modeling_apertus1p5":{"url":"https://raw.githubusercontent.com/swiss-ai/transformers/0c5e12f18a5ff5c93cb1ef755aebbd557e26e884/src/transformers/models/apertus1p5/modeling_apertus1p5.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2610.15542v1","kind":"paper"},"report_v1":{"url":"https://arxiv.org/pdf/2509.14233v2","kind":"paper"},"announcement":{"url":"https://web.archive.org/web/20261019095313/https://ai.epfl.ch/apertus-1-5-building-the-next-generation-of-open-ai-infrastructure/","kind":"blog"}},"facts":{"meta.released":{"value":"2026-07-24","source":"announcement","at":"EPFL AI Center news, Apertus 1.5: Building the next generation of open AI infrastructure","quote":"Published 24 July, 2026 EPFL, ETH Zurich and the Swiss National Supercomputing Centre (CSCS) today released Apertus 1.5","note":"Release of the Apertus 1.5 suite on Hugging Face; the separate -base repository was created 2026-09-23 and the report is dated 2026-10-11."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + audio","source":"report","at":"p.6 §1","quote":"The models accept image and audio inputs in addition to text"},"params.total":{"value":70000000000,"source":"card","at":"README.md; intro","quote":"Apertus 1.5 is a family of 8B and 70B fully open, multilingual, and multimodal models","note":"Size designation; no exact count is reported."},"params.active":{"value":70000000000,"source":"card","at":"README.md; intro","quote":"Apertus 1.5 is a family of 8B and 70B fully open, multilingual, and multimodal models","basis":"inferred","note":"Dense model; active equals the stated size."},"params.note":{"value":"70B is a size designation; the checkpoint's 71,942,010,689 parameters include the image and audio tokenizers and the 266,752-row input embedding.","source":"index","at":"model.safetensors.index.json; metadata","quote":"\"total_parameters\": 71942010689"},"shape.layers":{"value":80,"source":"config","at":"/text_config/num_hidden_layers"},"shape.d_model":{"value":8192,"source":"config","at":"/text_config/hidden_size"},"shape.vocab":{"value":266752,"source":"config","at":"/text_config/vocab_size","note":"Input embedding only: 131,072 text + 200 reserved + 131,072 image + 4,096 audio + 312 padding rows; the output head keeps the 131,072 text rows (/text_config/output_vocab_size)."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.tokenizer":{"value":"byte-level BPE (Mistral-Nemo tekken, 131,072) extended with 200 reserved slots, 131,072 image codes (Emu3.5 vision tokenizer, one per 16×16 patch) and 4,096 audio codes (WavTokenizer)","source":"report","at":"p.9 §2.1 Vocabulary Extension","quote":"We extend the 131,072-entry vocabulary of Apertus 1.0 with 200 reserved multimodal slots, including structure markers, followed by 131,072 image codes","note":"Text tokenizer adapted from Mistral-Nemo-Base-2407 per the Apertus 1.0 report §2.2."},"attn.kind":{"value":"GQA","source":"report_v1","at":"p.9 §2.1 Model Architecture","quote":"such as grouped-query attention (GQA), RoPE, and RMSNorm"},"attn.layout":{"value":"80 GQA","source":"report_v1","at":"p.9 §2.1","quote":"Apertus 70B, with 80 layers and 64 parallel attention heads."},"attn.global_layers":{"value":80,"source":"config","at":"/text_config/num_hidden_layers"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"modeling_apertus1p5","at":"modeling_apertus1p5.py; Apertus1p5TextAttention.__init__","quote":"self.head_dim = getattr(config, \"head_dim\", config.hidden_size // config.num_attention_heads)","basis":"inferred","note":"text_config has no head_dim, so hidden_size 8192 / 64 heads = 128."},"attn.position":{"value":"RoPE (Llama-3-style frequency scaling)","source":"report_v1","at":"p.9 §2.1 Rotary Positional Embeddings","quote":"following the LLaMA-3 implementation","note":"Released config: rope_type llama3, factor 32, low_freq_factor 1, high_freq_factor 4, original_max_position_embeddings 8192."},"attn.rope_theta":{"value":4000000,"source":"config","at":"/text_config/rope_parameters/rope_theta","note":"500,000 during Apertus 1.0 pretraining and Apertus 1.5 stages 1–3; 2M at 32K, 4M from 65K."},"attn.qk_norm":{"value":true,"source":"modeling_apertus1p5","at":"modeling_apertus1p5.py; Apertus1p5TextAttention.__init__","quote":"self.q_norm = Apertus1p5TextRMSNorm(self.head_dim, config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K before RoPE; config qk_norm true, code builds the norms unconditionally."},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"ffn.activation":{"value":"xIELU (non-gated; trainable α_p, α_n per layer)","source":"modeling_apertus1p5","at":"modeling_apertus1p5.py; Apertus1p5TextMLP.forward","quote":"return self.down_proj(self.act_fn(self.up_proj(x)))","note":"Config hidden_act xielu. xIELU(x) = α_p·x² + 0.5x for x > 0, α_n(eˣ − 1) − α_n·x + 0.5x otherwise (Apertus 1.0 report §2.1)."},"ffn.dim":{"value":43008,"source":"config","at":"/text_config/intermediate_size"},"ffn.bias":{"value":false,"source":"config","at":"/text_config/mlp_bias"},"norm.type":{"value":"RMSNorm","source":"modeling_apertus1p5","at":"modeling_apertus1p5.py; Apertus1p5TextDecoderLayer.__init__","quote":"self.attention_layernorm = Apertus1p5TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"modeling_apertus1p5","at":"modeling_apertus1p5.py; Apertus1p5TextDecoderLayer.forward","quote":"hidden_states = self.attention_layernorm(hidden_states)"},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.pretrain":{"value":4096,"source":"report_v1","at":"p.10 §2.1 Context length","quote":"Both Apertus 8B and Apertus 70B were trained with a context of 4,096 tokens (about 3,000 words) during pretraining.","note":"Apertus 1.5 continued pretraining stages 1–3 use 8,192."},"ctx.final":{"value":262144,"source":"report","at":"p.23 §3.3","quote":"from 8,192 to 32,768, 65,536, 131,072, and finally 262,144 tokens."},"ctx.max":{"value":262144,"source":"card","at":"README.md; intro","quote":"handle contexts up to 262,144 tokens","note":"Config max_position_embeddings 262144."},"ctx.extension":{"value":"8K → 32K → 65K → 131K → 262K after the stage-3 cooldown, 58.7B, 58.7B, 50.3B, 50.3B tokens; constant LR 2e-6, fresh optimizer per step; RoPE base 2M then 4M, Llama-3 scaling factor 8, 8, 16, 32; context parallelism 4, 8, 16, 32","source":"report","at":"p.107 App. G","quote":"The RoPE base grows to 2M for the first step and 4M thereafter, with scaling factors of 8, 8, 16, and 32.","note":"Apertus 1.5 continued pretraining resumes from the Apertus 1.0 checkpoint after its cooldown, at RoPE base 500,000 and factor 8; the Apertus 1.0 4K → 64K extension is not in this lineage."},"train.tokens":{"value":17232100000000,"source":"report","at":"p.6 §1; p.22 Table 3; p.106 Table F.2","quote":"trained from scratch on 15T tokens of openly documented text","basis":"inferred","note":"Sum of train.stages: 15T Apertus 1.0 + 2,014B Apertus 1.5 stages 1–3 (Table 3: 403B + 1,208B + 403B) + 218.1B long-context (Table F.2: 58.72B + 58.72B + 50.33B + 50.33B)."},"train.optimizer":{"value":"AdEMAMix (α 8, β3 0.9999, α and β3 warmup over 100,000 iterations)","source":"report","at":"p.107 App. G","quote":"Both models train with AdEMAMix (β 1 = 0.9, β 2 = 0.999, α = 8, β 3 = 0.9999, with the β 3 and α warmup over 100,000 iterations)","note":"Same settings in Apertus 1.0 pretraining (Apertus 1.0 report Table C.4); optimizer state reset at the start of continued pretraining."},"train.betas":{"value":"(0.9, 0.999)","source":"report","at":"p.107 App. G","quote":"Both models train with AdEMAMix (β 1 = 0.9, β 2 = 0.999, α = 8, β 3 = 0.9999"},"train.weight_decay":{"value":0.1,"source":"report","at":"p.107 App. G","quote":"gradient clipping at 0.1, and weight decay of 0.1, applied also to the xIELU parameters and the normalization gains."},"train.grad_clip":{"value":0.1,"source":"report_v1","at":"p.15 §2.6 Gradient Clipping","quote":"Our experiments led to set a clipping value of 0.1.","note":"Clipping is active at almost every step; also 0.1 in continued pretraining (Apertus 1.5 report App. G)."},"train.lr_peak":{"value":1e-05,"source":"report_v1","at":"p.11 Table 2","quote":"Apertus 70B AdEMAMix 4096 8.4M → 16.8M 1.1M 1.0e-5 15T","note":"Apertus 1.0 pretraining; Apertus 1.5 continued pretraining re-warms to 2e-5."},"train.lr_min":{"value":2e-06,"source":"report","at":"p.107 Table G.1","quote":"Minimum learning rate 1.1 × 10−5 2 × 10−6","note":"End of the Apertus 1.5 stage-3 cooldown, held through context extension; Apertus 1.0 cooled down to 1e-6."},"train.schedule":{"value":"Apertus 1.0: WSD, warmup from 0.1× peak, constant 1e-5, 1-sqrt cooldown to 1e-6 over 13.5T–15T; Apertus 1.5: re-warm 0 → 2e-5, constant through stages 1–2, 1-sqrt decay to 2e-6 over stage 3, constant 2e-6 in context extension","source":"report","at":"p.20 §3.2","quote":"the learning rate then stays at its peak through the first two stages and decays to 10% of the peak with a 1 − sqrt decay over the third.","note":"Apertus 1.0 schedule: Apertus 1.0 report §2.3 Learning Rate Schedule and Cooldown."},"train.warmup":{"value":"16.78B tokens from 0.1× peak (Apertus 1.0); 250 iterations, 4.2B tokens, from 0 (Apertus 1.5)","source":"report_v1","at":"p.12 §2.3; p.82 Table C.4","quote":"Our LR warmup for both models starts from 0.1 the peak LR and is linearly increased for 16.8B tokens.","note":"Table C.4: 16.78BT. Apertus 1.5 warmup from Table G.1 and §3.2."},"train.batch":{"value":"Apertus 1.0: 2,048 × 4,096 (8.4M tokens), doubled to 16.8M tokens at 4.4T; Apertus 1.5: 2,048 × 8,192 (16.8M tokens), held constant through context extension","source":"report_v1","at":"p.12 §2.3 Batch Size and Sequence Length","quote":"After 8T tokens for the 8B model and 4.4T for the 70B, we intentionally doubled both the number of nodes and the batch size at this stage, while keeping the learning rate unchanged.","note":"Apertus 1.5 values: Table G.1 global batch 2,048 at 8,192 and §3.3 (sequences halved per doubling of length)."},"train.batch_tokens":{"value":16777216,"source":"report_v1","at":"p.12 §2.3 Batch Size and Sequence Length","quote":"an initial batch size of 1024 (4.2M tokens) and 2048 (8.4M tokens) for the 8B and 70B models, respectively","basis":"inferred","note":"Apertus 1.0 after the doubling at 4.4T: 2 × 2,048 × 4,096; Apertus 1.5 keeps it (2,048 × 8,192)."},"train.precision":{"value":"BF16 (Apertus 1.0 pretraining; an FP8 trial on the 8B was rolled back)","source":"report_v1","at":"p.16 §2.6 FP8 Training","quote":"We therefore decided to roll back and continue with the BF16 training normally.","note":"The Apertus 1.5 report does not state the continued-pretraining precision."},"train.init":{"value":"Apertus 1.0: std 0.008944; Apertus 1.5: new image/audio embedding and output rows random at ~1/10 the norm of trained text rows","source":"report_v1","at":"p.82 Table C.4","quote":"Initialization std 0.008944","note":"Table C.4 lists one value without a per-size split. Row init from the Apertus 1.5 report §3.2."},"train.stability":{"value":"QK-norm; grad clip 0.1; weight decay on norm gains and xIELU parameters","source":"report","at":"p.21 §3.2","quote":"including normalization gains and the trainable parameters of the xIELU activation (Huang & Schlag, 2024), which we find helps reduce large activation and activation-gradient norms.","note":"Decay on gains and xIELU parameters is stated for continued pretraining; QK-norm and clip from the Apertus 1.0 report §2.1, §2.6."},"train.masking":{"value":"cross-document attention masking; position ids reset at document boundaries","source":"report","at":"p.107 App. G","quote":"All stages pack documents with cross-document attention masking: position identifiers reset at document boundaries, and end-of-document positions are excluded from the loss.","note":"Apertus 1.0 pretraining also masks cross-document attention (Apertus 1.0 report §2.1)."},"train.objective":{"value":"Goldfish loss (k = 50, h = 50: ~2% of targets dropped by hashed 50-token context); image and audio tokens are targets only in Apertus 1.5 stage 1","source":"report","at":"p.21 §3.2","quote":"We keep the Goldfish loss of Apertus 1.0 with its parameters unchanged","note":"k and h from Apertus 1.0 report Table C.4; ~0.3T of the 15T masked."},"train.data":{"value":"Apertus 1.0: 15T text in 1811 languages, robots.txt opt-outs applied retroactively, five data phases with rising math/code share (DCLM-Edu and FineWeb-2-HQ top 10% from 12T); Apertus 1.5 stages 1–3: 668B vision, 234B audio (both incl. paired text), 1,111B text-only tokens","source":"report_v1","at":"p.17 §3; p.22 §3.3","quote":"Our data contains 1811 languages (1868 language-script pairs)","note":"Apertus 1.5 per-stage modality split in its Table 3 and text mix in its Table 4."},"train.hardware":{"value":"NVIDIA GH200 on Alps (CSCS); Apertus 1.0 70B on 2048–4096 GPUs over several months; Apertus 1.5 on 512 nodes (2048 GPUs)","source":"report_v1","at":"p.43 §6.1.2","quote":"running, for the 70B model, at scales from 2048 to 4096 GPUs over several months.","note":"Apertus 1.5 node count from its Table G.1 and §6.1 (four GH200 per node)."},"train.accelerators":{"value":"2,048–4,096 GH200","source":"report_v1","at":"p.43 §6.1.2","quote":"running, for the 70B model, at scales from 2048 to 4096 GPUs over several months.","note":"Apertus 1.0 run; chip from the same report §6.1 (Alps GH200); Apertus 1.5 continued pretraining used 512 nodes (2,048 GPUs)."},"train.parallelism":{"value":"TP4 · PP8 · VPP2","source":"report","at":"p.107 Table G.1","quote":"TP4, PP8, VPP2","note":"Apertus 1.5 continued pretraining; §3.2 states it reuses the Apertus 1.0 tensor and pipeline parallelism."},"train.compute":{"value":"6.74e24 FLOPs and ~6M GPU-hours for the 15T Apertus 1.0 run; ~1.50M GPU-hours (logged segments) for Apertus 1.5 stages 1–3","source":"report_v1","at":"p.44 §6.2","quote":"the main run has consumed around 6 million GPU hours","note":"FLOPs from Apertus 1.0 report App. E; continued-pretraining GPU-hours from Apertus 1.5 report §6.3, excluding context extension."},"train.stages":{"value":[{"name":"Apertus 1.0 pretraining","tokens":15000000000000,"seq_len":4096,"lr":1e-05,"note":"From scratch; five data phases; batch 8.4M → 16.8M tokens at 4.4T; 1-sqrt cooldown 13.5T–15T to 1e-6.","quote":"trained from scratch on 15T tokens of openly documented text spanning over 1000 languages"},{"name":"Apertus 1.5 stage 1: unpaired multimodal","tokens":403000000000,"seq_len":8192,"lr":2e-05,"batch_sequences":2048,"note":"Text 74.8%, image 12.2%, audio 13.0%; loss on all modalities; fresh optimizer state, re-warmup over 250 iterations.","quote":"1 49B (12.2%) 52B (13.0%) 301B (74.8%) 403B"},{"name":"Apertus 1.5 stage 2: paired multimodal","tokens":1208000000000,"seq_len":8192,"lr":2e-05,"batch_sequences":2048,"note":"Image-text and audio-transcript pairs plus text; loss on text tokens only.","quote":"2 512B (42.4%) 90B (7.4%) 606B (50.1%) 1,208B"},{"name":"Apertus 1.5 stage 3: cooldown","tokens":403000000000,"seq_len":8192,"lr":2e-05,"batch_sequences":2048,"note":"Higher-quality data, ~3B instruction-formatted tokens; 1-sqrt decay 2e-5 → 2e-6.","quote":"3 107B (26.6%) 92B (22.7%) 204B (50.7%) 403B"},{"name":"Apertus 1.5 long-context 32K","tokens":58720000000,"seq_len":32768,"lr":2e-06,"note":"RoPE base 2M, factor 8; context parallelism 4.","quote":"Total 58,720,000,000 58,720,000,000 50,330,000,000 50,330,000,000"},{"name":"Apertus 1.5 long-context 65K","tokens":58720000000,"seq_len":65536,"lr":2e-06,"note":"RoPE base 4M, factor 8; context parallelism 8.","quote":"Total 58,720,000,000 58,720,000,000 50,330,000,000 50,330,000,000"},{"name":"Apertus 1.5 long-context 131K","tokens":50330000000,"seq_len":131072,"lr":2e-06,"note":"RoPE base 4M, factor 16; context parallelism 16.","quote":"Total 58,720,000,000 58,720,000,000 50,330,000,000 50,330,000,000"},{"name":"Apertus 1.5 long-context 262K","tokens":50330000000,"seq_len":262144,"lr":2e-06,"note":"RoPE base 4M, factor 32; context parallelism 32.","quote":"Total 58,720,000,000 58,720,000,000 50,330,000,000 50,330,000,000"}],"source":"report","at":"p.6 §1; p.22 Table 3; p.106 Table F.2; p.107 App. G"},"scaling.hp_rule":{"value":"peak LR 2e-5 = 2× Apertus 1.0 70B peak, chosen by ablation","source":"report","at":"p.20 §3.2","quote":"For the 70B model, we double the peak of Apertus 1.0 70B to 2 × 10−5 , which outperforms the original value in ablations","note":"Apertus 1.0 set no LR scaling rule (report_v1 p.16 §2.5)."},"scaling.ladder":{"value":"1.5B per-change ablations at 100B tokens, merged into a 3B run at 100B tokens","source":"report_v1","at":"p.13 §2.4 Table 3","quote":"we run a separate ablation experiment on a 1.5B model scale with 100B tokens of our main datamix. The baseline is a standard Llama-style decoder with AdamW and a tuned cosine learning rate schedule. After verification, we merge all successful changes into a 3B model with 100B tokens","note":"Apertus 1.0 architecture and optimizer ablations; Apertus 1.5 continues the 1.0 70B checkpoint."}},"not_disclosed":["params.active_non_embedding","mtp.depth"],"derived":{"params.counted_total":{"value":71711355200,"formula":"embedding + output head + all experts + other weights, swiss-ai/Apertus-v1.5-70B-base@0f3fbdd safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":3258974208,"formula":"input embedding + output head, swiss-ai/Apertus-v1.5-70B-base@0f3fbdd safetensors headers"},"params.counted_non_embedding":{"value":68452380992,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, swiss-ai/Apertus-v1.5-70B-base@0f3fbdd safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_active_non_embedding":{"value":68452380992.0,"formula":"all weights except embedding and output head, swiss-ai/Apertus-v1.5-70B-base@0f3fbdd safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":102.4,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":5.25,"formula":"ffn.dim / d_model"},"scaling.tpp":{"value":246.17285714285714,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":251.73850420212423,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":327680.0,"formula":"327680 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":42949672960.0,"formula":"327680 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":327680.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":4681.142857142857,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":613566756.5714285,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":327680.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":327680.0,"tokens_per_entry":1}],"formula":"Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"ax-k2","name":"A.X K2","org":"SK Telecom","country":"South Korea","kind":"reasoning","weights":"open","note":"Post-trained checkpoint (Think-Fusion SFT and RL); pretraining facts are from the technical report.","sources":{"card":{"url":"https://huggingface.co/skt/A.X-K2/resolve/2287ca456927eed33b899c404c0b2ffaf17aa09f/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/skt/A.X-K2/resolve/2287ca456927eed33b899c404c0b2ffaf17aa09f/config.json","kind":"config"},"weights-index":{"url":"https://huggingface.co/skt/A.X-K2/resolve/2287ca456927eed33b899c404c0b2ffaf17aa09f/model.safetensors.index.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/axk2/modeling_axk2.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2608.30181v1","kind":"paper"},"press":{"url":"https://web.archive.org/web/20261019095225/https://news.sktelecom.com/en/3204","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095049/https://artificialanalysis.ai/models/a-x-k2","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-29","source":"press","at":"SKT press release 'SK Telecom Unveils A.X K2', dated 2026.07.29; first paragraph","quote":"Starting today, the 688-billion-parameter A.X K2 model is available through the open-source community Hugging Face."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"report","at":"p.23 §7; Scope of Modalities","quote":"A.X K2 is currently text-only."},"params.active":{"value":33000000000,"source":"report","at":"p.3 §2.1","quote":"The A.X K2 model is a large-scale MoE language model with 688B total parameters and 33B active"},"params.total":{"value":688000000000,"source":"report","at":"p.3 §2.1","quote":"The A.X K2 model is a large-scale MoE language model with 688B total parameters and 33B active"},"shape.d_model":{"value":7168,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.tokenizer":{"value":"byte-level BPE, 163,840 tokens, inherited unchanged from A.X K1","source":"report","at":"p.4 §2.1","quote":"The tokenizer is inherited unchanged from A.X K1: a byte-level BPE (Sennrich et al., 2016) vocabulary of 163,840 tokens"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":163840,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.gate":{"value":"head-specific sigmoid gate on the attention output before o_proj, projected jointly with the query","source":"hf-transformers","at":"modeling_axk2.py; AXK2Attention.forward","quote":"attn_output = (attn_output * torch.sigmoid(gate_states.float())).to(attn_output.dtype)","note":"Report p.3 §2.1: the gate is applied throughout pre-training."},"attn.global_layers":{"value":61,"source":"report","at":"p.3 §2.1","quote":"Of the 61 transformer layers, the first is a dense feed-forward layer and the remaining 60 are MoE layers","note":"Every block uses MLA."},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA, DSA (SGA)","source":"report","at":"p.3 §2.1","quote":"The backbone uses Multi-head Latent Attention (MLA) (Liu et al., 2024), which improves Key-Value cache efficiency","note":"SGA = MLA plus the head-specific output gate and the top-k indexer (p.4 §2.2)."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"61 MLA","source":"report","at":"p.3 §2.1","quote":"Of the 61 transformer layers, the first is a dense feed-forward layer and the remaining 60 are MoE layers"},"attn.position":{"value":"partial RoPE (64-dim decoupled RoPE key shared across heads, 128 NoPE dims per head)","source":"hf-transformers","at":"modeling_axk2.py; AXK2Attention.forward","quote":"q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)","note":"q_rot/k_rot are the /qk_rope_head_dim slices; the /qk_nope_head_dim slices are not rotated."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank","note":"q_gate_proj up-projects the normalized and raw query latents concatenated (2 × 1536 inputs) to per-head queries and gates."},"attn.qk_norm":{"value":false,"source":"hf-transformers","at":"modeling_axk2.py; AXK2Attention.forward","quote":"k_pass = self.kv_a_layernorm(kv_pass).view(batch_size, 1, seq_length, self.kv_lora_rank)","note":"Report p.3 §2.1 says 'the query and key projections are normalized before the attention scores are computed (QK-normalization)', but the released code only RMS-normalizes the Q and KV latents (q_a_layernorm, kv_a_layernorm) before up-projection, as in DeepSeek-V3; per-head Q/K go to attention unnormalized."},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":1000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.sparse":{"value":"DSA-style indexer, top-2048 tokens per query; 64 indexer heads of dim 128, ReLU-scored","source":"report","at":"p.4 §2.2","quote":"which uses a lightweight indexer to select a small set of top-k tokens (k = 2048) per query","note":"Indexer shape from config /index_n_heads 64, /index_head_dim 128, /index_topk 2048; scores pass through ReLU (AXK2Indexer.forward)."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"attn.indexer_dim":{"value":128,"source":"config","at":"/index_head_dim","note":"AXK2Indexer caches one key head: wk projects hidden_size to index_head_dim."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_axk2.py; AXK2MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (/hidden_act)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_axk2.py; AXK2MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"moe.balancing":{"value":"global auxiliary loss (4e-3 general and reasoning stages, 1e-3 long-context, 5e-4 SGA adaptation) plus a learnable expert bias (update rate 1e-3, lowered to 1e-4)","source":"report","at":"p.8 §3.6","quote":"the (global, sequence-level) auxiliary-loss coefficients are set to (4 × 10−3 , off)","note":"Sequence-level term is monitored but not applied in the general and reasoning stages; it is switched on at 1e-5 for the long-context stages and 2e-5 for SGA adaptation."},"moe.dense_layers":{"value":1,"source":"config","at":"/first_k_dense_replace","note":"moe_layer_freq 1: every later layer is MoE."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.groups":{"value":"256 experts in 8 groups, top-4 groups per token","source":"report","at":"p.8 §3.6","quote":"the 256 routed experts are partitioned into 8 groups, and each token activates the top 8 experts under a group top-k of 4","note":"Config /n_group 8, /topk_group 4."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.released":{"value":false,"source":"weights-index","at":"/weight_map | has mtp.","note":"No MTP, nextn or layers.61 tensors in the weight map; /num_nextn_predict_layers is 0; the report does not mention MTP."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_axk2.py; AXK2DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_axk2.py; AXK2GatedRMSNorm","quote":"return (y * torch.sigmoid(self.mlp(y).float())).to(y.dtype)","note":"Gated Norm: each block RMSNorm except the dense layer 0's pre-FFN norm is followed by an input-dependent sigmoid gate from a rank-16 SiLU MLP (/gated_norm_rank)."},"ctx.extension":{"value":"ABF: RoPE base 1e4 through stages 1-2, raised to 1e6 for the 32K and 128K stages; 256K at inference by raising the YaRN factor","source":"report","at":"p.7 §3.4","quote":"the RoPE base is kept at its default value of 104 throughout Stages 1 and 2 (sequence length 4,096), raised to 106 at the start of the 32K extension (Stage 3A)","note":"Released config: rope_type yarn, factor 2 over original_max_position_embeddings 131072, beta_fast 32, beta_slow 1, mscale 1, mscale_all_dim 1 (the code also multiplies the softmax scale by (0.1·ln 2 + 1)²)."},"ctx.final":{"value":131072,"source":"report","at":"p.7 §3.4","quote":"we natively extend the context length to 32K (Stage 3A, approximately 0.22T tokens) and subsequently to 128K (Stage 3B)"},"ctx.max":{"value":262144,"source":"config","at":"/max_position_embeddings"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.6 §3.3","quote":"A.X K2 is trained on a corpus of 6.4 trillion tokens at a sequence length of 4,096"},"train.averaging":{"value":"weight-space average of the last 6 checkpoints, ~1.6B tokens apart, with 1-sqrt weighting","source":"report","at":"p.8 §3.5","quote":"we average the parameters of the last six checkpoints, spaced approximately 1.6B tokens apart (25 optimizer steps of ≈64M tokens each), directly in weight space."},"train.batch":{"value":"16,384 × 4,096 (67.1M tokens); ramped from 2,048 sequences in steps of 2,048 over the first 72M sequences (~295B tokens)","source":"report","at":"p.8 §3.6","quote":"linearly increasing the batch size from 2,048 to 16,384 in increments of 2,048 over the first 72 million sequences (approximately 295B tokens at a sequence length of 4,096)"},"train.batch_tokens":{"value":67108864,"source":"report","at":"p.8 §3.6","quote":"the global batch size to 16,384 sequences (with a micro-batch size of 4), corresponding to approximately 64M tokens per step at a sequence length of 4,096","basis":"inferred","note":"16,384 sequences × 4,096 tokens, after the 2,048 → 16,384 ramp."},"train.data":{"value":"web, code, academic papers, STEM, books, Q&A and synthetic; stage-1 designed mix English 72.7%, Korean 15.4%, code 8.3%","source":"card","at":"README.md; Training Data","quote":"The designed language mix of the first stage is English 72.7%, Korean 15.4%, code 8.3%","note":"A 16.2T candidate pool was filtered to the 8.2T trained on (report p.5 §3.2); English web is primarily Nemotron-CC v2.1, code Nemotron-Pretraining-Code-v2, multilingual FineWeb2."},"train.hardware":{"value":"512 NVIDIA B200 GPUs (64 nodes of 8) for ~70 days","source":"report","at":"p.9 §3.7","quote":"Pre-training was conducted on 512 NVIDIA B200 GPUs (64 nodes of 8 GPUs) over approximately 70 days.","note":"PP 8 with virtual-pipeline degree 2, EP 8, no tensor parallelism; CP 2 for the 32K stage and 8 for 128K."},"train.accelerators":{"value":"512 B200","source":"report","at":"p.9 §3.7","quote":"Pre-training was conducted on 512 NVIDIA B200 GPUs (64 nodes of 8 GPUs) over approximately 70 days."},"train.parallelism":{"value":"PP8 · VPP2 · EP8 · DP (distributed optimizer)","source":"report","at":"p.9 §3.7","quote":"We used pipeline parallelism (PP) of 8 with an interleaved virtual-pipeline degree of 2, and expert parallelism (EP) of 8, with the remaining ranks assigned to data parallelism; tensor parallelism was not used.","note":"No TP; CP 2 for the 32K stage and 8 for the 128K stage."},"train.lr_min":{"value":2.2e-05,"source":"report","at":"p.9 §3.6","quote":"the 128K stage warms up to and then holds 2.2 × 10−5 , a level maintained through the SGA-adaptation stage"},"train.lr_peak":{"value":0.00022,"source":"report","at":"p.8 §3.6","quote":"The peak learning rate was set to 2.2 × 10−4 and the global batch size to 16,384 sequences"},"train.optimizer":{"value":"AdamW (eps 1e-8)","source":"report","at":"p.8 §3.6","quote":"We optimize with AdamW (β 1 = 0.9, β 2 = 0.95, ϵ = 10−8 ), following GPT-3"},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"p.8 §3.6","quote":"We optimize with AdamW (β 1 = 0.9, β 2 = 0.95, ϵ = 10−8 ), following GPT-3"},"train.precision":{"value":"MXFP8 (E4M3) forward and backward, one scale per 32 consecutive elements; FP32 master weights and gradients, BF16 optimizer moments; FP8 parameter all-gather","source":"report","at":"p.10 §3.7","quote":"We used the E4M3 format for both the forward and backward passes.","note":"Released checkpoint is blockwise FP8 with 128×128 weight blocks, a separate serving recipe."},"train.schedule":{"value":"WSD: warmup over ~25B tokens, constant 2.2e-4 through stage 1, cosine to 7e-5 across stage 2; 32K stage re-warms to 7e-5 and cosine-decays to 2.2e-5; 128K stage warms to and holds 2.2e-5","source":"report","at":"p.8-9 §3.6","quote":"the 32K stage warms up to 7 × 10−5 and cosine-decays to 2.2 × 10−5"},"train.stability":{"value":"Gated Norm (sigmoid gate after the block RMSNorms), replacing the dual-normalization scheme of A.X K1","source":"report","at":"p.3 §2.1","quote":"with GN in place, training remained sufficiently stable without the dual-normalization scheme used in A.X K1.","note":"Gating costs ~5% training throughput."},"train.stages":{"value":[{"name":"stage 1 - general knowledge","tokens":6400000000000,"seq_len":4096,"quote":"A.X K2 is trained on a corpus of 6.4 trillion tokens at a sequence length of 4,096"},{"name":"stage 2 - high-quality reasoning","tokens":1400000000000,"seq_len":4096,"quote":"Using a curated corpus of 1.4 trillion tokens at a sequence length of 4,096, this stage is strictly filtered for high-complexity web data"},{"name":"stage 3A - 32K extension","tokens":"~0.22T","seq_len":32768,"quote":"we natively extend the context length to 32K (Stage 3A, approximately 0.22T tokens)"},{"name":"stage 3B/3C - 128K extension and SGA adaptation","tokens":"~0.14T","seq_len":131072,"quote":"Stages 3B and 3C together consume approximately 0.14T tokens."}],"source":"report","at":"p.6-7 §3.3, §3.4","note":"Stage 3C adds the sparse indexer: an indexer-only sparse warmup with the backbone frozen, then joint training with the indexer loss down-weighted."},"train.tokens":{"value":8200000000000,"source":"report","at":"p.5 §3.1","quote":"A.X K2 is pre-trained on approximately 8.2 trillion tokens","note":"~8.5T including post-training (p.1 §1)."},"train.warmup":{"value":"~6.1M sequences (~25B tokens)","source":"report","at":"p.8 §3.6","quote":"the learning rate is linearly warmed up over the first ∼6.1M sequences (≈25B tokens)"},"scaling.hp_rule":{"value":"LR and batch magnitudes from the MoE scaling laws of Tian et al. (2025)","source":"report","at":"p.8 §3.6","quote":"The magnitudes of the learning rate and batch size were selected based on the MoE scaling laws proposed by Tian et al. (2025)"},"scaling.ladder":{"value":"20B-A3B dry run at sequence length 4096 (Gated Norm ablation)","source":"report","at":"p.4 Figure 3","quote":"run, 20B-A3B, sequence length 4096). Adding GN"},"eval.aa_index":{"value":21,"source":"aa","at":"Artificial Analysis model page /models/a-x-k2, Wayback snapshot 2026-10-19","quote":"A.X-K2 scores 21 on the Artificial Analysis Intelligence Index","note":"Variant: A.X-K2. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","train.weight_decay","train.grad_clip"],"derived":{"params.counted_total":{"value":691690231040,"formula":"embedding + output head + all experts + other weights, skt/A.X-K2@2287ca4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":2348810240,"formula":"input embedding + output head, skt/A.X-K2@2287ca4 safetensors headers"},"params.counted_non_embedding":{"value":689341420800,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, skt/A.X-K2@2287ca4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_active_non_embedding":{"value":34023363840.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, skt/A.X-K2@2287ca4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2857142857142857,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04796511627906977,"formula":"params.active / params.total"},"scaling.tpp":{"value":11.918604651162791,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":241.01085473387454,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":85888.0,"formula":"85888 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":11257511936.0,"formula":"85888 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":85888.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2602.6666666666665,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":341136725.3333333,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":85888.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":85888.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (704 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"beam-1-0","name":"Beam-1.0-501B-A23B","org":"Reflection AI","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"announcement":{"url":"https://web.archive.org/web/20261016133639/https://reflection.ai/blog/introducing-beam","kind":"blog"},"card":{"url":"https://huggingface.co/reflection/Beam-1.0-501B-A23B/resolve/78f8f654d40a40919b8e9954b40dd879ecb9a692/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/reflection/Beam-1.0-501B-A23B/resolve/78f8f654d40a40919b8e9954b40dd879ecb9a692/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/beam/modeling_beam.py","kind":"code"},"available":{"url":"https://web.archive.org/web/20261016165656/https://reflection.ai/blog/beam-is-available","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095120/https://artificialanalysis.ai/models/reflection-beam","kind":"leaderboard"}},"facts":{"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.released":{"value":"2026-10-16","source":"available","at":"Beam is now available (post header)","quote":"Beam is now available October 16, 2026","note":"Post: 'Today, we have released Beam’s model weights on Hugging Face'; the Oct 5 announcement preceded the weights."},"meta.modality":{"value":"text","source":"card","at":"README.md; opening description","quote":"Beam takes text as input and generates text as output."},"params.total":{"value":501300000000,"source":"card","at":"README.md; Model Overview table","quote":"Total Parameters 501.3B"},"params.active":{"value":23400000000,"source":"card","at":"README.md; Model Overview table","quote":"Active Parameters 23.4B"},"shape.d_model":{"value":5120,"source":"config","at":"/hidden_size"},"shape.layers":{"value":52,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":151936,"source":"config","at":"/vocab_size"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; Model Overview table; Attention","quote":"Gated GQA + Sliding Window Attention"},"attn.layout":{"value":"5 SWA : 1 global ×8, then 3 SWA : 1 global","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","basis":"inferred","note":"/layer_types has full_attention at layers 5, 11, ..., 47 and 51; all others sliding_attention. Ordered 52-layer backbone from /layer_types; auxiliary modules are excluded.","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":9,"source":"config","at":"/layer_types | count full_attention"},"attn.local_layers":{"value":43,"source":"config","at":"/layer_types | count sliding_attention"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads","note":"Same count on SWA and global layers (BeamAttention)."},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.window":{"value":512,"source":"config","at":"/sliding_window"},"attn.local_heads":{"value":48,"source":"config","at":"/num_attention_heads","note":"BeamAttention builds q/k/v from the same num_attention_heads, num_key_value_heads and head_dim for both layer types; only sliding_window differs."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.local_rope_theta":{"value":10000.0,"source":"config","at":"/rope_parameters/sliding_attention/rope_theta"},"attn.position":{"value":"RoPE (SWA), NoPE (global)","source":"hf-transformers","at":"modeling_beam.py; BeamAttention.forward","quote":"# Sliding-attention layers use RoPE; full-attention layers use NoPE with position-dependent query scaling.","note":"Config /rope_parameters: sliding_attention rope_theta 10000.0, full_attention null."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_beam.py; BeamAttention.__init__","quote":"self.q_norm = BeamRMSNorm(self.head_dim, eps=config.rms_norm_eps)"},"attn.scale":{"value":"1/sqrt(head_dim); global (NoPE) layers also scale q by (1 + n/2048)^0.1 at 1-based position n (SSMax-style)","source":"hf-transformers","at":"modeling_beam.py; compute_attention_scaling","quote":"Scalable-softmax query scale, `(1 + position / 2048) ** 0.1`, for layers that run without RoPE.","note":"Code computes exp(0.1 · log1p((position_ids + 1) / 2048)) with 0-based position_ids; SWA layers are unscaled."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.gate":{"value":"elementwise sigmoid gate, gate_proj of the normed attention input plus a per-head bias, before o_proj","source":"hf-transformers","at":"modeling_beam.py; BeamAttention.forward","quote":"gate = self.gate_proj(hidden_states).view_as(attn_output) + self.gate_bias[None, None, :, None]"},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_beam.py; BeamMLP.forward","quote":"# Beam evaluates the SwiGLU activation and product in FP32.","note":"act_fn is config /hidden_act silu; routed experts use the same gate × up form (BeamExperts._apply_gate)."},"ffn.dim":{"value":20480,"source":"config","at":"/intermediate_size"},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_beam.py; BeamMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Routed experts hold only gate_up_proj and down_proj weights (BeamExperts)."},"moe.dense_layers":{"value":3,"source":"config","at":"/mlp_layer_types | count dense","note":"Card: 52: 3 dense + 49 MoE."},"moe.expert_dim":{"value":2560,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":0,"source":"hf-transformers","at":"modeling_beam.py; BeamMoE","quote":"final_hidden_states = final_hidden_states * self.routed_scaling_factor","basis":"inferred","note":"BeamMoE holds only the router and routed experts; no shared-expert module."},"moe.router":{"value":"sigmoid","source":"hf-transformers","at":"modeling_beam.py; BeamTopKRouter.forward","quote":"scores = router_logits.sigmoid()"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.groups":{"value":"8 groups, top-4 groups per token (group score = sum of top-2 biased scores)","source":"hf-transformers","at":"modeling_beam.py; BeamTopKRouter.forward","quote":"group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1)[1]","note":"Config /n_group 8, /topk_group 4."},"moe.balancing":{"value":"auxiliary-loss-free bias with cosine-decayed bias updates, plus sequence-level balancing","source":"announcement","at":"Pretraining a Foundation for Reasoning; Stable and balanced MoE optimization","quote":"introducing cosine decay of expert-bias updates to reduce routing perturbations later in training","note":"Also: 'Sequence-level balancing further encourages balanced expert utilization'. Coefficients not disclosed."},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_beam.py; BeamDecoderLayer.__init__","quote":"self.input_layernorm = BeamRMSNorm(config.hidden_size, config.rms_norm_eps)"},"norm.placement":{"value":"sandwich","source":"announcement","at":"Pretraining a Foundation for Reasoning; Stable and balanced MoE optimization","quote":"Combined with SandwichNorm, elementwise attention gating, and FP32 residual accumulation","note":"Post-norms apply before the depth-scaled residual add."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.residual":{"value":"depth-scaled sum r·sqrt((k-1)/k) + h/sqrt(k), k = 2(i+1) for attention and 2(i+1)+1 for FFN in 0-indexed layer i; FP32 residual accumulation","source":"announcement","at":"Pretraining a Foundation for Reasoning; Stable and balanced MoE optimization","quote":"we developed a depth-based scaling approach that counteracts activation growth as sublayer outputs accumulate, helping keep residual norms stable as model depth increases. Combined with SandwichNorm, elementwise attention gating, and FP32 residual accumulation","note":"Formula from modeling_beam.py depth_scaled_residual_add and BeamDecoderLayer.__init__."},"norm.logit_softcap":{"value":30,"source":"config","at":"/final_logit_softcapping"},"ctx.max":{"value":1048576,"source":"config","at":"/max_position_embeddings","note":"Card: context window of 1M tokens."},"ctx.extension":{"value":"extended to 1M during midtraining","source":"announcement","at":"Pretraining a Foundation for Reasoning; Building a strong prior for RL in Midtraining","quote":"Midtraining also extends Beam’s effective context length to 1M tokens.","note":"Method and pretraining sequence length not disclosed."},"train.tokens":{"value":23800000000000,"source":"announcement","at":"Introduction","quote":"We pretrained the model on 23.8 trillion diverse, curated, high-quality tokens from the web and proprietary licensed datasets","note":"Whether midtraining tokens are included is not stated."},"train.stages":{"value":[{"name":"pretraining","tokens":23800000000000,"quote":"Beam was pretrained on 23.8 trillion diverse high-quality tokens from the web, public sources, and proprietary licensed datasets."},{"name":"midtraining","note":"Reasoning-rich, tool-use and long-context data; extends effective context to 1M.","quote":"Midtraining also extends Beam’s effective context length to 1M tokens."}],"source":"announcement","at":"Pretraining a Foundation for Reasoning"},"train.hardware":{"value":"6,144 NVIDIA GB300 (NVL72), under four weeks","source":"announcement","at":"Pretraining a Foundation for Reasoning; Frontier-grade pretraining infrastructure","quote":"Beam was pretrained end-to-end in under four weeks on a cluster of 6,144 NVIDIA GB300 NVL72 GPUs."},"train.accelerators":{"value":"6,144 GB300","source":"announcement","at":"Pretraining a Foundation for Reasoning; Frontier-grade pretraining infrastructure","quote":"Beam was pretrained end-to-end in under four weeks on a cluster of 6,144 NVIDIA GB300 NVL72 GPUs."},"train.data":{"value":"web, code, PDF-derived STEM and licensed data; code and technical content repeated multiple times","source":"announcement","at":"Pretraining a Foundation for Reasoning; Quality-centric data curation","quote":"We repeated code and technical content multiple times to increase the model’s exposure over the course of its training horizon.","note":"About 95% of raw Internet tokens are eliminated through parsing, deduplication and curation."},"train.stability":{"value":"nine semi-automatic rewinds for gradient-norm spikes or suspected SDCs","source":"announcement","at":"Pretraining a Foundation for Reasoning; Frontier-grade pretraining infrastructure","quote":"We executed nine semi-automatic rewinds throughout, attributed either to non-deterministic gradient norm spikes or to suspected SDCs."},"train.cutoff":{"value":"2026-06-30 (knowledge cutoff)","source":"card","at":"README.md; Model Overview table; Knowledge cutoff","quote":"Knowledge cutoff June 30, 2026","note":"Card's knowledge cutoff for the released instruction-tuned model; no separate pretraining data cutoff is stated."},"eval.aa_index":{"value":30,"source":"aa","at":"Artificial Analysis model page /models/reflection-beam, Wayback snapshot 2026-10-19","quote":"Beam 1.0 scores 30 on the Artificial Analysis Intelligence Index","note":"Variant: Beam 1.0. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":501267612352,"formula":"embedding + output head + all experts + other weights, reflection/Beam-1.0-501B-A23B@78f8f65 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1555824640,"formula":"input embedding + output head, reflection/Beam-1.0-501B-A23B@78f8f65 safetensors headers"},"params.counted_non_embedding":{"value":499711787712,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, reflection/Beam-1.0-501B-A23B@78f8f65 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":10172518960,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, reflection/Beam-1.0-501B-A23B@78f8f65 safetensors headers"},"params.counted_active_non_embedding":{"value":21875704512.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, reflection/Beam-1.0-501B-A23B@78f8f65 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":98.46153846153847,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":4.777777777777778,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.0466786355475763,"formula":"params.active / params.total"},"scaling.tpp":{"value":47.476560941551966,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1087.9649607144959,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":36864.0,"formula":"36864 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":4922015744.0,"formula":"36864 B × 131072 (Global KV) + 176128 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":37552.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1604.786324786325,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":210342553.16239318,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":36864.0,"windows":[[176128.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":36864.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":176128.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 9 × (2048 × 16/8) B/entry; SWA KV: 43 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"command-a-plus","name":"Command A+ (command-a-plus-05-2026)","org":"Cohere","country":"Canada","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"blog":{"url":"https://web.archive.org/web/20261010184826/https://cohere.com/blog/command-a-plus","kind":"blog"},"card":{"url":"https://huggingface.co/CohereLabs/command-a-plus-05-2026-bf16/resolve/5fb6fde5fd12ff89356aae552e11883bc49f069b/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/CohereLabs/command-a-plus-05-2026-bf16/resolve/5fb6fde5fd12ff89356aae552e11883bc49f069b/config.json","kind":"config"},"index":{"url":"https://huggingface.co/CohereLabs/command-a-plus-05-2026-bf16/resolve/5fb6fde5fd12ff89356aae552e11883bc49f069b/model.safetensors.index.json","kind":"config"},"modeling_cohere2_moe":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019095258/https://artificialanalysis.ai/models/command-a-plus","kind":"leaderboard"}},"facts":{"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.released":{"value":"2026-05-20","source":"blog","at":"post header","quote":"May 20, 2026 4 minute read Introducing Command A+","note":"Post: 'Today, we’re releasing Command A+ open-source.'"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Model Details","quote":"**Input**: Text and images."},"params.active":{"value":25000000000,"source":"card","at":"README.md; Model Summary","quote":"Model Size: 25B active parameters, 218B total parameters"},"params.note":{"value":"Card does not state scope; the checkpoint index lists 218,750,277,872 parameters including the SigLIP vision tower and projector.","source":"index","at":"model.safetensors.index.json; metadata.total_parameters","quote":"\"total_parameters\": 218750277872"},"params.total":{"value":218000000000,"source":"card","at":"README.md; Model Summary","quote":"Model Size: 25B active parameters, 218B total parameters"},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":32,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":true,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.tokenizer":{"value":"New Cohere tokenizer, first used in Command A+","source":"blog","at":"Efficiency section","quote":"Command A+ is the first model to use our latest tokenizer"},"shape.vocab":{"value":262144,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":8,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":128,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; Model Architecture","quote":"The attention layers interleave sliding-window attention layers with Rotational Positional Embeddings and global attention layers without positional embeddings in a 3:1 ratio"},"attn.kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 SWA : 1 global","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Ordered 32-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.local_layers":{"value":24,"source":"config","at":"/text_config/layer_types | count sliding_attention"},"attn.position":{"value":"RoPE (SWA), NoPE (global)","source":"card","at":"README.md; Model Architecture","quote":"The attention layers interleave sliding-window attention layers with Rotational Positional Embeddings and global attention layers without positional embeddings","note":"config rope_theta 50000 applies only to the sliding-window layers; rotation is GPT-J-style interleaved pairs."},"attn.qk_norm":{"value":false,"source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeAttention.forward","quote":"query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)"},"attn.window":{"value":4096,"source":"config","at":"/text_config/sliding_window"},"attn.local_heads":{"value":128,"source":"config","at":"/text_config/num_attention_heads","note":"Cohere2MoeAttention uses the same num_attention_heads, num_key_value_heads and head_dim for both layer types; only sliding_window and RoPE differ."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.local_rope_theta":{"value":50000,"source":"config","at":"/text_config/rope_parameters/rope_theta","note":"Cohere2MoeRotaryEmbedding reads rope_parameters rope_theta; one rotary table, applied only on sliding-window layers."},"ffn.activation":{"value":"SwiGLU","source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn from config hidden_act silu; the shared expert (Cohere2MoeMLP) uses the same gate × up form."},"ffn.bias":{"value":false,"source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.balancing":{"value":"additive expert bias","source":"card","at":"README.md; Model Architecture","quote":"We use additive-bias-based load balancing to encourage balanced token load across all experts","note":"Dropless, token-choice routing."},"moe.dense_layers":{"value":0,"source":"config","at":"/text_config/first_k_dense_replace"},"moe.expert_dim":{"value":4096,"source":"config","at":"/text_config/intermediate_size"},"moe.experts":{"value":128,"source":"config","at":"/text_config/num_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_topk_prob"},"moe.router":{"value":"normalized sigmoid (over the top-k logits)","source":"card","at":"README.md; Model Architecture","quote":"swap out the softmax router activation function with a normalized sigmoid over the topk expert logits per token","note":"Config expert_selection_fn sigmoid with norm_topk_prob; Cohere2MoeTopKRouter takes top-k of raw logits, then sigmoid and renormalizes."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Architecture","quote":"and a single shared expert is applied to all tokens"},"moe.shared_dim":{"value":16384,"source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeSparseMoeBlock.__init__","quote":"intermediate_size=config.intermediate_size * config.num_shared_experts,","basis":"inferred","note":"intermediate_size 4096 × num_shared_experts 4 (config.json), built as one MLP; checkpoint shared_experts.gate_proj is [16384, 4096]. Combined as (routed + shared) / 2 (shared_expert_combination_strategy average)."},"moe.top_k":{"value":8,"source":"config","at":"/text_config/num_experts_per_tok"},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/layer_norm_eps"},"norm.placement":{"value":"parallel","source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeDecoderLayer.forward","quote":"hidden_states = residual + hidden_states_attention + hidden_states_mlp","note":"One pre-norm (input_layernorm) feeds both attention and MoE."},"norm.type":{"value":"LayerNorm (no bias)","source":"modeling_cohere2_moe","at":"modeling_cohere2_moe.py; Cohere2MoeDecoderLayer.__init__","quote":"else Cohere2MoeLayerNorm(hidden_size=config.hidden_size, eps=config.layer_norm_eps)","note":"rms_norm_eps is null in config.json, so LayerNorm (mean-subtracted, weight only) is used."},"ctx.max":{"value":131072,"source":"card","at":"README.md; Context Length","quote":"Command A+ supports a context length of 128K & 64K output length.","note":"Card: 128K input plus 64K output; config max_position_embeddings is 200,000."},"train.data":{"value":"48 languages","source":"card","at":"README.md; Languages covered","quote":"The model has been trained on 48 languages"},"eval.aa_index":{"value":13,"source":"aa","at":"Artificial Analysis model page /models/command-a-plus, Wayback snapshot 2026-10-19","quote":"Command A+ scores 13 on the Artificial Analysis Intelligence Index","note":"Variant: Command A+. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":218254938112,"formula":"embedding + output head + all experts + other weights, CohereLabs/command-a-plus-05-2026-bf16@5fb6fde safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_embedding":{"value":1073741824,"formula":"input embedding + output head, CohereLabs/command-a-plus-05-2026-bf16@5fb6fde safetensors headers"},"params.counted_non_embedding":{"value":217181196288,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, CohereLabs/command-a-plus-05-2026-bf16@5fb6fde safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_max_layer":{"value":6786912256,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, CohereLabs/command-a-plus-05-2026-bf16@5fb6fde safetensors headers"},"params.counted_active_non_embedding":{"value":23907667968.0,"formula":"non-expert weights + routed experts × 8/128, excluding embedding and output head, CohereLabs/command-a-plus-05-2026-bf16@5fb6fde safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"shape.aspect":{"value":128.0,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":1.0,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":16.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.11467889908256881,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":32768.0,"formula":"32768 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":4697620480.0,"formula":"32768 B × 131072 (Global KV) + 98304 B × min(131072, 4096) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":35840.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1433.6,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":187904819.2,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":32768.0,"windows":[[98304.0,4096]],"groups":[{"name":"Global KV","bytes_per_entry":32768.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":98304.0,"tokens_per_entry":1,"window":4096}],"formula":"Global KV: 8 × (2048 × 16/8) B/entry; SWA KV: 24 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"deepseek-v3","name":"DeepSeek-V3-Base","org":"DeepSeek","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/resolve/afb92e1fa402c2be2a9eb085312bb02e0384d6c7/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/resolve/afb92e1fa402c2be2a9eb085312bb02e0384d6c7/config.json","kind":"config"},"modeling_deepseek":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/resolve/afb92e1fa402c2be2a9eb085312bb02e0384d6c7/modeling_deepseek.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2412.19437v2","kind":"paper"},"weights-doc":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/resolve/afb92e1fa402c2be2a9eb085312bb02e0384d6c7/README_WEIGHTS.md","kind":"model-card"},"license":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/resolve/afb92e1fa402c2be2a9eb085312bb02e0384d6c7/LICENSE-MODEL","kind":"model-card"},"announcement":{"url":"https://web.archive.org/web/20261017101856/https://api-docs.deepseek.com/news/news1226/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095333/https://artificialanalysis.ai/models/deepseek-v3","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2024-12-26","source":"announcement","at":"DeepSeek API Docs news, Introducing DeepSeek-V3","quote":"Introducing DeepSeek-V3 2024/12/26"},"meta.license":{"value":"other (DeepSeek License Agreement)","source":"license","at":"LICENSE-MODEL, title","quote":"DEEPSEEK LICENSE AGREEMENT","note":"Model weights; code is MIT (README.md §7)."},"meta.modality":{"value":"text","basis":"inferred","source":"announcement","at":"DeepSeek API Docs news, Introducing DeepSeek-V3","quote":"Look forward to multimodal support and other cutting-edge features in the DeepSeek ecosystem.","note":"Multimodal support is announced as future work and the config (DeepseekV3ForCausalLM) has no vision tower."},"params.active":{"value":37000000000,"source":"report","at":"p.22 §4.2","quote":"DeepSeek-V3 comprises 671B total parameters, of which 37B are activated for each token."},"params.active_non_embedding":{"value":35700000000,"source":"weights-doc","at":"README_WEIGHTS.md; Main Model Weights, Parameter Count","quote":"Activation parameters: 36.6B (including 0.9B for the output Head).","basis":"inferred","note":"36.6B − 129,280 × 7,168 (untied output head, stated as included; input embedding not listed) = ~35.7B. The report rounds the activated count to 37B."},"params.note":{"value":"671B excludes the MTP module; the released checkpoint is 685B (671B main model + 14B MTP).","source":"card","at":"README.md; Model Downloads","quote":"The total size of DeepSeek-V3 models on HuggingFace is 685B, which includes 671B of the Main Model weights and 14B of the Multi-Token Prediction (MTP) Module weights."},"params.total":{"value":671000000000,"source":"report","at":"p.22 §4.2","quote":"DeepSeek-V3 comprises 671B total parameters, of which 37B are activated for each token."},"shape.d_model":{"value":7168,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.tokenizer":{"value":"byte-level BPE, 128K vocab; pretokenizer merges punctuation with line breaks, a proportion of these split at random in training","source":"report","at":"p.22 §4.1","quote":"The tokenizer for DeepSeek-V3 employs Byte-level BPE (Shibata et al., 1999) with an extended vocabulary of 128K tokens."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":129280,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":61,"source":"report","at":"p.22 §4.2","quote":"We set the number of Transformer layers to 61 and the hidden dimension to 7168.","note":"Every block uses MLA."},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim"},"attn.heads":{"value":128,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA","source":"report","at":"p.6 §2.1","quote":"the basic architecture of DeepSeek-V3 is still within the Transformer (Vaswani et al., 2017) framework. For efficient inference and economical training, DeepSeek-V3 also adopts MLA and DeepSeekMoE"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"61 MLA","source":"report","at":"p.22 §4.2","quote":"We set the number of Transformer layers to 61 and the hidden dimension to 7168."},"attn.position":{"value":"partial RoPE (decoupled 64-dim RoPE key shared across heads + 128 NoPE dims per head)","source":"report","at":"p.8 §2.1.1","quote":"to produce the decoupled key that carries Rotary Positional Embedding (RoPE)"},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"modeling_deepseek","at":"modeling_deepseek.py lines 750–836; DeepseekV3Attention.forward","quote":"torch.matmul(query_states, key_states.transpose(2, 3)) * self.softmax_scale","note":"RMSNorm is applied only to the Q and KV latents before up-projection; projected per-head Q/K go straight to the scaled dot product."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":10000,"source":"config","at":"/rope_theta"},"attn.scale":{"value":"1/sqrt(192) × (0.1·ln 40 + 1)² (YaRN mscale, s = 40)","source":"modeling_deepseek","at":"modeling_deepseek.py lines 689–695; DeepseekV3Attention.__init__","quote":"self.softmax_scale = self.softmax_scale * mscale * mscale","note":"mscale = 0.1·mscale_all_dim·ln(factor) + 1 with /rope_scaling/mscale_all_dim 1 and factor 40; the YaRN temperature from context extension (p.23 §4.3)."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_deepseek","at":"modeling_deepseek.py line 389; DeepseekV3MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (/hidden_act)."},"ffn.bias":{"value":false,"source":"modeling_deepseek","at":"modeling_deepseek.py line 383; DeepseekV3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Up and down projections also bias=False; dense FFN, routed and shared experts all use DeepseekV3MLP."},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"moe.balancing":{"value":"aux-loss-free bias (γ=0.001, frozen for the last 500B tokens) + sequence-wise loss α=1e-4","source":"report","at":"p.23 §4.2","quote":"we set the bias update speed 𝛾 to 0.001 for the first 14.3T tokens, and to 0.0 for the remaining 500B tokens. For the balance loss, we set 𝛼 to 0.0001","note":"Bias enters top-k selection only (config /topk_method noaux_tc)."},"moe.dense_layers":{"value":3,"source":"config","at":"/first_k_dense_replace","note":"All later layers are MoE (/moe_layer_freq 1)."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.groups":{"value":"node-limited: each token sent to at most 4 of 8 nodes (groups)","source":"report","at":"p.23 §4.2","quote":"As for the node-limited routing, each token will be sent to at most 4 nodes","note":"Nodes ranked by the sum of the top K_r/M affinities on each node (p.10 §2.1.2); config /n_group 8, /topk_group 4."},"moe.norm_topk":{"value":true,"source":"report","at":"p.9 §2.1.2","quote":"applies a normalization among all selected affinity scores to produce the gating values.","note":"config /norm_topk_prob true."},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"report","at":"p.22 §4.2","quote":"The multi-token prediction depth 𝐷 is set to 1, i.e., besides the exact next token, each token will predict one additional token."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.block":{"value":"h' = M_k[RMSNorm(h); RMSNorm(Emb(t_{i+k}))] (2d→d projection), then one Transformer block identical to the main layers (MLA + MoE)","source":"report","at":"p.10 §2.2, Eq. 21","quote":"a Transformer block TRM𝑘 (·), and a projection matrix 𝑀𝑘 ∈ R𝑑 ×2𝑑"},"mtp.ffn":{"value":"MoE","source":"weights-doc","at":"README_WEIGHTS.md; Multi-Token Prediction (MTP) Modules, Structural Details","quote":"model.layers.61.self_attn & mlp (structure identical to the Main Model hidden layers)","note":"Layer 61 is past /first_k_dense_replace 3, so 256 routed + 1 shared experts."},"mtp.shares_embedding":{"value":true,"source":"report","at":"p.10 §2.2","quote":"Note that for each MTP module, its embedding layer is shared with the main model.","note":"Output head is shared as well (same section)."},"mtp.params":{"value":11500000000,"source":"weights-doc","at":"README_WEIGHTS.md; Multi-Token Prediction (MTP) Modules, Parameter Count","quote":"Parameters: **11.5B unique parameters** (excluding the shared 0.9B Embedding and 0.9B output Head).","note":"Excludes the shared embedding and head; the model card gives 14B for the MTP module weights."},"mtp.loss_weight":{"value":"0.3 for the first 10T tokens, 0.1 for the remaining 4.8T","source":"report","at":"p.23 §4.2","quote":"The MTP loss weight 𝜆 is set to 0.3 for the first 10T tokens, and to 0.1 for the remaining 4.8T tokens."},"mtp.released":{"value":true,"source":"card","at":"README.md; Model Downloads","quote":"The total size of DeepSeek-V3 models on HuggingFace is 685B, which includes 671B of the Main Model weights and 14B of the Multi-Token Prediction (MTP) Module weights."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_deepseek","at":"modeling_deepseek.py lines 1195–1217; DeepseekV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_deepseek","at":"modeling_deepseek.py lines 1160–1165; DeepseekV3DecoderLayer.__init__","quote":"self.input_layernorm = DeepseekV3RMSNorm("},"ctx.extension":{"value":"YaRN (s=40, α=1, β=32) on the decoupled key; 4K → 32K (1000 steps, batch 1920) → 128K (1000 steps, batch 480), LR 7.3e-6","source":"report","at":"p.23 §4.3","quote":"we apply YaRN (Peng et al., 2023a) for context extension and perform two additional training phases, each comprising 1000 steps, to progressively expand the context window from 4K to 32K and then to 128K.","note":"config /rope_scaling: type yarn, factor 40, beta_fast 32, beta_slow 1, mscale 1, original_max_position_embeddings 4096."},"ctx.final":{"value":131072,"source":"report","at":"p.23 §4.3","quote":"During the second phase, the sequence length is increased to 128K, and the batch size is reduced to 480."},"ctx.max":{"value":131072,"source":"card","at":"README.md; Model Downloads table","quote":"DeepSeek-V3-Base | 671B | 37B | 128K","note":"config /max_position_embeddings 163840 is 4096 × YaRN factor 40, not the advertised length."},"ctx.pretrain":{"value":4096,"source":"report","at":"p.22 §4.2","quote":"We set the maximum sequence length to 4K during pre-training"},"train.batch":{"value":"3,072 → 15,360 × 4,096 (62.9M tokens) over the first 469B tokens, then constant","source":"report","at":"p.23 §4.2","quote":"the batch size is gradually increased from 3072 to 15360 in the training of the first 469B tokens, and then keeps 15360 in the remaining training."},"train.batch_tokens":{"value":62914560,"source":"report","at":"p.23 §4.2","quote":"and then keeps 15360 in the remaining training.","basis":"inferred","note":"15360 sequences × 4096 tokens (4K pretraining sequence length, p.22 §4.2), after the ramp from 3072 over the first 469B tokens."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"p.22 §4.2","quote":"with hyper-parameters set to 𝛽1 = 0.9, 𝛽2 = 0.95, and weight_decay = 0.1."},"train.grad_clip":{"value":1,"source":"report","at":"p.23 §4.2","quote":"The gradient clipping norm is set to 1.0."},"train.compute":{"value":"2.664M H800 GPU-hours pretraining + 119K context extension (2.788M with 5K post-training)","source":"report","at":"p.5 §1, Table 1","quote":"costs 2664K GPU hours. Combined with 119K GPU hours for the context length extension and 5K GPU hours for post-training"},"train.data":{"value":"14.8T tokens; math and code share raised vs DeepSeek-V2; multilingual coverage beyond English and Chinese","source":"report","at":"p.21 §4.1","quote":"enhancing the ratio of mathematical and programming samples"},"train.hardware":{"value":"2048 H800 GPUs","source":"report","at":"p.5 §1","quote":"3.7 days on our cluster with 2048 H800 GPUs","note":"Pretraining took less than two months (same paragraph)."},"train.accelerators":{"value":"2,048 H800","source":"report","at":"p.11 §3.1","quote":"DeepSeek-V3 is trained on a cluster equipped with 2048 NVIDIA H800 GPUs."},"train.parallelism":{"value":"PP16 · EP64 · ZeRO-1 DP","source":"report","at":"p.11 §3.2","quote":"DeepSeek-V3 applies 16-way Pipeline Parallelism (PP) (Qi et al., 2023a), 64-way Expert Parallelism (EP) (Lepikhin et al., 2021) spanning 8 nodes, and ZeRO-1 Data Parallelism (DP)"},"train.init":{"value":"random init, std 0.006 for all learnable parameters","source":"report","at":"p.22 §4.2","quote":"All learnable parameters are randomly initialized with a standard deviation of 0.006."},"train.lr_min":{"value":7.3e-06,"source":"report","at":"p.23 §4.2","quote":"of 7.3 × 10−6 in the remaining 167B tokens."},"train.masking":{"value":"packed documents, no cross-sample attention masking","source":"report","at":"p.22 §4.1","quote":"we implement the document packing method for data integrity but do not incorporate cross-sample attention masking during training."},"train.objective":{"value":"FIM at rate 0.1, PSM format, applied per document before packing","source":"report","at":"p.22 §4.1","quote":"The FIM strategy is applied at a rate of 0.1, consistent with the PSM framework."},"train.lr_peak":{"value":0.00022,"source":"report","at":"p.22 §4.2","quote":"we first linearly increase it from 0 to 2.2 × 10−4 during the first 2K steps. Then, we keep a constant learning rate of 2.2 × 10−4 until the model consumes 10T training tokens."},"train.optimizer":{"value":"AdamW","source":"report","at":"p.22 §4.2","quote":"We employ the AdamW optimizer (Loshchilov and Hutter, 2017) with hyper-parameters set to 𝛽1 = 0.9, 𝛽2 = 0.95, and weight_decay = 0.1."},"train.precision":{"value":"FP8 (E4M3) for Fprop, Dgrad and Wgrad of Linear layers; scales per 1×128 activation tile and 128×128 weight block; embedding, output head, MoE gating, norms and attention kept in BF16/FP32","source":"report","at":"p.15 §3.3.1","quote":"we maintain the original precision (e.g., BF16 or FP32) for the following components: the embedding module, the output head, MoE gating modules, normalization operators, and attention operators.","note":"E4M3 on all tensors and 1×128 / 128×128 scaling: p.16–17 §3.3.2. Optimizer moments in BF16, master weights and gradients in FP32 (p.18 §3.3.3)."},"train.schedule":{"value":"2K-step warmup, constant to 10T, cosine to 2.2e-5 over 4.3T, then 2.2e-5 for 333B and 7.3e-6 for 167B","source":"report","at":"p.22 §4.2","quote":"Subsequently, we gradually decay the learning rate to 2.2 × 10−5 in 4.3T tokens, following a cosine decay curve. During the training of the final 500B tokens, we keep a constant learning rate of 2.2 × 10−5 in the first 333B tokens"},"train.stages":{"value":[{"name":"main","tokens":14800000000000,"seq_len":4096,"quote":"We set the maximum sequence length to 4K during pre-training, and pre-train DeepSeek-V3 on 14.8T tokens."},{"name":"context-1","seq_len":32768,"lr":7.3e-06,"batch_sequences":1920,"steps":1000,"quote":"In the first phase, the sequence length is set to 32K, and the batch size is 1920."},{"name":"context-2","seq_len":131072,"lr":7.3e-06,"batch_sequences":480,"steps":1000,"quote":"During the second phase, the sequence length is increased to 128K, and the batch size is reduced to 480."}],"source":"report","at":"p.22–23 §4.2, §4.3"},"train.tokens":{"value":14800000000000,"source":"report","at":"p.22 §4.2","quote":"We set the maximum sequence length to 4K during pre-training, and pre-train DeepSeek-V3 on 14.8T tokens."},"train.warmup":{"value":"2000 steps","source":"report","at":"p.22 §4.2","quote":"we first linearly increase it from 0 to 2.2 × 10−4 during the first 2K steps."},"train.weight_decay":{"value":0.1,"source":"report","at":"p.22 §4.2","quote":"with hyper-parameters set to 𝛽1 = 0.9, 𝛽2 = 0.95, and weight_decay = 0.1."},"scaling.ladder":{"value":"15.7B total (2.4B active) on 1.33T tokens; 228.7B total (20.9B active) on 540B tokens","source":"report","at":"p.26 §4.5.1 Table 4","quote":"At the small scale, we train a baseline MoE model comprising 15.7B total parameters on 1.33T tokens. At the large scale, we train a baseline MoE model comprising 228.7B total parameters on 540B tokens.","note":"MTP ablation; aux-loss-free ablation uses the same sizes at 1.33T/578B tokens (Table 5), FP8 validation ~16B/1.33T and ~230B/~0.9T (App. B.1); active counts from Table 4."},"eval.aa_index":{"value":8,"source":"aa","at":"Artificial Analysis model page /models/deepseek-v3, Wayback snapshot 2026-10-19","quote":"DeepSeek V3 (Dec '24) scores 8 on the Artificial Analysis Intelligence Index","note":"Score of DeepSeek-V3, post-trained from this base. Variant: DeepSeek V3 (Dec '24). Index v4.3.2."}},"not_disclosed":[],"derived":{"params.counted_total":{"value":671026419200,"formula":"embedding + output head + all experts + other weights, deepseek-ai/DeepSeek-V3-Base@afb92e1 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":1853358080,"formula":"input embedding + output head, deepseek-ai/DeepSeek-V3-Base@afb92e1 safetensors headers"},"params.counted_non_embedding":{"value":669173061120,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, deepseek-ai/DeepSeek-V3-Base@afb92e1 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":11507286272,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, deepseek-ai/DeepSeek-V3-Base@afb92e1 safetensors headers"},"params.counted_active_non_embedding":{"value":35698939392.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, deepseek-ai/DeepSeek-V3-Base@afb92e1 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":24576,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2857142857142857,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.05514157973174367,"formula":"params.active / params.total"},"scaling.tpp":{"value":22.056631892697467,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":414.5781429942601,"formula":"train.tokens / params.counted_active_non_embedding"},"scaling.adamw_tau":{"value":0.19322653562653558,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":70272.0,"formula":"70272 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9210691584.0,"formula":"70272 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":70272.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1899.2432432432433,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":248937610.3783784,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":70272.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":70272.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"deepseek-v32","name":"DeepSeek-V3.2","org":"DeepSeek","country":"China","kind":"reasoning","weights":"open","note":"Post-trained release; the card names deepseek-ai/DeepSeek-V3.2-Exp-Base as its base. Recipe facts cover the DSA continued pre-training from a DeepSeek-V3.1-Terminus base checkpoint (already extended to 128K).","sources":{"flashmla-cache":{"url":"https://raw.githubusercontent.com/deepseek-ai/FlashMLA/07a1089857b63e74e3133630c02b083b75e8d4b2/README.md","kind":"code"},"config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/config.json","kind":"config"},"inference-config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/inference/config_671B_v3.2.json","kind":"config"},"inference-model":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/inference/model.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2512.02556v1","kind":"paper"},"transparency":{"url":"https://web.archive.org/web/20261019095912id_/https://fe-static.deepseek.com/chat/transparency/deepseek-v3.2-model-card-0414-EN.pdf","kind":"paper"},"weight-index":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/model.safetensors.index.json","kind":"config"},"card":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019095458/https://artificialanalysis.ai/models/deepseek-v3-2-reasoning","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-12-01","source":"transparency","at":"p.2; General Information","quote":"Release date December 1, 2025"},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","source":"transparency","at":"Model card PDF; Modalities","quote":"Both input and output are in Text format"},"params.active":{"value":37000000000,"source":"transparency","at":"p.3; Model properties; Total model size","quote":"671B parameters, of which 37B are activated for each token"},"params.total":{"value":671000000000,"source":"transparency","at":"p.3; Model properties; Total model size","quote":"671B parameters, of which 37B are activated for each token"},"shape.d_model":{"value":7168,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":129280,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim"},"attn.heads":{"value":128,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA, DSA","source":"report","at":"p.3 §2.1","quote":"we instantiate DSA based on MLA (DeepSeek-AI, 2024) for DeepSeek-V3.2."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.indexer_dim":{"value":128,"source":"config","at":"/index_head_dim","note":"One indexer key per token per layer (Indexer.wk projects dim to index_head_dim; k_cache stored FP8 with per-128 scales); /index_n_heads 64 query heads, /index_topk 2048."},"attn.kv_groups":{"value":[{"name":"MLA latent","layers":61,"elements":512,"quote":"self.register_buffer(\"kv_cache\", torch.zeros(args.max_batch_size, args.max_seq_len, self.kv_lora_rank), persistent=False)"},{"name":"RoPE key","layers":61,"elements":64,"quote":"self.register_buffer(\"pe_cache\", torch.zeros(args.max_batch_size, args.max_seq_len, self.qk_rope_head_dim), persistent=False)"},{"name":"DSA indexer key","layers":61,"elements":128,"quote":"self.register_buffer(\"k_cache\", torch.zeros(args.max_batch_size, args.max_seq_len, self.head_dim, dtype=torch.float8_e4m3fn), persistent=False)"}],"source":"inference-model","at":"inference/model.py; MLA.__init__ lines 541–542 and Indexer.__init__ lines 453–454"},"attn.kv_precision":{"value":[{"name":"MLA latent","group":"MLA latent","elements":512,"bits":8,"format":"FP8 E4M3; FP32 scales per 128 channels","scale_count":4,"scale_bits":32,"at":"README.md; V3.2 layout: 512 quantized latent bytes + 16 scale bytes","quote":"Next 16 bytes: Scale factors, containing 4 `float32` values."},{"name":"RoPE key","group":"RoPE key","elements":64,"bits":16,"format":"BF16","at":"README.md; V3.2 unquantized RoPE key","quote":"Last 128 bytes: The \"RoPE\" part, containing 64 `bfloat16` values."},{"name":"DSA indexer key","group":"DSA indexer key","elements":128,"bits":8,"format":"FP8 E4M3; one FP32 scale","scale_count":1,"scale_bits":32,"source":"inference-model","at":"inference/model.py lines 15, 453–454; block_size 128","quote":"self.register_buffer(\"k_scale_cache\", torch.zeros(args.max_batch_size, args.max_seq_len, self.head_dim // block_size, dtype=torch.float32), persistent=False)"}],"source":"flashmla-cache","at":"README.md; MLA Decoding / FP8 KV Cache; per-entry storage layouts","note":"Serving layout: each layer retains 656 bytes of main MLA KV and 132 bytes of indexer K per token. 61 × (656 + 132) = 48,068 bytes/token, matching DeepSeek-V4.1 report p.1 Figure 1(b). The reference model simulates main-latent FP8 then retains BF16 buffers; this fact describes the packed FlashMLA serving layout."},"attn.qk_norm":{"value":false,"source":"inference-model","at":"inference/model.py; MLA.forward","quote":"q = q.view(bsz, seqlen, self.n_local_heads, self.qk_head_dim)","note":"q_norm and kv_norm act on the Q and KV latents before up-projection; per-head Q/K are not normalized."},"attn.layout":{"value":"61 DSA","source":"inference-model","at":"inference/model.py lines 498–545; MLA.__init__","quote":"self.indexer = Indexer(args)","note":"Every MLA layer builds a lightning indexer; 61 layers per /num_hidden_layers."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":10000,"source":"config","at":"/rope_theta","note":"YaRN on the rope dims: /rope_scaling type yarn, factor 40, beta_fast 32, beta_slow 1, mscale 1, mscale_all_dim 1, original_max_position_embeddings 4096."},"attn.scale":{"value":"1/sqrt(192) × (0.1·ln 40 + 1)² (YaRN mscale, s = 40)","source":"inference-model","at":"inference/model.py lines 533–537; MLA.__init__","quote":"self.softmax_scale = self.softmax_scale * mscale * mscale","note":"Applied when the configured max_seq_len exceeds original_seq_len 4096 (default max_seq_len 16384); mscale = 0.1·ln(rope_factor 40) + 1."},"attn.position":{"value":"partial RoPE (decoupled 64-dim RoPE key shared across heads + 128 NoPE dims per head)","source":"inference-model","at":"inference/model.py line 568; MLA.forward","quote":"k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis)"},"attn.sparse":{"value":"DSA: lightning indexer (64 heads × 128, ReLU, FP8), top-2048 KV tokens per query","source":"report","at":"p.4 §2.1.1","quote":"select 2048 key-value tokens for each query token","note":"Indexer heads and dim from config /index_n_heads, /index_head_dim; ReLU and FP8 from p.3 §2.1."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"inference-model","at":"inference/model.py line 643 (MLP.forward); same in Expert.forward","quote":"return self.w2((F.silu(self.w1(x).float()) * self.w3(x).float()).type_as(x))","note":"config /hidden_act silu."},"ffn.bias":{"value":false,"source":"inference-model","at":"inference/model.py lines 620–631 (MLP.__init__) and 721–733 (Expert.__init__)","quote":"self.w1 = ColumnParallelLinear(dim, inter_dim)","note":"Linear and its parallel subclasses default to bias=False."},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":3,"source":"config","at":"/first_k_dense_replace","note":"All later layers are MoE (/moe_layer_freq 1)."},"moe.balancing":{"value":"auxiliary-loss-free expert bias (added to scores for top-k selection only); coefficients not stated","source":"inference-model","at":"inference/model.py; Gate.forward","quote":"scores = scores + self.bias","note":"config /topk_method noaux_tc; gating weights come from the unbiased scores."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.groups":{"value":"8 expert groups, top-4 groups per token (group score: sum of its top-2 biased expert scores)","source":"inference-config","at":"inference/config_671B_v3.2.json; /n_expert_groups, /n_limited_groups","quote":"\"n_expert_groups\": 8, \"n_limited_groups\": 4","note":"Applied in Gate.forward of inference/model.py; config /n_group 8, /topk_group 4."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.released":{"value":true,"source":"weight-index","at":"model.safetensors.index.json; /weight_map","quote":"\"model.layers.61.eh_proj.weight\"","note":"1 MTP layer (/num_nextn_predict_layers) with MLA+DSA and MoE; the V3.2 report does not describe MTP training."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"inference-model","at":"inference/model.py lines 831–851; Block.forward","quote":"x, residual = self.attn_norm(x), x"},"norm.type":{"value":"RMSNorm","source":"inference-model","at":"inference/model.py lines 828–829 (Block.__init__) and 884 (Transformer.__init__)","quote":"self.norm = RMSNorm(args.dim)","note":"Quote is the final norm; Block.__init__ builds attn_norm and ffn_norm with the same RMSNorm(args.dim)."},"ctx.max":{"value":131072,"source":"transparency","at":"p.3; Model properties","quote":"Context length 128K","note":"config /max_position_embeddings 163840 is 4096 × YaRN factor 40."},"ctx.final":{"value":131072,"source":"report","at":"p.4 §2.1.1","quote":"with each step consisting of 480 sequences of 128K tokens, resulting in a total of 943.7B tokens."},"train.stages":{"value":[{"name":"dense warm-up (indexer only)","tokens":2100000000,"seq_len":131072,"lr":0.001,"steps":1000,"batch_sequences":16,"note":"Dense attention; all parameters frozen except the indexer, trained with a KL loss to the main attention distribution.","quote":"We train the indexer for only 1000 steps, with each step consisting of 16 sequences of 128K tokens, resulting in a total of 2.1B tokens."},{"name":"sparse training","tokens":943700000000,"seq_len":131072,"lr":7.3e-06,"steps":15000,"batch_sequences":480,"note":"Top-2048 selection; all parameters trained; indexer input detached and trained only by the indexer KL loss.","quote":"We train both the main model and the indexer for 15000 steps, with each step consisting of 480 sequences of 128K tokens, resulting in a total of 943.7B tokens."}],"source":"report","at":"p.4 §2.1.1"},"train.data":{"value":"same distribution as the DeepSeek-V3.1-Terminus 128K long-context extension data","source":"report","at":"p.3 §2.1.1","quote":"the distribution of training data is totally aligned with the 128K long context extension data used for DeepSeek-V3.1-Terminus."},"train.objective":{"value":"lightning-indexer KL loss to the head-summed main attention distribution; indexer input detached from the LM loss","source":"report","at":"p.4 §2.1.1","quote":"we set a KL-divergence loss as the training objective of the indexer"},"eval.aa_index":{"value":21,"source":"aa","at":"Artificial Analysis model page /models/deepseek-v3-2-reasoning, Wayback snapshot 2026-10-19","quote":"DeepSeek V3.2 (Reasoning) scores 21 on the Artificial Analysis Intelligence Index","note":"Variant: DeepSeek V3.2 (Reasoning). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":671877944064,"formula":"embedding + output head + all experts + other weights, deepseek-ai/DeepSeek-V3.2@a7e62ac safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":1853358080,"formula":"input embedding + output head, deepseek-ai/DeepSeek-V3.2@a7e62ac safetensors headers"},"params.counted_non_embedding":{"value":670024585984,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, deepseek-ai/DeepSeek-V3.2@a7e62ac safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":11521245696,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, deepseek-ai/DeepSeek-V3.2@a7e62ac safetensors headers"},"params.counted_active_non_embedding":{"value":36550464256.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, deepseek-ai/DeepSeek-V3.2@a7e62ac safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":24576,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2857142857142857,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.05514157973174367,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":85888.0,"formula":"62464 B / 1 tokens (MLA latent) + 7808 B / 1 tokens (RoPE key) + 15616 B / 1 tokens (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":11257511936.0,"formula":"62464 B × 131072 (MLA latent) + 7808 B × 131072 (RoPE key) + 15616 B × 131072 (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":85888.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2321.2972972972975,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":304257079.3513514,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":85888.0,"windows":[],"groups":[{"name":"MLA latent","bytes_per_entry":62464.0,"tokens_per_entry":1},{"name":"RoPE key","bytes_per_entry":7808.0,"tokens_per_entry":1},{"name":"DSA indexer key","bytes_per_entry":15616.0,"tokens_per_entry":1}],"formula":"MLA latent: 61 × (512 × 16/8) B/entry; RoPE key: 61 × (64 × 16/8) B/entry; DSA indexer key: 61 × (128 × 16/8) B/entry","formula_version":"kv-2","sourced":{"per_token":48068.0,"windows":[],"groups":[{"name":"MLA latent","bytes_per_entry":32208.0,"tokens_per_entry":1},{"name":"RoPE key","bytes_per_entry":7808.0,"tokens_per_entry":1},{"name":"DSA indexer key","bytes_per_entry":8052.0,"tokens_per_entry":1}],"formula":"MLA latent: 61 × (ceil(512 × 8/8) + 4 × 32/8 scales) B/entry; RoPE key: 61 × (ceil(64 × 16/8)) B/entry; DSA indexer key: 61 × (ceil(128 × 8/8) + 1 × 32/8 scales) B/entry","formula_version":"kv-2"}}},{"id":"deepseek-v4-flash-base","name":"DeepSeek-V4-Flash-Base","org":"DeepSeek","country":"China","kind":"base","weights":"open","note":null,"sources":{"inference-kernel":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/resolve/60d8d70770c6776ff598c94bb586a859a38244f1/inference/kernel.py","kind":"code"},"flashmla-cache":{"url":"https://raw.githubusercontent.com/deepseek-ai/FlashMLA/07a1089857b63e74e3133630c02b083b75e8d4b2/README.md","kind":"code"},"config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Base/resolve/8855555deef230a27a21a8d6f294b7b7497759b6/config.json","kind":"config"},"report":{"url":"https://arxiv.org/pdf/2606.19348v1","kind":"paper"},"v4-flash-inference-model":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/resolve/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py","kind":"code"},"weight-index":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Base/resolve/8855555deef230a27a21a8d6f294b7b7497759b6/model.safetensors.index.json","kind":"config"},"license":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Base/resolve/8855555deef230a27a21a8d6f294b7b7497759b6/LICENSE","kind":"model-card"},"announcement":{"url":"https://web.archive.org/web/20261017101832/https://api-docs.deepseek.com/news/news260424/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095633/https://artificialanalysis.ai/models/deepseek-v4-flash-0420","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-24","source":"announcement","at":"DeepSeek API Docs news, DeepSeek-V4 Preview Release","quote":"DeepSeek-V4 Preview Release 2026/04/24","note":"Same post announces open weights; arXiv v1 is dated 26 Apr 2026."},"meta.license":{"value":"mit","source":"license","at":"LICENSE (repository has no model card)","quote":"MIT License"},"meta.modality":{"value":"text","basis":"inferred","source":"report","at":"p.45 §6 Conclusion, Limitations, and Future Directions","quote":"We are also working on incorporating multimodal capabilities to our models.","note":"The report names multimodality as future work and the config (DeepseekV4ForCausalLM) has no vision tower; vision came later as the separate V4-Flash-Vision-Exp."},"params.active":{"value":13000000000,"source":"report","at":"p.24 §4.2.1","quote":"DeepSeek-V4-Flash comprises 284B total parameters, of which 13B are activated for each token."},"params.total":{"value":284000000000,"source":"report","at":"p.24 §4.2.1","quote":"DeepSeek-V4-Flash comprises 284B total parameters, of which 13B are activated for each token."},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":43,"source":"config","at":"/num_hidden_layers"},"shape.tokenizer":{"value":"DeepSeek-V3 byte-level BPE plus special tokens for context construction; 128K vocab","source":"report","at":"p.24 §4.1","quote":"on top of the DeepSeek-V3 tokenizer, we introduce a few special tokens for context construction"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":129280,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.global_layers":{"value":41,"basis":"inferred","source":"report","at":"p.24 §4.2.1","quote":"For the first two layers, we use pure sliding window attention. For the subsequent layers, CSA and HCA are used in an interleaved manner.","note":"43 layers minus 2 pure-SWA layers; /compress_ratios has 21 × 4 and 20 × 128."},"attn.head_dim":{"value":512,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"Grouped low-rank output projection: /o_groups 8 groups, /o_lora_rank 1024 each."},"attn.local_heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"Pure-SWA layers (compress ratio 0) use the same Attention class with n_heads, head_dim and one shared KV head."},"attn.local_kv_heads":{"value":1,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":512,"source":"config","at":"/head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_theta","note":"Compress-ratio-0 layers use rope_theta without YaRN (Attention.__init__)."},"attn.kv_groups":{"value":[{"name":"CSA main KV","layers":21,"elements":512,"tokens_per_entry":4,"note":"Layers with /compress_ratios 4; one shared K=V entry of head_dim per 4 tokens.","quote":"kv_cache_size = args.window_size + (args.max_seq_len // self.compress_ratio if self.compress_ratio else 0)"},{"name":"CSA indexer K","layers":21,"elements":128,"tokens_per_entry":4,"note":"One index_head_dim key per compressed entry (Indexer.kv_cache).","quote":"self.register_buffer(\"kv_cache\", torch.zeros(args.max_batch_size, args.max_seq_len // compress_ratio, self.head_dim), persistent=False)"},{"name":"HCA main KV","layers":20,"elements":512,"tokens_per_entry":128,"note":"Layers with /compress_ratios 128; no indexer.","quote":"self.register_buffer(\"kv_cache\", torch.zeros(args.max_batch_size, kv_cache_size, self.head_dim), persistent=False)"},{"name":"SWA window","layers":43,"elements":512,"window":128,"note":"Ring buffer in every backbone layer (the 2 pure-SWA layers and the SWA branch of CSA/HCA); MTP block excluded.","quote":"self.kv_cache[:bsz, start_pos % win] = kv.squeeze(1)"}],"source":"v4-flash-inference-model","at":"V4-Flash inference/model.py; Attention.__init__, Attention.forward, Indexer.__init__"},"attn.kind":{"value":"SWA, CSA, HCA (CSA and HCA each with a 128-token SWA branch)","source":"report","at":"p.24 §4.2.1","quote":"For the first two layers, we use pure sliding window attention. For the subsequent layers, CSA and HCA are used in an interleaved manner."},"attn.kv_precision":{"value":[{"name":"CSA main KV NoPE","group":"CSA main KV","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"CSA main KV RoPE","group":"CSA main KV","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"HCA main KV NoPE","group":"HCA main KV","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"HCA main KV RoPE","group":"HCA main KV","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"SWA window NoPE","group":"SWA window","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"SWA window RoPE","group":"SWA window","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"CSA indexer key","group":"CSA indexer K","elements":128,"bits":4,"format":"MXFP4 E2M1; E8M0 scales per 32 channels","scale_count":4,"scale_bits":8,"source":"inference-kernel","at":"inference/kernel.py; fp4_act_quant, default block_size 32","quote":"s = z.new_empty(*z.size()[:-1], N // block_size, dtype=torch.float8_e8m0fnu)"}],"source":"flashmla-cache","at":"README.md; MLA Decoding / FP8 KV Cache; per-entry storage layouts","note":"Serving layout from the official FlashMLA kernels and post-training FP4 indexer QAT; main KV keeps the RoPE tail in BF16. Each main/SWA entry is 584 bytes, each indexer entry 68 bytes. This serving recipe is documented for the post-trained V4-Flash sibling; it does not establish FP4 QAT of the base checkpoint."},"attn.kv_heads":{"value":1,"source":"config","at":"/num_key_value_heads","note":"Shared-KV MQA: one KV head serves as both key and value (p.11 §2.3.1)."},"attn.layout":{"value":"2 SWA, then 1 CSA : 1 HCA (21 CSA + 20 HCA)","source":"config","at":"config.json; /compress_ratios","quote":"\"compress_ratios\": [0, 0, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0]","note":"Counts from /compress_ratios (0 = SWA, 4 = CSA, 128 = HCA; trailing 0 is the MTP block). First 43 /compress_ratios entries only; trailing MTP/DSpark entries excluded. Nonzero ratios retain an SWA branch in the same layer.","basis":"inferred","sequence":["SWA","SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA"]},"attn.local_layers":{"value":2,"source":"report","at":"p.24 §4.2.1","quote":"For the first two layers, we use pure sliding window attention."},"attn.position":{"value":"partial RoPE (last 64 dims of Q and KV entries; inverse rotation on attention output)","source":"report","at":"p.13 §2.3.3","quote":"we partially employ the Rotary Positional Embedding (RoPE)"},"attn.q_lora_rank":{"value":1024,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":true,"source":"report","at":"p.12 §2.3.3","quote":"we perform an additional RMSNorm operation on each head of the queries and the only head of the compressed KV entries, just before the core attention operation."},"attn.rope_fraction":{"value":0.125,"basis":"inferred","source":"report","at":"p.13 §2.3.3","quote":"we apply RoPE to its last 64 dimensions","note":"64 (/qk_rope_head_dim) of 512 head dims (/head_dim)."},"attn.rope_theta":{"value":160000,"source":"config","at":"/compress_rope_theta","note":"Compressed (CSA/HCA) layers use compress_rope_theta with YaRN (factor 16, original 65536); the 2 pure-SWA layers use rope_theta=10000 without YaRN (V4-Flash inference/model.py lines 475–479)."},"attn.sinks":{"value":"learnable per-head sink logit in the softmax denominator, every layer (CSA, HCA and the 2 pure-SWA layers)","source":"weight-index","at":"model.safetensors.index.json; /weight_map","quote":"\"layers.0.attn.attn_sink\"","note":"Layers 0–1 are the pure-SWA layers and carry attn_sink weights; sink formula p.13 §2.3.3 (described for CSA/HCA)."},"attn.sparse":{"value":"CSA: lightning indexer (64 heads × 128) selects top-512 of 4×-compressed KV entries; HCA (128×) attends densely","source":"report","at":"p.24 §4.2.1","quote":"the number of KV entries selected for sparse attention (i.e., attention top-k) to 512","note":"HCA does not employ sparse attention (p.11 §2.3.2); config /index_n_heads 64, /index_head_dim 128, /index_topk 512."},"attn.window":{"value":128,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"v4-flash-inference-model","at":"V4-Flash inference/model.py line 603; Expert.forward","quote":"x = F.silu(gate) * up","note":"config /hidden_act silu."},"ffn.bias":{"value":false,"source":"v4-flash-inference-model","at":"V4-Flash inference/model.py lines 589–594; Expert.__init__","quote":"self.w1 = Linear(dim, inter_dim, dtype=dtype)","note":"Linear defaults to bias=False; shared and routed experts both use Expert."},"ffn.clamp":{"value":"up (linear) clamped to [−10, 10], gate capped at 10; routed and shared experts in every layer","source":"report","at":"p.26 §4.2.3","quote":"we clamped the linear component of SwiGLU to the range of [−10, 10], while capping the upper bound of the gate component at 10.","note":"Applied throughout the training of both V4 models; inference code applies the same clamp (Expert.forward, /swiglu_limit 10)."},"moe.balancing":{"value":"aux-loss-free bias (update speed 0.001) + sequence-wise loss 1e-4; no node-limited routing","source":"report","at":"p.25 §4.2.2","quote":"For auxiliary-loss-free load balancing, we set the bias update speed to 0.001. For the balance loss, we set its loss weight to 0.0001 to avoid extreme imbalance within single sequences.","note":"Node limit removed: p.7 §2.1. Bias enters top-k selection only (/topk_method noaux_tc)."},"moe.dense_layers":{"value":0,"source":"report","at":"p.24 §4.2.1","quote":"We employ MoE layers in all Transformer blocks, but use the Hash routing strategy for the first 3 MoE layers."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":1.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sqrt(softplus); hash on token ID (first 3 MoE layers)","source":"report","at":"p.7 §2.1","quote":"we change the activation function that computes the affinity scores from Sigmoid(·) into Sqrt(Softplus(·)).","note":"Hash routing for the first 3 MoE layers: p.24 §4.2.1 (/num_hash_layers 3); /scoring_func sqrtsoftplus."},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":6,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"report","at":"p.24 §4.2.1","quote":"The multi-token prediction depth is set to 1."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.block":{"value":"e_proj(RMSNorm(Emb(t_{i+1}))) + h_proj(RMSNorm(h)), then one block with 128-token pure-SWA attention and MoE FFN inside mHC, own mHC head and final norm","source":"v4-flash-inference-model","at":"V4-Flash inference/model.py; MTPBlock.forward","quote":"x = self.e_proj(e).unsqueeze(2) + self.h_proj(x)","note":"Attention is SWA-only because the MTP entry of /compress_ratios is 0; the report states the MTP configuration is identical to DeepSeek-V3 (p.7 §2)."},"mtp.ffn":{"value":"MoE","source":"v4-flash-inference-model","at":"V4-Flash inference/model.py; MTPBlock, Block.__init__","quote":"class MTPBlock(Block):","note":"Block builds self.ffn = MoE(layer_id, args); weight index has mtp.0.ffn.experts.*."},"mtp.shares_embedding":{"value":true,"source":"v4-flash-inference-model","at":"V4-Flash inference/model.py; Transformer.__init__","quote":"self.mtp[-1].embed = self.embed","note":"Output head shared as well; the Base checkpoint also stores copies (mtp.0.emb.tok_emb, mtp.0.head)."},"mtp.loss_weight":{"value":"0.3, reduced to 0.1 at the start of LR decay","source":"report","at":"p.25 §4.2.2","quote":"The MTP loss weight is set to 0.3 for most of the training, and to 0.1 upon the start of learning rate decay."},"mtp.released":{"value":true,"source":"weight-index","at":"model.safetensors.index.json; weight_map mtp.0.*","quote":"\"mtp.0.hc_head_base\""},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"v4-flash-inference-model","at":"V4-Flash inference/model.py lines 688–700; Block.forward","quote":"x = self.attn_norm(x)","note":"Sublayers sit inside manifold-constrained hyper-connections (hc_pre/hc_post), expansion 4."},"norm.type":{"value":"RMSNorm","source":"v4-flash-inference-model","at":"V4-Flash inference/model.py lines 658–659; Block.__init__","quote":"self.attn_norm = RMSNorm(args.dim, self.norm_eps)"},"norm.residual":{"value":"mHC: 4 residual streams, residual mixing projected to doubly stochastic by Sinkhorn-Knopp (20 iterations), sigmoid-bounded input/output maps","source":"report","at":"p.24 §4.2.1","quote":"As for mHC, the expansion factor 𝑛hc is set to 4, and the number of Sinkhorn-Knopp iterations 𝑡max is set to 20.","note":"mHC design: p.7–8 §2.2. config /hc_mult 4, /hc_sinkhorn_iters 20, /hc_eps 1e-6."},"ctx.extension":{"value":"4K → 16K → 64K → 1M during pretraining; compressed layers use YaRN (factor 16, original 65536)","source":"report","at":"p.25 §4.2.2","quote":"The training starts with a sequence length of 4K, and we gradually extend the training sequence length to 16K, 64K, and 1M.","note":"config /rope_scaling: type yarn, factor 16, beta_fast 32, beta_slow 1, original_max_position_embeddings 65536."},"ctx.final":{"value":1048576,"source":"report","at":"p.25 §4.2.2","quote":"The training starts with a sequence length of 4K, and we gradually extend the training sequence length to 16K, 64K, and 1M."},"ctx.max":{"value":1048576,"source":"config","at":"/max_position_embeddings"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.25 §4.2.2","quote":"The training starts with a sequence length of 4K","note":"First stage; token split across the 4K/16K/64K/1M stages is not disclosed."},"train.batch":{"value":"ramped from a small size to 75.5M tokens, then constant for most of training","source":"report","at":"p.25 §4.2.2","quote":"increases the batch size (in tokens) from a small size to 75.5M and then keeps it at 75.5M during most of the training"},"train.batch_tokens":{"value":75500000,"source":"report","at":"p.25 §4.2.2","quote":"increases the batch size (in tokens) from a small size to 75.5M and then keeps it at 75.5M during most of the training","note":"After the batch-size ramp; ramp length not disclosed."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"p.25 §4.2.2","quote":"For AdamW, we set its hyper-parameters to 𝛽1 = 0.9, 𝛽2 = 0.95, 𝜀 = 10−20 , and weight_decay = 0.1."},"train.lr_min":{"value":2.7e-05,"source":"report","at":"p.25 §4.2.2","quote":"we finally decay the learning rate to 2.7 × 10−5 following a cosine schedule"},"train.lr_peak":{"value":0.00027,"source":"report","at":"p.25 §4.2.2","quote":"The learning rate is linearly warmed up in the first 2000 steps, maintained at 2.7 × 10−4 for most of the training."},"train.optimizer":{"value":"Muon + AdamW (AdamW for embedding, prediction head and RMSNorm weights)","source":"report","at":"p.25 §4.2.2","quote":"for the majority of parameters, but use the AdamW optimizer (Loshchilov and Hutter, 2017) for the embedding module, the prediction head module, and the weights of all RMSNorm modules.","note":"Muon: momentum 0.95, weight decay 0.1, update RMS rescaled to 0.18 (same paragraph)."},"train.schedule":{"value":"linear warmup (2000 steps), constant 2.7e-4 for most of training, cosine decay to 2.7e-5 near the end","source":"report","at":"p.25 §4.2.2","quote":"The learning rate is linearly warmed up in the first 2000 steps, maintained at 2.7 × 10−4 for most of the training."},"train.stages":{"value":[{"name":"context-4096","seq_len":4096,"note":"Dense attention.","quote":"The training starts with a sequence length of 4K"},{"name":"context-16384","seq_len":16384,"quote":"we gradually extend the training sequence length to 16K, 64K, and 1M."},{"name":"context-65536","seq_len":65536,"note":"Sparse attention introduced here after a short indexer warm-up and dense attention for the first 1T tokens; kept for the rest of training.","quote":"introduce sparse attention at the sequence length of 64K and keep sparse attention during the rest of the training."},{"name":"context-1048576","seq_len":1048576,"quote":"we gradually extend the training sequence length to 16K, 64K, and 1M."}],"source":"report","at":"p.25 §4.2.2"},"train.tokens":{"value":32000000000000,"source":"report","at":"p.25 §4.2.2","quote":"We train DeepSeek-V4-Flash on 32T tokens"},"train.warmup":{"value":"2000 steps","source":"report","at":"p.25 §4.2.2","quote":"The learning rate is linearly warmed up in the first 2000 steps"},"train.weight_decay":{"value":0.1,"source":"report","at":"p.25 §4.2.2","quote":"For AdamW, we set its hyper-parameters to 𝛽1 = 0.9, 𝛽2 = 0.95, 𝜀 = 10−20 , and weight_decay = 0.1."},"train.data":{"value":">32T tokens: math, code, web, long documents (scientific papers, technical reports), larger multilingual corpus; agentic data added in mid-training","source":"report","at":"p.24 §4.1","quote":"our pre-training corpus comprises more than 32T tokens, containing mathematical contents, codes, web pages, long documents, and other high-quality categories."},"train.masking":{"value":"sample-level attention masking within packed sequences","source":"report","at":"p.24 §4.1","quote":"Different from DeepSeek-V3, we employ sample-level attention masking during pre-training."},"train.objective":{"value":"FIM, strategy inherited from DeepSeek-V3","source":"report","at":"p.24 §4.1","quote":"We also inherit the token-splitting (DeepSeek-AI, 2024) and Fill-in-Middle (FIM) (DeepSeek-AI, 2024) strategies from DeepSeek-V3.","note":"The FIM rate is not restated for V4."},"train.stability":{"value":"Anticipatory Routing (routing indices computed with parameters from Δt steps earlier), auto-triggered with a short rollback on loss spikes; SwiGLU clamping","source":"report","at":"p.26 §4.2.3","quote":"we introduced an automatic detection mechanism that triggers a short rollback and activates Anticipatory Routing exclusively when a loss spike occurs","note":"SwiGLU clamp values under ffn.clamp."},"scaling.hp_rule":{"value":"AdamW hyper-parameters reused for Muon by rescaling the update RMS","source":"report","at":"p.14 §2.4","quote":"rescale the Root Mean Square (RMS) of the update matrix for reutilization of our AdamW hyper-parameters","note":"Update RMS 0.18 (p.25 §4.2.2)."},"eval.aa_index":{"value":24,"source":"aa","at":"Artificial Analysis model page /models/deepseek-v4-flash-0420, Wayback snapshot 2026-10-19","quote":"DeepSeek V4 Flash 0420 (Max) scores 24 on the Artificial Analysis Intelligence Index","note":"Score of DeepSeek-V4-Flash (0420 release), post-trained from this base. Variant: DeepSeek V4 Flash 0420 (Max). AA entry for the 0420 release (the 0731 update has its own entry). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":284334567252,"formula":"embedding + output head + all experts + other weights, deepseek-ai/DeepSeek-V4-Flash-Base@8855555 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":529530880,"formula":"input embedding + output head, deepseek-ai/DeepSeek-V4-Flash-Base@8855555 safetensors headers"},"params.counted_non_embedding":{"value":283805036372,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, deepseek-ai/DeepSeek-V4-Flash-Base@8855555 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":6596334320,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, deepseek-ai/DeepSeek-V4-Flash-Base@8855555 safetensors headers"},"params.counted_active_non_embedding":{"value":13272428372.0,"formula":"non-expert weights + routed experts × 6/256, excluding embedding and output head, deepseek-ai/DeepSeek-V4-Flash-Base@8855555 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":95.25581395348837,"formula":"d_model / layers"},"attn.qk_width":{"value":32768,"formula":"heads × head_dim"},"attn.group_size":{"value":64.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":0.04878048780487805,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":42.666666666666664,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.045774647887323945,"formula":"params.active / params.total"},"scaling.tpp":{"value":112.67605633802818,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":2411.013199928686,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":6880.0,"formula":"21504 B / 4 tokens (CSA main KV) + 5376 B / 4 tokens (CSA indexer K) + 20480 B / 128 tokens (HCA main KV); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":907411456.0,"formula":"21504 B × floor(131072 / 4) (CSA main KV) + 5376 B × floor(131072 / 4) (CSA indexer K) + 20480 B × floor(131072 / 128) (HCA main KV) + 44032 B × min(131072, 128) (SWA window); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":6923.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":532.5384615384615,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":69800881.23076923,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":6880.0,"windows":[[44032.0,128]],"groups":[{"name":"CSA main KV","bytes_per_entry":21504.0,"tokens_per_entry":4},{"name":"CSA indexer K","bytes_per_entry":5376.0,"tokens_per_entry":4},{"name":"HCA main KV","bytes_per_entry":20480.0,"tokens_per_entry":128},{"name":"SWA window","bytes_per_entry":44032.0,"tokens_per_entry":1,"window":128}],"formula":"CSA main KV: 21 × (512 × 16/8) B/entry; CSA indexer K: 21 × (128 × 16/8) B/entry; HCA main KV: 20 × (512 × 16/8) B/entry; SWA window: 43 × (512 × 16/8) B/entry","formula_version":"kv-2","sourced":{"per_token":3514.25,"windows":[[25112.0,128]],"groups":[{"name":"CSA main KV","bytes_per_entry":12264.0,"tokens_per_entry":4},{"name":"CSA indexer K","bytes_per_entry":1428.0,"tokens_per_entry":4},{"name":"HCA main KV","bytes_per_entry":11680.0,"tokens_per_entry":128},{"name":"SWA window","bytes_per_entry":25112.0,"tokens_per_entry":1,"window":128}],"formula":"CSA main KV: 21 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry; CSA indexer K: 21 × (ceil(128 × 4/8) + 4 × 8/8 scales) B/entry; HCA main KV: 20 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry; SWA window: 43 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry","formula_version":"kv-2"}}},{"id":"deepseek-v4-pro","name":"DeepSeek-V4-Pro","org":"DeepSeek","country":"China","kind":"reasoning","weights":"open","note":null,"sources":{"inference-kernel":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/inference/kernel.py","kind":"code"},"flashmla-cache":{"url":"https://raw.githubusercontent.com/deepseek-ai/FlashMLA/07a1089857b63e74e3133630c02b083b75e8d4b2/README.md","kind":"code"},"config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/config.json","kind":"config"},"inference-model":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/inference/model.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2606.19348v1","kind":"paper"},"weight-index":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/model.safetensors.index.json","kind":"config"},"card":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/README.md","kind":"model-card"},"announcement":{"url":"https://web.archive.org/web/20261017101832/https://api-docs.deepseek.com/news/news260424/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095717/https://artificialanalysis.ai/models/deepseek-v4-pro-0424","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-24","source":"announcement","at":"DeepSeek API Docs news, DeepSeek-V4 Preview Release","quote":"DeepSeek-V4 Preview Release 2026/04/24","note":"Same post announces open weights; arXiv v1 is dated 26 Apr 2026."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","basis":"inferred","source":"report","at":"p.45 §6 Conclusion, Limitations, and Future Directions","quote":"We are also working on incorporating multimodal capabilities to our models.","note":"The report names multimodality as future work and the config (DeepseekV4ForCausalLM) has no vision tower; vision came later as the separate V4-Flash-Vision-Exp."},"params.active":{"value":49000000000,"source":"report","at":"p.25 §4.2.1","quote":"DeepSeek-V4-Pro comprises 1.6T total parameters, of which 49B are activated for each token."},"params.total":{"value":1600000000000,"source":"report","at":"p.25 §4.2.1","quote":"DeepSeek-V4-Pro comprises 1.6T total parameters, of which 49B are activated for each token."},"shape.d_model":{"value":7168,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.tokenizer":{"value":"DeepSeek-V3 byte-level BPE plus special tokens for context construction; 128K vocab","source":"report","at":"p.24 §4.1","quote":"on top of the DeepSeek-V3 tokenizer, we introduce a few special tokens for context construction"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":129280,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.global_layers":{"value":61,"source":"report","at":"p.25 §4.2.1","quote":"For the first two layers, we use HCA. For the subsequent layers, CSA and HCA are used in an interleaved manner.","note":"All 61 layers are CSA or HCA; /compress_ratios has 30 × 4 and 31 × 128, plus a trailing 0 for the MTP block."},"attn.head_dim":{"value":512,"source":"config","at":"/head_dim"},"attn.heads":{"value":128,"source":"config","at":"/num_attention_heads","note":"Grouped low-rank output projection: /o_groups 16 groups, /o_lora_rank 1024 each."},"attn.kind":{"value":"CSA, HCA (each with a 128-token SWA branch)","source":"report","at":"p.25 §4.2.1","quote":"For the first two layers, we use HCA. For the subsequent layers, CSA and HCA are used in an interleaved manner.","note":"SWA branch inside CSA and HCA: p.13 §2.3.3."},"attn.local_layers":{"value":0,"source":"config","at":"/compress_ratios/0:61 | count 0","note":"No pure-SWA backbone layer; every CSA/HCA layer also attends to its own 128-token window (attn.kv_groups)."},"attn.kv_groups":{"value":[{"name":"CSA main KV","layers":30,"elements":512,"tokens_per_entry":4,"note":"Layers with /compress_ratios 4; one shared K=V entry of head_dim per 4 tokens.","quote":"kv_cache_size = args.window_size + (args.max_seq_len // self.compress_ratio if self.compress_ratio else 0)"},{"name":"CSA indexer K","layers":30,"elements":128,"tokens_per_entry":4,"note":"One index_head_dim key per compressed entry (Indexer.kv_cache).","quote":"self.register_buffer(\"kv_cache\", torch.zeros(args.max_batch_size, args.max_seq_len // compress_ratio, self.head_dim), persistent=False)"},{"name":"HCA main KV","layers":31,"elements":512,"tokens_per_entry":128,"note":"Layers with /compress_ratios 128; no indexer.","quote":"self.register_buffer(\"kv_cache\", torch.zeros(args.max_batch_size, kv_cache_size, self.head_dim), persistent=False)"},{"name":"SWA window","layers":61,"elements":512,"window":128,"note":"Uncompressed ring buffer in every backbone layer; MTP block (pure SWA) excluded.","quote":"self.kv_cache[:bsz, start_pos % win] = kv.squeeze(1)"}],"source":"inference-model","at":"inference/model.py; Attention.__init__, Attention.forward, Indexer.__init__"},"attn.kv_precision":{"value":[{"name":"CSA main KV NoPE","group":"CSA main KV","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"CSA main KV RoPE","group":"CSA main KV","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"HCA main KV NoPE","group":"HCA main KV","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"HCA main KV RoPE","group":"HCA main KV","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"SWA window NoPE","group":"SWA window","elements":448,"bits":8,"format":"FP8 E4M3; E8M0 scales per 64 channels","scale_count":7,"scale_bits":8,"padding_bytes":1,"at":"README.md; V4 layout: 448 NoPE bytes, 7 scale bytes and 1 padding byte","quote":"The scale row is 8 Bytes"},{"name":"SWA window RoPE","group":"SWA window","elements":64,"bits":16,"format":"BF16","at":"README.md; V4 layout: 128 RoPE bytes","quote":"the 64 `bfloat16` RoPE values (not quantized)"},{"name":"CSA indexer key","group":"CSA indexer K","elements":128,"bits":4,"format":"MXFP4 E2M1; E8M0 scales per 32 channels","scale_count":4,"scale_bits":8,"source":"inference-kernel","at":"inference/kernel.py; fp4_act_quant, default block_size 32","quote":"s = z.new_empty(*z.size()[:-1], N // block_size, dtype=torch.float8_e8m0fnu)"}],"source":"flashmla-cache","at":"README.md; MLA Decoding / FP8 KV Cache; per-entry storage layouts","note":"Serving layout from the official FlashMLA kernels and post-training FP4 indexer QAT; main KV keeps the RoPE tail in BF16. Each main/SWA entry is 584 bytes, each indexer entry 68 bytes."},"attn.kv_heads":{"value":1,"source":"config","at":"/num_key_value_heads","note":"Shared-KV MQA: one KV head serves as both key and value (p.11 §2.3.1)."},"attn.layout":{"value":"2 HCA, then 1 CSA : 1 HCA (30 CSA + 31 HCA)","source":"config","at":"config.json; /compress_ratios","quote":"\"compress_ratios\": [128, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0]","note":"Counts from /compress_ratios (4 = CSA, 128 = HCA). First 61 /compress_ratios entries only; trailing MTP/DSpark entries excluded. Nonzero ratios retain an SWA branch in the same layer.","basis":"inferred","sequence":["HCA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA","HCA + SWA","CSA + SWA"]},"attn.position":{"value":"partial RoPE (last 64 dims of Q and KV entries; inverse rotation on attention output)","source":"report","at":"p.13 §2.3.3","quote":"we partially employ the Rotary Positional Embedding (RoPE)"},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":true,"source":"report","at":"p.12 §2.3.3","quote":"we perform an additional RMSNorm operation on each head of the queries and the only head of the compressed KV entries, just before the core attention operation."},"attn.rope_fraction":{"value":0.125,"basis":"inferred","source":"report","at":"p.13 §2.3.3","quote":"we apply RoPE to its last 64 dimensions","note":"64 (/qk_rope_head_dim) of 512 head dims (/head_dim)."},"attn.rope_theta":{"value":160000,"source":"config","at":"/compress_rope_theta","note":"Compressed (CSA/HCA) layers use compress_rope_theta with YaRN (factor 16, original 65536); rope_theta=10000 without YaRN applies only to compress_ratio 0 layers, i.e. only the MTP block here (inference/model.py lines 475–479)."},"attn.sinks":{"value":"learnable per-head sink logit in the softmax denominator, all CSA and HCA layers","source":"report","at":"p.13 §2.3.3","quote":"we set a series of learnable sink logits"},"attn.sparse":{"value":"CSA: lightning indexer (64 heads × 128) selects top-1024 of 4×-compressed KV entries; HCA (128×) attends densely","source":"report","at":"p.25 §4.2.1","quote":"the number of KV entries selected for sparse attention (i.e., attention top-k) to 1024","note":"HCA does not employ sparse attention (p.11 §2.3.2); config /index_n_heads 64, /index_head_dim 128, /index_topk 1024."},"attn.window":{"value":128,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"inference-model","at":"inference/model.py line 603; Expert.forward","quote":"x = F.silu(gate) * up","note":"config /hidden_act silu."},"ffn.bias":{"value":false,"source":"inference-model","at":"inference/model.py lines 589–594; Expert.__init__","quote":"self.w1 = Linear(dim, inter_dim, dtype=dtype)","note":"Linear defaults to bias=False; shared and routed experts both use Expert."},"ffn.clamp":{"value":"up (linear) clamped to [−10, 10], gate capped at 10; routed and shared experts in every layer","source":"report","at":"p.26 §4.2.3","quote":"we clamped the linear component of SwiGLU to the range of [−10, 10], while capping the upper bound of the gate component at 10.","note":"Applied throughout the training of both V4 models; inference code applies the same clamp (Expert.forward, /swiglu_limit 10)."},"moe.balancing":{"value":"aux-loss-free bias (update speed 0.001) + sequence-wise loss 1e-4; no node-limited routing","source":"report","at":"p.26 §4.2.2","quote":"For auxiliary-loss-free load balancing, we set the bias update speed to 0.001. For the balance loss, we set its loss weight to 0.0001 to avoid extreme imbalance within single sequences.","note":"Node limit removed: p.7 §2.1. Bias enters top-k selection only (/topk_method noaux_tc)."},"moe.dense_layers":{"value":0,"source":"report","at":"p.25 §4.2.1","quote":"We employ MoE layers in all Transformer blocks, but use the Hash routing strategy for the first 3 MoE layers."},"moe.expert_dim":{"value":3072,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/n_routed_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sqrt(softplus); hash on token ID (first 3 MoE layers)","source":"report","at":"p.7 §2.1","quote":"we change the activation function that computes the affinity scores from Sigmoid(·) into Sqrt(Softplus(·)).","note":"Hash routing for the first 3 MoE layers: p.25 §4.2.1 (/num_hash_layers 3); /scoring_func sqrtsoftplus."},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":3072,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":6,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"report","at":"p.25 §4.2.1","quote":"The multi-token prediction depth is set to 1."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.block":{"value":"e_proj(RMSNorm(Emb(t_{i+1}))) + h_proj(RMSNorm(h)), then one block with 128-token pure-SWA attention and MoE FFN inside mHC, own mHC head and final norm","source":"inference-model","at":"inference/model.py; MTPBlock.forward","quote":"x = self.e_proj(e).unsqueeze(2) + self.h_proj(x)","note":"Attention is SWA-only because the MTP entry of /compress_ratios is 0; the report states the MTP configuration is identical to DeepSeek-V3 (p.7 §2)."},"mtp.ffn":{"value":"MoE","source":"inference-model","at":"inference/model.py; MTPBlock, Block.__init__","quote":"class MTPBlock(Block):","note":"Block builds self.ffn = MoE(layer_id, args); weight index has mtp.0.ffn.experts.*."},"mtp.shares_embedding":{"value":true,"source":"inference-model","at":"inference/model.py; Transformer.__init__","quote":"self.mtp[-1].embed = self.embed","note":"Output head shared as well (self.mtp[-1].head = self.head)."},"mtp.loss_weight":{"value":"0.3, reduced to 0.1 at the start of LR decay","source":"report","at":"p.26 §4.2.2","quote":"The MTP loss weight is set to 0.3 for most of the training, and to 0.1 upon the start of learning rate decay."},"mtp.released":{"value":true,"source":"weight-index","at":"model.safetensors.index.json; weight_map mtp.0.*","quote":"\"mtp.0.hc_head_base\""},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"inference-model","at":"inference/model.py lines 688–700; Block.forward","quote":"x = self.attn_norm(x)","note":"Sublayers sit inside manifold-constrained hyper-connections (hc_pre/hc_post), expansion 4."},"norm.type":{"value":"RMSNorm","source":"inference-model","at":"inference/model.py lines 658–659; Block.__init__","quote":"self.attn_norm = RMSNorm(args.dim, self.norm_eps)"},"norm.residual":{"value":"mHC: 4 residual streams, residual mixing projected to doubly stochastic by Sinkhorn-Knopp (20 iterations), sigmoid-bounded input/output maps","source":"report","at":"p.25 §4.2.1","quote":"As for mHC, the expansion factor 𝑛hc is set to 4, and the number of Sinkhorn-Knopp iterations 𝑡max is set to 20.","note":"mHC design: p.7–8 §2.2. config /hc_mult 4, /hc_sinkhorn_iters 20, /hc_eps 1e-6."},"ctx.extension":{"value":"4K → 16K → 64K → 1M during pretraining; compressed layers use YaRN (factor 16, original 65536)","source":"report","at":"p.25–26 §4.2.2","quote":"extended to 16K, 64K, and 1M.","note":"config /rope_scaling: type yarn, factor 16, beta_fast 32, beta_slow 1, original_max_position_embeddings 65536."},"ctx.final":{"value":1048576,"source":"report","at":"p.26 §4.2.2","quote":"extended to 16K, 64K, and 1M."},"ctx.max":{"value":1048576,"source":"config","at":"/max_position_embeddings"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.25 §4.2.2","quote":"The training also starts with a sequence length of 4K","note":"First stage; token split across the 4K/16K/64K/1M stages is not disclosed."},"train.batch":{"value":"ramped to a maximum of 94.4M tokens","source":"report","at":"p.25 §4.2.2","quote":"with the maximum batch size being 94.4M tokens"},"train.batch_tokens":{"value":94400000,"source":"report","at":"p.25 §4.2.2","quote":"with the maximum batch size being 94.4M tokens","note":"Maximum after the batch-size ramp; ramp length not disclosed."},"train.betas":{"value":"(0.9, 0.95)","basis":"inferred","source":"report","at":"p.25 §4.2.2","quote":"The hyper-parameters of AdamW and Muon are the same as those of DeepSeek-V4-Flash.","note":"Flash AdamW: β1 = 0.9, β2 = 0.95, ε = 1e-20 (same page)."},"train.lr_min":{"value":2e-05,"source":"report","at":"p.25 §4.2.2","quote":"the end learning rate is set to 2.0 × 10−5"},"train.lr_peak":{"value":0.0002,"source":"report","at":"p.25 §4.2.2","quote":"the peak learning rate is set to 2.0 × 10−4"},"train.optimizer":{"value":"Muon + AdamW (AdamW for embedding, prediction head and RMSNorm weights)","source":"report","at":"p.25 §4.2.2","quote":"We employ the Muon optimizer for the majority of parameters, but use the AdamW optimizer for the embedding module, the prediction head module, and the weights of all RMSNorm modules.","note":"Muon hyper-parameters as Flash: momentum 0.95, weight decay 0.1, update RMS rescaled to 0.18."},"train.schedule":{"value":"largely as Flash: linear warmup, constant 2.0e-4 for most of training, cosine decay to 2.0e-5 near the end","basis":"inferred","source":"report","at":"p.25 §4.2.2","quote":"The learning rate scheduling strategy is largely the same as that of DeepSeek-V4-Flash","note":"Shape from the Flash paragraph (same page), which the Pro paragraph says is largely shared; Pro's peak and end LR are stated in the Pro paragraph."},"train.stages":{"value":[{"name":"context-4096","seq_len":4096,"note":"Starts with dense attention, a longer dense stage than Flash's 1T tokens.","quote":"The training also starts with a sequence length of 4K"},{"name":"context-16384","seq_len":16384,"quote":"extended to 16K, 64K, and 1M."},{"name":"context-65536","seq_len":65536,"quote":"extended to 16K, 64K, and 1M."},{"name":"context-1048576","seq_len":1048576,"quote":"extended to 16K, 64K, and 1M."}],"source":"report","at":"p.25–26 §4.2.2"},"train.tokens":{"value":33000000000000,"source":"report","at":"p.25 §4.2.2","quote":"We train DeepSeek-V4-Pro on 33T tokens"},"train.weight_decay":{"value":0.1,"basis":"inferred","source":"report","at":"p.25 §4.2.2","quote":"The hyper-parameters of AdamW and Muon are the same as those of DeepSeek-V4-Flash.","note":"Flash: AdamW and Muon weight decay 0.1 (same page)."},"train.data":{"value":">32T tokens: math, code, web, long documents (scientific papers, technical reports), larger multilingual corpus; agentic data added in mid-training","source":"report","at":"p.24 §4.1","quote":"our pre-training corpus comprises more than 32T tokens, containing mathematical contents, codes, web pages, long documents, and other high-quality categories."},"train.masking":{"value":"sample-level attention masking within packed sequences","source":"report","at":"p.24 §4.1","quote":"Different from DeepSeek-V3, we employ sample-level attention masking during pre-training."},"train.objective":{"value":"FIM, strategy inherited from DeepSeek-V3","source":"report","at":"p.24 §4.1","quote":"We also inherit the token-splitting (DeepSeek-AI, 2024) and Fill-in-Middle (FIM) (DeepSeek-AI, 2024) strategies from DeepSeek-V3.","note":"The FIM rate is not restated for V4."},"train.stability":{"value":"Anticipatory Routing (routing indices computed with parameters from Δt steps earlier), auto-triggered with a short rollback on loss spikes; SwiGLU clamping","source":"report","at":"p.26 §4.2.3","quote":"we introduced an automatic detection mechanism that triggers a short rollback and activates Anticipatory Routing exclusively when a loss spike occurs","note":"SwiGLU clamp values under ffn.clamp."},"scaling.hp_rule":{"value":"AdamW hyper-parameters reused for Muon by rescaling the update RMS","source":"report","at":"p.14 §2.4","quote":"rescale the Root Mean Square (RMS) of the update matrix for reutilization of our AdamW hyper-parameters","note":"Update RMS 0.18 (p.25 §4.2.2)."},"eval.aa_index":{"value":30,"source":"aa","at":"Artificial Analysis model page /models/deepseek-v4-pro-0424, Wayback snapshot 2026-10-19","quote":"DeepSeek V4 Pro 0424 (Max) scores 30 on the Artificial Analysis Intelligence Index","note":"Variant: DeepSeek V4 Pro 0424 (Max). AA entry for the 0424 release, which matches this record's weights (the 0813 update has its own entry). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.warmup","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":1572999528436,"formula":"embedding + output head + all experts + other weights, deepseek-ai/DeepSeek-V4-Pro@b5968e9 safetensors headers; reproduces stated total (MTP) and active (non-embedding)"},"params.counted_embedding":{"value":926679040,"formula":"input embedding + output head, deepseek-ai/DeepSeek-V4-Pro@b5968e9 safetensors headers"},"params.counted_non_embedding":{"value":1572072849396,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, deepseek-ai/DeepSeek-V4-Pro@b5968e9 safetensors headers; reproduces stated total (MTP) and active (non-embedding)"},"params.counted_max_layer":{"value":25769422128,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, deepseek-ai/DeepSeek-V4-Pro@b5968e9 safetensors headers"},"params.counted_active_non_embedding":{"value":48854728692.0,"formula":"non-expert weights + routed experts × 6/384, excluding embedding and output head, deepseek-ai/DeepSeek-V4-Pro@b5968e9 safetensors headers; reproduces stated total (MTP) and active (non-embedding)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":65536,"formula":"heads × head_dim"},"attn.group_size":{"value":128.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.42857142857142855,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.030625,"formula":"params.active / params.total"},"scaling.tpp":{"value":20.625,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":675.4719734100944,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":9848.0,"formula":"30720 B / 4 tokens (CSA main KV) + 7680 B / 4 tokens (CSA indexer K) + 31744 B / 128 tokens (HCA main KV); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1298792448.0,"formula":"30720 B × floor(131072 / 4) (CSA main KV) + 7680 B × floor(131072 / 4) (CSA indexer K) + 31744 B × floor(131072 / 128) (HCA main KV) + 62464 B × min(131072, 128) (SWA window); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":9909.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":202.22448979591837,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":26505968.326530613,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":9848.0,"windows":[[62464.0,128]],"groups":[{"name":"CSA main KV","bytes_per_entry":30720.0,"tokens_per_entry":4},{"name":"CSA indexer K","bytes_per_entry":7680.0,"tokens_per_entry":4},{"name":"HCA main KV","bytes_per_entry":31744.0,"tokens_per_entry":128},{"name":"SWA window","bytes_per_entry":62464.0,"tokens_per_entry":1,"window":128}],"formula":"CSA main KV: 30 × (512 × 16/8) B/entry; CSA indexer K: 30 × (128 × 16/8) B/entry; HCA main KV: 31 × (512 × 16/8) B/entry; SWA window: 61 × (512 × 16/8) B/entry","formula_version":"kv-2","sourced":{"per_token":5031.4375,"windows":[[35624.0,128]],"groups":[{"name":"CSA main KV","bytes_per_entry":17520.0,"tokens_per_entry":4},{"name":"CSA indexer K","bytes_per_entry":2040.0,"tokens_per_entry":4},{"name":"HCA main KV","bytes_per_entry":18104.0,"tokens_per_entry":128},{"name":"SWA window","bytes_per_entry":35624.0,"tokens_per_entry":1,"window":128}],"formula":"CSA main KV: 30 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry; CSA indexer K: 30 × (ceil(128 × 4/8) + 4 × 8/8 scales) B/entry; HCA main KV: 31 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry; SWA window: 61 × (ceil(448 × 8/8) + 7 × 8/8 scales + 1 padding + ceil(64 × 16/8)) B/entry","formula_version":"kv-2"}}},{"id":"deepseek-v41","name":"DeepSeek-V4.1-Flash","org":"DeepSeek","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint. Recipe facts are from the technical report.","sources":{"flashmla-cache":{"url":"https://raw.githubusercontent.com/deepseek-ai/FlashMLA/07a1089857b63e74e3133630c02b083b75e8d4b2/README.md","kind":"code"},"config":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/resolve/314113f8a7dadb388215396ba5fbee2c610a04d6/config.json","kind":"config"},"inference-model":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/resolve/314113f8a7dadb388215396ba5fbee2c610a04d6/inference/model.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2609.19969v1","kind":"paper"},"card":{"url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/resolve/314113f8a7dadb388215396ba5fbee2c610a04d6/README.md","kind":"model-card"},"announcement":{"url":"https://web.archive.org/web/20261019003838/https://api-docs.deepseek.com/news/news260910/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019095908/https://artificialanalysis.ai/models/deepseek-v4-1-flash","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-09-10","source":"announcement","at":"DeepSeek API Docs news, DeepSeek-V4.1-Flash Release","quote":"DeepSeek-V4.1-Flash Release 2026/09/10","note":"Post links the Hugging Face weights and report; arXiv v1 is dated 17 Sep 2026."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Introduction","quote":"The model natively processes images and text, and generates text autoregressively."},"params.active_prefill":{"value":8000000000,"source":"report","at":"p.22 §4.2.1","quote":"DeepSeek-V4.1-Flash comprises 552B backbone parameters, with 8B activated per token during prefill and 16B during decode.","note":"Serving-system figure: prompt tokens run the 20-layer encoder plus decoder SWA replay of the last 128 tokens; the reference inference/model.py runs all 40 layers."},"params.active":{"value":16000000000,"source":"report","at":"p.22 §4.2.1","quote":"DeepSeek-V4.1-Flash comprises 552B backbone parameters, with 8B activated per token during prefill and 16B during decode.","note":"Per decoded token; prefill figure under params.active_prefill."},"params.note":{"value":"Backbone only; the 196B Engram (conditional-memory) parameters are reported separately.","source":"report","at":"p.7 §2.1","quote":"has 552B backbone parameters and 196B Engram parameters"},"params.total":{"value":552000000000,"source":"report","at":"p.22 §4.2.1","quote":"DeepSeek-V4.1-Flash comprises 552B backbone parameters, with 8B activated per token during prefill and 16B during decode."},"shape.d_model":{"value":5120,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":40,"source":"config","at":"/text_config/num_hidden_layers"},"shape.memory":{"value":"Engram, 196B params in 2 modules at layers 1 and 14 (0-indexed); N-gram orders {2, 3, 4}, 8 hash heads, 2048 dims per order, ~16M-entry prime-sized tables, FP8, no short conv","source":"report","at":"p.13 §2.4.2","quote":"We allocate 196B Engram parameters evenly across two modules.","note":"Orders, heads, table sizes, FP8 and layer placement in the same paragraph. config /text_config/engram_layer_ids [1, 14], engram_num_embeddings, engram_max_ngram_size 4, engram_vocab_size 16000000, engram_n_heads 8, engram_head_dim 256, engram_compressed_vocab_size 99092."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":129280,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":38,"basis":"inferred","source":"report","at":"p.7 §2.1","quote":"Each layer incorporates both global attention and sliding window attention (SWA), except for the first two layers, which use SWA only.","note":"40 layers minus 2 SWA-only layers. Global KV is shared across layers: only layers 2, 8, 14 and 20 compute it (/text_config/kv_source_layer_ids)."},"attn.head_dim":{"value":512,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads","note":"Grouped low-rank output projection: /text_config/o_groups 8 groups, /text_config/o_lora_rank 1024 each."},"attn.local_heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads","note":"SWA-only layers (compress ratio 0) use the same Attention class with n_heads, head_dim and one shared KV head."},"attn.local_kv_heads":{"value":1,"source":"config","at":"/text_config/num_key_value_heads"},"attn.local_head_dim":{"value":512,"source":"config","at":"/text_config/head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/text_config/rope_theta","note":"Compress-ratio-0 layers use rope_theta without YaRN (Attention.__init__)."},"attn.qk_norm":{"value":false,"source":"inference-model","at":"inference/model.py; Attention.forward","quote":"q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.head_dim))","note":"Only the single shared KV head is RMSNormed (kv_norm, compressor norm); queries get no per-head norm, unlike V4."},"attn.kv_sharing":{"value":"Main KV and indexer K owned by layers 2, 8, 14 (encoder, m=2, each serving its 6-layer group) and 20 (decoder, m=1, serving layers 20–39, computed from the last encoder layer's output); other CSA2 layers read them. Each layer keeps its own 128-token SWA KV.","source":"report","at":"p.22 §4.2.1; p.10–11 §2.3.1","quote":"In each group, the first layer operates in Full Mode, and the remaining five layers operate in Reuse Mode.","note":"Owners in /text_config/kv_source_layer_ids; /text_config/index_source_layer_ids adds Reindex layers 24, 28, 32, 36 that rescore the shared cache with their own indexer Q."},"attn.kv_groups":{"value":[{"name":"encoder main KV (m=2)","layers":3,"elements":512,"tokens_per_entry":2,"note":"Layers 2, 8, 14; one shared K=V entry per 2 tokens.","quote":"compress_ratio > 0 does not mean the layer compresses its own KV: only kv_source_layers do, the rest read that same cache."},{"name":"encoder indexer K (m=2)","layers":3,"elements":128,"tokens_per_entry":2,"note":"Indexer.k_cache of the same owner layers.","quote":"torch.zeros(args.max_batch_size, args.max_seq_len // self.compress_ratio, args.index_head_dim)"},{"name":"decoder main KV (m=1)","layers":1,"elements":512,"tokens_per_entry":1,"note":"Layer 20; ratio-1 compressor is a plain projection.","quote":"if ratio == 1:  # one token per group: nothing to pool, so no gate and no fp32"},{"name":"decoder indexer K (m=1)","layers":1,"elements":128,"tokens_per_entry":1,"note":"Layer 20.","quote":"self.owns_k = layer_id in args.kv_source_layers"},{"name":"SWA window","layers":40,"elements":512,"window":128,"note":"window_kv_cache in every backbone layer; DSpark layers excluded.","quote":"torch.zeros(args.max_batch_size, args.window_size, self.head_dim)"}],"source":"inference-model","at":"inference/model.py; Attention.__init__, Compressor.forward, Indexer.__init__"},"attn.kind":{"value":"SWA, CSA2","source":"report","at":"p.7 Figure 3","quote":"The first two encoder layers use sliding window attention (SWA); the rest use Compressed Sparse Attention 2 (CSA2)"},"attn.kv_precision":{"value":[{"name":"encoder main KV (m=2)","group":"encoder main KV (m=2)","elements":512,"bits":4,"format":"FP4 E2M1; E4M3 scales per 16 channels","scale_count":32,"scale_bits":8,"at":"README.md; V4.1 fp4 layout: 256 data bytes + 32 scale bytes","quote":"The scale row is 32 Bytes of `float8_e4m3`"},{"name":"decoder main KV (m=1)","group":"decoder main KV (m=1)","elements":512,"bits":4,"format":"FP4 E2M1; E4M3 scales per 16 channels","scale_count":32,"scale_bits":8,"at":"README.md; V4.1 fp4 layout: 256 data bytes + 32 scale bytes","quote":"The scale row is 32 Bytes of `float8_e4m3`"},{"name":"encoder indexer K (m=2)","group":"encoder indexer K (m=2)","elements":128,"bits":4,"format":"MXFP4 E2M1; E8M0 scales per 32 channels","scale_count":4,"scale_bits":8,"source":"inference-model","at":"inference/model.py lines 28–30, 546; fp4_act_quant(k, fp4_block_size, True)","quote":"fp4_block_size = 32"},{"name":"decoder indexer K (m=1)","group":"decoder indexer K (m=1)","elements":128,"bits":4,"format":"MXFP4 E2M1; E8M0 scales per 32 channels","scale_count":4,"scale_bits":8,"source":"inference-model","at":"inference/model.py lines 28–30, 546; fp4_act_quant(k, fp4_block_size, True)","quote":"fp4_block_size = 32"},{"name":"SWA window","group":"SWA window","elements":512,"bits":8,"format":"FP8 E4M3; E8M0 scales per 32 channels","scale_count":16,"scale_bits":8,"at":"README.md; V4.1 FP8 layout: 512 data bytes + 16 scale bytes","quote":"The scale row is 16 Bytes of `float8_e8m0`"}],"source":"flashmla-cache","at":"README.md; MLA Decoding / FP8 KV Cache; per-entry storage layouts","note":"Post-training/serving recipe, not pretraining precision. Main KV: 288 bytes/entry; indexer K: 68; SWA: 528. Global growth = (3/2 + 1) × (288 + 68) = 890 bytes/token, matching report p.1 Figure 1(b). The finite SWA rings are additional runtime cache, not part of that global figure. FP4 main-cache QAT is introduced in post-training (§2.4.4)."},"attn.kv_heads":{"value":1,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"2 SWA + 18 CSA2 m=2 (encoder: 3 × [1 Full, 5 Reuse]) + 20 CSA2 m=1 (decoder: [1 Full, 3 Reuse], then 4 × [1 Reindex, 3 Reuse])","source":"config","at":"config.json; /text_config/compress_ratios, /text_config/kv_source_layer_ids, /text_config/index_source_layer_ids","quote":"\"compress_ratios\": [ 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 ]","note":"First 40 compress_ratios entries describe the backbone; the last 3 are DSpark layers. Indices 0–19 are encoder, 20–39 decoder (report §4.2.1). Nonzero ratios retain a 128-token SWA branch in the same layer. Full modes own KV at kv_source_layer_ids [2, 8, 14, 20]; Reindex layers appear in index_source_layer_ids but do not own KV ([24, 28, 32, 36]); the remaining CSA2 layers reuse both. See inference/model.py Attention.__init__.","basis":"inferred","sequence":["SWA","SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=2 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA","CSA2 m=1 + SWA"]},"attn.local_layers":{"value":2,"source":"report","at":"p.22 §4.2.1","quote":"For the first two layers, we use pure sliding window attention.","note":"Counts SWA-only layers; the 38 CSA2 layers also have a 128-token SWA branch."},"attn.position":{"value":"partial RoPE (last 64 dims of Q and KV; inverse rotation on attention output)","source":"inference-model","at":"inference/model.py lines 632–633 (Attention.__init__) and 766–782 (Attention.forward)","quote":"apply_rotary_emb(o[..., -rd:], freqs_cis, True)","note":"forward rotates q[..., -rd:] and the KV tail (rope_head_dim 64), and applies the inverse rotation to o[..., -rd:]."},"attn.q_lora_rank":{"value":1280,"source":"config","at":"/text_config/q_lora_rank"},"attn.rope_fraction":{"value":0.125,"basis":"inferred","source":"inference-model","at":"inference/model.py lines 632–633; Attention.__init__","quote":"self.nope_head_dim = args.head_dim - args.rope_head_dim","note":"/text_config/qk_rope_head_dim 64 of /text_config/head_dim 512."},"attn.rope_theta":{"value":160000,"source":"config","at":"/text_config/compress_rope_theta","note":"CSA2 layers use compress_rope_theta with YaRN (factor 16, original 65536); the 2 SWA-only layers use rope_theta=10000 without YaRN (inference/model.py lines 681–688)."},"attn.sinks":{"value":"learnable per-head sink logit in the softmax denominator, every layer","source":"inference-model","at":"inference/model.py line 640; Attention.__init__","quote":"self.attn_sink = nn.Parameter(torch.empty(self.n_local_heads, dtype=torch.float32))"},"attn.sparse":{"value":"CSA2 indexer (32 heads × 128), top-512 KV entries; decoder hierarchical indexer limited to a 2048-block × 8-position candidate pool","source":"report","at":"p.22 §4.2.1","quote":"For all CSA2 layers, we set the number of indexer query heads to 32, the indexer head dimension to 128, and the number of KV entries selected for sparse attention (i.e., attention top-k) to 512.","note":"The hierarchical candidate pool is introduced in post-training (p.12 §2.3.2); config /text_config/candidate_source_layer_id 20, candidate_topk_blocks 2048, candidate_block_size 8; index_n_heads 32, index_head_dim 128, index_topk 512."},"attn.window":{"value":128,"source":"config","at":"/text_config/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"report","at":"p.22 §4.2.1","quote":"using SwiGLU activation function with clamping (OpenAI, 2025) at a threshold of 10","note":"config /text_config/hidden_act silu."},"ffn.bias":{"value":false,"source":"inference-model","at":"inference/model.py lines 836–841; Expert.__init__","quote":"self.w1 = Linear(dim, inter_dim, dtype=dtype)","note":"Linear defaults to bias=False; shared and routed experts both use Expert."},"ffn.clamp":{"value":"up (linear) clamped to [−10, 10], gate capped at 10; routed and shared experts in every layer","source":"inference-model","at":"inference/model.py lines 846–849; Expert.forward","quote":"up = torch.clamp(up, min=-self.swiglu_limit, max=self.swiglu_limit)","note":"Gate is clamped with max only; limit 10 from /text_config/swiglu_limit and p.22 §4.2.1."},"moe.balancing":{"value":"aux-loss-free bias with separate text and image biases (update speed 0.001) + sequence-level loss 1e-4","source":"report","at":"p.22 §4.2.2","quote":"we set the bias update speed to 0.001 for both image and text tokens, while retaining a small sequence-level balance loss with a loss weight of 0.0001","note":"Modality-specific biases: p.8 §2.1.1. Bias enters top-k selection only (/text_config/topk_method noaux_tc)."},"moe.dense_layers":{"value":0,"source":"report","at":"p.22 §4.2.1","quote":"We employ MoE layers in all Transformer blocks"},"moe.expert_dim":{"value":2304,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/text_config/n_routed_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_topk_prob"},"moe.routed_scale":{"value":1.5,"source":"config","at":"/text_config/routed_scaling_factor"},"moe.router":{"value":"sqrt(softplus)","source":"inference-model","at":"inference/model.py line 819; Gate.forward","quote":"scores = F.softplus(scores).sqrt()","note":"/text_config/scoring_func is sqrtsoftplus."},"moe.shared":{"value":1,"source":"config","at":"/text_config/n_shared_experts"},"moe.shared_dim":{"value":2304,"source":"config","at":"/text_config/moe_intermediate_size","note":"1 shared expert (/text_config/n_shared_experts) of the routed-expert width."},"moe.top_k":{"value":6,"source":"config","at":"/text_config/num_experts_per_tok"},"mtp.depth":{"value":0,"source":"report","at":"p.8 §2.1","quote":"We omit the MTP module during backbone pre-training and use DSpark (Cheng et al., 2026a) for speculative decoding.","note":"The DSpark drafter is trained after pre-training with the backbone frozen."},"norm.eps":{"value":1e-20,"source":"config","at":"/text_config/rms_norm_eps","note":"Used by attn_norm, ffn_norm, q/kv norms and the final norm in inference/model.py (norm_eps); the mHC Sinkhorn eps is separate (hc_eps 1e-6)."},"norm.placement":{"value":"pre","source":"inference-model","at":"inference/model.py lines 983–995; Block.forward","quote":"x = self.attn_norm(x)","note":"Sublayers sit inside Single-Pass mHC (hc_pre/hc_post), expansion 4."},"norm.type":{"value":"RMSNorm","source":"inference-model","at":"inference/model.py lines 935–936; Block.__init__","quote":"self.attn_norm = RMSNorm(args.dim, self.norm_eps)"},"norm.residual":{"value":"Single-Pass mHC: 4 residual streams, Sinkhorn-Knopp (20 iterations); each sublayer's input mixing uses the coefficients predicted at the previous sublayer","source":"report","at":"p.22 §4.2.1","quote":"As for mHC, the expansion factor is set to 4, and the number of Sinkhorn-Knopp iterations is set to 20.","note":"Single-Pass shift: p.13 §2.4.1 (pre-training keeps the multi-kernel implementation); Block.forward in inference/model.py. config /text_config/hc_mult 4, hc_sinkhorn_iters 20."},"ctx.extension":{"value":"64K → 1M at 34T tokens; CSA2 layers use YaRN (factor 16, original 65536)","source":"report","at":"p.22 §4.2.2","quote":"We train the model from scratch with sparse attention at a sequence length of 64K and extend the sequence length to 1M at 34T tokens.","note":"config /text_config/rope_scaling: rope_type yarn, factor 16, beta_fast 32, beta_slow 1, original_max_position_embeddings 65536."},"ctx.final":{"value":1048576,"source":"report","at":"p.22 §4.2.2","quote":"extend the sequence length to 1M at 34T tokens."},"ctx.max":{"value":1048576,"source":"config","at":"/text_config/max_position_embeddings"},"ctx.pretrain":{"value":65536,"source":"report","at":"p.22 §4.2.2","quote":"We train the model from scratch with sparse attention at a sequence length of 64K"},"train.batch":{"value":"100.6M tokens, fixed","source":"report","at":"p.22 §4.2.2","quote":"We keep the batch size fixed at 100.6 million tokens throughout training."},"train.batch_tokens":{"value":100600000,"source":"report","at":"p.22 §4.2.2","quote":"We keep the batch size fixed at 100.6 million tokens throughout training."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"p.22 §4.2.2","quote":"For AdamW, we set its hyper-parameters to 𝛽1 = 0.9, 𝛽2 = 0.95, 𝜀 = 10 −20 , and weight_decay = 0.1."},"train.lr_min":{"value":2.6e-05,"source":"report","at":"p.22 §4.2.2","quote":"Between 28T and 40T tokens, we decay the learning rate to 2.6 × 10−5 following a cosine schedule."},"train.lr_peak":{"value":0.00026,"source":"report","at":"p.22 §4.2.2","quote":"The learning rate is linearly warmed up over the first 2000 steps and then maintained at 2.6 × 10−4 until 28T tokens."},"train.optimizer":{"value":"Muon + AdamW (Muon for linear weights, head-wise for Q/K; AdamW for RMSNorm and other non-matrix params; Sinkhorn-balanced momentum update for embeddings, Engram tables and prediction head)","source":"report","at":"p.22 §4.2.2","quote":"We employ the Muon optimizer (Jordan et al., 2024; Liu et al., 2025) for the parameters of linear transformations, use AdamW optimizer (Loshchilov and Hutter, 2019) for the weights of all RMSNorm modules and other non-matrix parameters, and use Sinkhorn-balanced update for all embeddings and prediction head.","note":"Muon: Nesterov momentum 0.95, weight decay 0.1, update RMS 0.18. Sinkhorn: K = 11, τ = 1e-3, ε = 1e-20, no weight decay. Engram LR scaled 5×. Head-wise Q/K Muon: p.15 §2.5."},"train.schedule":{"value":"2000-step linear warmup, constant 2.6e-4 to 28T, cosine to 2.6e-5 at 40T, constant to 45T","source":"report","at":"p.22 §4.2.2","quote":"Between 28T and 40T tokens, we decay the learning rate to 2.6 × 10−5 following a cosine schedule. We keep the learning rate at this value from 40T to 45T tokens."},"train.stages":{"value":[{"name":"main","tokens":34000000000000,"seq_len":65536,"note":"Sparse attention from scratch.","quote":"We train the model from scratch with sparse attention at a sequence length of 64K"},{"name":"long","tokens":11000000000000,"seq_len":1048576,"note":"34T to 45T.","quote":"extend the sequence length to 1M at 34T tokens."}],"source":"report","at":"p.22 §4.2.2"},"train.tokens":{"value":45000000000000,"source":"report","at":"p.22 §4.2.2","quote":"We train DeepSeek-V4.1-Flash on 45T tokens of multimodal data with no instability."},"train.warmup":{"value":"2000 steps","source":"report","at":"p.22 §4.2.2","quote":"The learning rate is linearly warmed up over the first 2000 steps"},"train.weight_decay":{"value":0.1,"source":"report","at":"p.22 §4.2.2","quote":"For AdamW, we set its hyper-parameters to 𝛽1 = 0.9, 𝛽2 = 0.95, 𝜀 = 10 −20 , and weight_decay = 0.1."},"train.data":{"value":"text-only : multimodal = 7:1 tokens; multimodal = image-text pairs, interleaved image-text, domain-specific (grounding, OCR, image-code, computer-use)","source":"report","at":"p.21 §4.1","quote":"the resulting corpus uses a 7:1 token ratio of text-only to multimodal data."},"train.masking":{"value":"sample-level attention masking within packed sequences","source":"report","at":"p.22 §4.2.2","quote":"Similar to DeepSeek-V4, we employ sample-level attention masking during pre-training."},"train.objective":{"value":"multimodal (image + text) input from the start of pretraining","source":"report","at":"p.7 §2.1","quote":"with multimodal data incorporated from the start of language-model pre-training","note":"Vision encoder frozen until LR decay, then trained jointly at a smaller LR (p.15 §2.5)."},"scaling.hp_rule":{"value":"AdamW LR reused for Muon by rescaling update RMS to 0.18","source":"report","at":"p.22 §4.2.2","quote":"rescale the RMS of each update matrix to 0.18 for reutilization of the AdamW learning rate"},"scaling.ladder":{"value":"ladder over model parameters and training data (sizes not disclosed)","source":"report","at":"p.20 §4.1","quote":"a scaling ladder over model parameters and training data is carefully designed to guide large-scale training runs"},"eval.aa_index":{"value":39,"source":"aa","at":"Artificial Analysis model page /models/deepseek-v4-1-flash, Wayback snapshot 2026-10-19","quote":"DeepSeek V4.1 Flash (Max) scores 39 on the Artificial Analysis Intelligence Index","note":"Variant: DeepSeek V4.1 Flash (Max). AA entry for the 0910 release, which matches this record's weights (the 1017 update has its own entry). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":551639601024,"formula":"embedding + output head + all experts + other weights, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"params.counted_embedding":{"value":661913600,"formula":"input embedding + output head, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers"},"params.counted_non_embedding":{"value":550977687424,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"params.counted_max_layer":{"value":13765227760,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers"},"params.counted_active_non_embedding":{"value":15889354624.0,"formula":"non-expert weights + routed experts × 6/384, excluding embedding and output head, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"params.counted_lookup":{"value":196928504320,"formula":"n-gram / per-layer embedding tables, deepseek-ai/DeepSeek-V4.1-Flash@314113f safetensors headers"},"shape.aspect":{"value":128.0,"formula":"d_model / layers"},"attn.qk_width":{"value":32768,"formula":"heads × head_dim"},"attn.group_size":{"value":64.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.45,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":0.05263157894736842,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.028985507246376812,"formula":"params.active / params.total"},"scaling.tpp":{"value":81.52173913043478,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":2832.084818097644,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":3200.0,"formula":"3072 B / 2 tokens (encoder main KV (m=2)) + 768 B / 2 tokens (encoder indexer K (m=2)) + 1024 B / 1 tokens (decoder main KV (m=1)) + 256 B / 1 tokens (decoder indexer K (m=1)); uniform BF16, payload only; quantization overhead excluded. encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":424673280.0,"formula":"3072 B × floor(131072 / 2) (encoder main KV (m=2)) + 768 B × floor(131072 / 2) (encoder indexer K (m=2)) + 1024 B × 131072 (decoder main KV (m=1)) + 256 B × 131072 (decoder indexer K (m=1)) + 40960 B × min(131072, 128) (SWA window); uniform BF16, payload only; quantization overhead excluded. encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":3240.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":202.5,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":26542080.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":3200.0,"windows":[[40960.0,128]],"groups":[{"name":"encoder main KV (m=2)","bytes_per_entry":3072.0,"tokens_per_entry":2},{"name":"encoder indexer K (m=2)","bytes_per_entry":768.0,"tokens_per_entry":2},{"name":"decoder main KV (m=1)","bytes_per_entry":1024.0,"tokens_per_entry":1},{"name":"decoder indexer K (m=1)","bytes_per_entry":256.0,"tokens_per_entry":1},{"name":"SWA window","bytes_per_entry":40960.0,"tokens_per_entry":1,"window":128}],"formula":"encoder main KV (m=2): 3 × (512 × 16/8) B/entry; encoder indexer K (m=2): 3 × (128 × 16/8) B/entry; decoder main KV (m=1): 1 × (512 × 16/8) B/entry; decoder indexer K (m=1): 1 × (128 × 16/8) B/entry; SWA window: 40 × (512 × 16/8) B/entry","formula_version":"kv-2","sourced":{"per_token":890.0,"windows":[[21120.0,128]],"groups":[{"name":"encoder main KV (m=2)","bytes_per_entry":864.0,"tokens_per_entry":2},{"name":"encoder indexer K (m=2)","bytes_per_entry":204.0,"tokens_per_entry":2},{"name":"decoder main KV (m=1)","bytes_per_entry":288.0,"tokens_per_entry":1},{"name":"decoder indexer K (m=1)","bytes_per_entry":68.0,"tokens_per_entry":1},{"name":"SWA window","bytes_per_entry":21120.0,"tokens_per_entry":1,"window":128}],"formula":"encoder main KV (m=2): 3 × (ceil(512 × 4/8) + 32 × 8/8 scales) B/entry; encoder indexer K (m=2): 3 × (ceil(128 × 4/8) + 4 × 8/8 scales) B/entry; decoder main KV (m=1): 1 × (ceil(512 × 4/8) + 32 × 8/8 scales) B/entry; decoder indexer K (m=1): 1 × (ceil(128 × 4/8) + 4 × 8/8 scales) B/entry; SWA window: 40 × (ceil(512 × 8/8) + 16 × 8/8 scales) B/entry","formula_version":"kv-2"}}},{"id":"gemma-4-26b-a4b","name":"Gemma-4-26B-A4B","org":"Google DeepMind","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/google/gemma-4-26B-A4B/resolve/24548b62aa021d562695c04aaf7758a1ea47990b/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/google/gemma-4-26B-A4B/resolve/24548b62aa021d562695c04aaf7758a1ea47990b/config.json","kind":"config"},"modeling_gemma4":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/gemma4/modeling_gemma4.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2607.02770v2","kind":"paper"},"configuration_gemma4":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/gemma4/configuration_gemma4.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20261012212433/https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019100045/https://artificialanalysis.ai/models/gemma-4-26b-a4b","kind":"leaderboard"}},"facts":{"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.released":{"value":"2026-04-02","source":"blog","at":"Gemma 4 launch post header","quote":"Gemma 4: Byte for byte, the most capable open models Apr 02, 2026"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Mixture-of-Experts (MoE) Model table","quote":"| **Expert Count** | 8 active / 128 total and 1 shared | | **Supported Modalities** | Text, Image |","note":"The card also lists video understanding as processing sequences of frames; audio is limited to E2B, E4B and 12B."},"params.active":{"value":3800000000,"source":"card","at":"README.md; Mixture-of-Experts (MoE) Model table","quote":"Active Parameters | 3.8B","note":"Report text also says 3.8B; report Table 1 lists 2,800M active einsums plus a 740M embedder."},"params.active_non_embedding":{"value":2800000000,"source":"report","at":"PDF page 2; Table 1","quote":"26B-A4B* - 550M 740M 24,500M / 2,800M (active) 430M","note":"Active Einsums column; the tied 740M embedder is counted separately."},"params.embedding":{"value":740000000,"source":"report","at":"PDF page 2; Table 1","quote":"26B-A4B* - 550M 740M 24,500M / 2,800M (active) 430M"},"params.non_embedding":{"value":24500000000,"source":"report","at":"PDF page 2; Table 1","quote":"26B-A4B* - 550M 740M 24,500M / 2,800M (active) 430M"},"params.note":{"value":"Language model only (740M embedder + 24,500M einsums, 2,800M active einsums). Excludes the 550M frozen vision encoder and the 430M MTP drafter.","source":"report","at":"PDF page 2; Table 1","quote":"26B-A4B* - 550M 740M 24,500M / 2,800M (active) 430M"},"params.total":{"value":25200000000,"source":"card","at":"README.md; Mixture-of-Experts (MoE) Model table","quote":"Total Parameters | 25.2B"},"shape.d_model":{"value":2816,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":30,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":true,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.tokenizer":{"value":"SentencePiece, same as Gemini: split digits, preserved whitespace, byte-level encodings","source":"report","at":"PDF page 3; §2.4 Tokenizer","quote":"digits, preserved whitespace, and byte-level en"},"shape.vocab":{"value":262144,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":5,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":512,"source":"config","at":"/text_config/global_head_dim","note":"Global layers only; sliding-window layers use head_dim 256."},"attn.heads":{"value":16,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; Models Overview","quote":"interleaves local sliding window attention with full global attention, ensuring the final layer is always global.","note":"Global layers reuse keys as values (attention_k_eq_v: no v_proj; V is the K projection before RoPE and k_norm). use_bidirectional_attention vision: image tokens attend bidirectionally, text is causal."},"attn.kv_heads":{"value":2,"source":"config","at":"/text_config/num_global_key_value_heads"},"attn.layout":{"value":"5 SWA : 1 global","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Ordered 30-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.local_head_dim":{"value":256,"source":"config","at":"/text_config/head_dim","note":"configuration_gemma4.py overrides head_dim (global_head_dim) and num_key_value_heads (num_global_key_value_heads) only for full_attention layers; sliding layers keep head_dim, num_key_value_heads and num_attention_heads."},"attn.local_heads":{"value":16,"source":"config","at":"/text_config/num_attention_heads","note":"Same num_attention_heads attribute for both layer types (Gemma4TextAttention)."},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/text_config/rope_parameters/sliding_attention/rope_theta","note":"rope_type default."},"attn.local_kv_heads":{"value":8,"source":"configuration_gemma4","at":"configuration_gemma4.py; Gemma4TextConfig.__post_init__","quote":"num_key_value_heads = num_global_key_value_heads if getattr(self, \"attention_k_eq_v\", True) else None","note":"This override is applied only to full_attention layers, so sliding layers use /text_config/num_key_value_heads = 8."},"attn.local_layers":{"value":25,"source":"config","at":"/text_config/layer_types | count sliding_attention"},"attn.position":{"value":"partial RoPE (global, p-RoPE p=0.25), RoPE (SWA)","source":"report","at":"PDF page 2; §2 Long-context efficiency","quote":"with p-RoPE with p = 0.25 on global attention"},"attn.qk_norm":{"value":true,"source":"report","at":"PDF page 2; §2","quote":"and QKNorm (Henry et al., 2020)."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/full_attention/partial_rotary_factor","note":"Global layers use rope_type proportional (p-RoPE)."},"attn.rope_theta":{"value":1000000,"source":"config","at":"/text_config/rope_parameters/full_attention/rope_theta"},"attn.scale":{"value":"1.0 (no 1/sqrt(head_dim)); V RMS-normalized without learned scale","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextAttention.__init__","quote":"self.num_key_value_groups = config.num_attention_heads // layer_config.num_key_value_heads self.scaling = 1.0","note":"v_norm is Gemma4RMSNorm(head_dim, with_scale=False), applied in every attention layer."},"attn.window":{"value":1024,"source":"config","at":"/text_config/sliding_window"},"ffn.activation":{"value":"GeGLU (tanh-approximate GELU)","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn = ACT2FN[/text_config/hidden_activation] = gelu_pytorch_tanh."},"ffn.bias":{"value":false,"source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.dense_layers":{"value":0,"source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextDecoderLayer.__init__","quote":"self.enable_moe_block = config.enable_moe_block if self.enable_moe_block: self.router = Gemma4TextRouter(config)","note":"enable_moe_block is a single model-wide flag (true in config.json), so every layer has the MoE block."},"moe.expert_dim":{"value":704,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":128,"source":"config","at":"/text_config/num_experts"},"moe.norm_topk":{"value":true,"source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextRouter.forward","quote":"top_k_weights /= top_k_weights.sum(dim=-1, keepdim=True)","note":"A learned per-expert scale multiplies the renormalized weights."},"moe.router":{"value":"softmax","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextRouter.forward","quote":"router_probabilities = nn.functional.softmax(expert_scores, dim=-1, dtype=torch.float32)","note":"Router input is the residual stream entering the FFN sub-block, RMS-normalized without weight, times a learned vector and d_model^-0.5."},"moe.shared":{"value":1,"source":"card","at":"README.md; Mixture-of-Experts (MoE) Model table","quote":"Expert Count | 8 active / 128 total and 1 shared"},"moe.shared_dim":{"value":2112,"source":"config","at":"/text_config/intermediate_size","note":"The shared expert is the layer's dense GeGLU MLP; its output and the routed-expert output each get their own post-RMSNorm and are summed."},"moe.top_k":{"value":8,"source":"config","at":"/text_config/top_k_experts"},"norm.embedding":{"value":"× sqrt(d_model)","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextModel.__init__","quote":"config.vocab_size, config.hidden_size, self.padding_idx, embed_scale=self.config.hidden_size**0.5"},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.logit_softcap":{"value":30.0,"source":"config","at":"/text_config/final_logit_softcapping"},"norm.placement":{"value":"sandwich","source":"report","at":"PDF page 2; §2","quote":"Our models have pre-norm and post-norm"},"norm.type":{"value":"RMSNorm","source":"report","at":"PDF page 2; §2","quote":"with RMSNorm (Zhang and Sennrich, 2019),"},"ctx.max":{"value":262144,"source":"card","at":"README.md; Mixture-of-Experts (MoE) Model table","quote":"Context Length | 256K tokens | Vocabulary Size","note":"config max_position_embeddings is also 262144."},"train.cutoff":{"value":"January 2025","source":"report","at":"PDF page 3; §2.4 Training data","quote":"with a cutoff date of January 2025."},"train.data":{"value":"Web documents (140+ languages), code, mathematics, images","source":"card","at":"README.md; Training Dataset","quote":"The training dataset includes content in over 140 languages."},"train.hardware":{"value":"6,144 TPUv6e chips (16 data × 16 sequence × 24 replica)","source":"report","at":"PDF page 2; Table 2","quote":"26B-A4B* v6e 6,144 16 16 24"},"train.accelerators":{"value":"6,144 TPU v6e","source":"report","at":"PDF page 2; Table 2","quote":"26B-A4B* v6e 6,144 16 16 24"},"train.parallelism":{"value":"data16 · seq16 · replica24","source":"report","at":"PDF page 2; Table 2","quote":"26B-A4B* v6e 6,144 16 16 24","note":"Table 2 sharding axes (data, sequence, replica) as named by the report, not mapped to TP/CP/DP; §2.7 adds ZeRO-3 optimizer-state sharding."},"eval.aa_index":{"value":17,"source":"aa","at":"Artificial Analysis model page /models/gemma-4-26b-a4b, Wayback snapshot 2026-10-19","quote":"Gemma 4 26B A4B (Reasoning) scores 17 on the Artificial Analysis Intelligence Index","note":"Score of Gemma-4-26B-A4B-it, post-trained from this base. Variant: Gemma 4 26B A4B (Reasoning). Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":25233053470,"formula":"embedding + output head + all experts + other weights, google/gemma-4-26B-A4B@24548b6 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_embedding":{"value":738197504,"formula":"input embedding + output head, google/gemma-4-26B-A4B@24548b6 safetensors headers"},"params.counted_non_embedding":{"value":24494855966,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, google/gemma-4-26B-A4B@24548b6 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_max_layer":{"value":828510465,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, google/gemma-4-26B-A4B@24548b6 safetensors headers"},"params.counted_active_non_embedding":{"value":3084244766.0,"formula":"non-expert weights + routed experts × 8/128, excluding embedding and output head, google/gemma-4-26B-A4B@24548b6 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"shape.aspect":{"value":93.86666666666666,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":5.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":16.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.15079365079365079,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":20480.0,"formula":"20480 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":2894069760.0,"formula":"20480 B × 131072 (Global KV) + 204800 B × min(131072, 1024) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":22080.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":5810.526315789474,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":761597305.263158,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":20480.0,"windows":[[204800.0,1024]],"groups":[{"name":"Global KV","bytes_per_entry":20480.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":204800.0,"tokens_per_entry":1,"window":1024}],"formula":"Global KV: 5 × (2048 × 16/8) B/entry; SWA KV: 25 × (4096 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"gemma-4-31b","name":"Gemma-4-31B","org":"Google DeepMind","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/google/gemma-4-31B/resolve/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/google/gemma-4-31B/resolve/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89/config.json","kind":"config"},"modeling_gemma4":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/gemma4/modeling_gemma4.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2607.02770v2","kind":"paper"},"configuration_gemma4":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/gemma4/configuration_gemma4.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20261012212433/https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019100237/https://artificialanalysis.ai/models/gemma-4-31b","kind":"leaderboard"}},"facts":{"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.released":{"value":"2026-04-02","source":"blog","at":"Gemma 4 launch post header","quote":"Gemma 4: Byte for byte, the most capable open models Apr 02, 2026"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Dense Models table (31B Dense column)","quote":"| **Supported Modalities** | Text, Image, Audio | Text, Image, Audio | Text, Image, Audio | Text, Image |","note":"The card also lists video understanding as processing sequences of frames; audio is limited to E2B, E4B and 12B."},"params.embedding":{"value":1410000000,"source":"report","at":"PDF page 2; Table 1","quote":"31B - 550M 1,410M 29,290M 500M"},"params.non_embedding":{"value":29290000000,"source":"report","at":"PDF page 2; Table 1","quote":"31B - 550M 1,410M 29,290M 500M"},"params.note":{"value":"Language model only (1,410M embedder + 29,290M einsums). Excludes the 550M frozen vision encoder and the 500M MTP drafter.","source":"report","at":"PDF page 2; Table 1","quote":"31B - 550M 1,410M 29,290M 500M"},"params.active":{"value":30700000000,"source":"card","at":"README.md; Dense Models table","quote":"11.95B | 30.7B","basis":"inferred","note":"Dense model; active equals the stated total."},"params.active_non_embedding":{"value":29290000000,"source":"report","at":"PDF page 2; Table 1","quote":"31B - 550M 1,410M 29,290M 500M","note":"Einsums column of the dense model; the tied 1,410M embedder is counted separately."},"params.total":{"value":30700000000,"source":"card","at":"README.md; Dense Models table","quote":"11.95B | 30.7B"},"shape.d_model":{"value":5376,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":60,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":true,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.tokenizer":{"value":"SentencePiece, same as Gemini: split digits, preserved whitespace, byte-level encodings","source":"report","at":"PDF page 3; §2.4 Tokenizer","quote":"digits, preserved whitespace, and byte-level en"},"shape.vocab":{"value":262144,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":10,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":512,"source":"config","at":"/text_config/global_head_dim","note":"Global layers only; sliding-window layers use head_dim 256."},"attn.heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; Models Overview","quote":"interleaves local sliding window attention with full global attention, ensuring the final layer is always global.","note":"Global layers reuse keys as values (attention_k_eq_v: no v_proj; V is the K projection before RoPE and k_norm). use_bidirectional_attention vision: image tokens attend bidirectionally, text is causal."},"attn.kv_heads":{"value":4,"source":"config","at":"/text_config/num_global_key_value_heads"},"attn.layout":{"value":"5 SWA : 1 global","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Ordered 60-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.local_head_dim":{"value":256,"source":"config","at":"/text_config/head_dim","note":"configuration_gemma4.py overrides head_dim (global_head_dim) and num_key_value_heads (num_global_key_value_heads) only for full_attention layers; sliding layers keep head_dim, num_key_value_heads and num_attention_heads."},"attn.local_heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads","note":"Same num_attention_heads attribute for both layer types (Gemma4TextAttention)."},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/text_config/rope_parameters/sliding_attention/rope_theta","note":"rope_type default."},"attn.local_kv_heads":{"value":16,"source":"configuration_gemma4","at":"configuration_gemma4.py; Gemma4TextConfig.__post_init__","quote":"num_key_value_heads = num_global_key_value_heads if getattr(self, \"attention_k_eq_v\", True) else None","note":"This override is applied only to full_attention layers, so sliding layers use /text_config/num_key_value_heads = 16."},"attn.local_layers":{"value":50,"source":"config","at":"/text_config/layer_types | count sliding_attention"},"attn.position":{"value":"partial RoPE (global, p-RoPE p=0.25), RoPE (SWA)","source":"report","at":"PDF page 2; §2 Long-context efficiency","quote":"with p-RoPE with p = 0.25 on global attention"},"attn.qk_norm":{"value":true,"source":"report","at":"PDF page 2; §2","quote":"and QKNorm (Henry et al., 2020)."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/full_attention/partial_rotary_factor","note":"Global layers use rope_type proportional (p-RoPE)."},"attn.rope_theta":{"value":1000000,"source":"config","at":"/text_config/rope_parameters/full_attention/rope_theta"},"attn.scale":{"value":"1.0 (no 1/sqrt(head_dim)); V RMS-normalized without learned scale","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextAttention.__init__","quote":"self.num_key_value_groups = config.num_attention_heads // layer_config.num_key_value_heads self.scaling = 1.0","note":"v_norm is Gemma4RMSNorm(head_dim, with_scale=False), applied in every attention layer."},"attn.window":{"value":1024,"source":"config","at":"/text_config/sliding_window"},"ffn.activation":{"value":"GeGLU (tanh-approximate GELU)","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn = ACT2FN[/text_config/hidden_activation] = gelu_pytorch_tanh."},"ffn.bias":{"value":false,"source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":21504,"source":"config","at":"/text_config/intermediate_size"},"norm.embedding":{"value":"× sqrt(d_model)","source":"modeling_gemma4","at":"modeling_gemma4.py; Gemma4TextModel.__init__","quote":"config.vocab_size, config.hidden_size, self.padding_idx, embed_scale=self.config.hidden_size**0.5"},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.logit_softcap":{"value":30.0,"source":"config","at":"/text_config/final_logit_softcapping"},"norm.placement":{"value":"sandwich","source":"report","at":"PDF page 2; §2","quote":"Our models have pre-norm and post-norm"},"norm.type":{"value":"RMSNorm","source":"report","at":"PDF page 2; §2","quote":"with RMSNorm (Zhang and Sennrich, 2019),"},"ctx.max":{"value":262144,"source":"card","at":"README.md; Dense Models table","quote":"Context Length | 128K tokens | 128K tokens | 256K tokens | 256K tokens","note":"config max_position_embeddings is also 262144."},"train.cutoff":{"value":"January 2025","source":"report","at":"PDF page 3; §2.4 Training data","quote":"with a cutoff date of January 2025."},"train.data":{"value":"Web documents (140+ languages), code, mathematics, images","source":"card","at":"README.md; Training Dataset","quote":"The training dataset includes content in over 140 languages."},"train.hardware":{"value":"10,240 TPUv6e chips (16 data × 16 sequence × 40 replica)","source":"report","at":"PDF page 2; Table 2","quote":"31B v6e 10,240 16 16 40"},"train.accelerators":{"value":"10,240 TPU v6e","source":"report","at":"PDF page 2; Table 2","quote":"31B v6e 10,240 16 16 40"},"train.parallelism":{"value":"data16 · seq16 · replica40","source":"report","at":"PDF page 2; Table 2","quote":"31B v6e 10,240 16 16 40","note":"Table 2 sharding axes (data, sequence, replica) as named by the report, not mapped to TP/CP/DP; §2.7 adds ZeRO-3 optimizer-state sharding."},"eval.aa_index":{"value":15,"source":"aa","at":"Artificial Analysis model page /models/gemma-4-31b, Wayback snapshot 2026-10-19","quote":"Gemma 4 31B (Reasoning) scores 15 on the Artificial Analysis Intelligence Index","note":"Score of Gemma-4-31B-it, post-trained from this base. Variant: Gemma 4 31B (Reasoning). Index v4.3.2."}},"not_disclosed":["mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":30697345340,"formula":"embedding + output head + all experts + other weights, google/gemma-4-31B@5bbc2fb safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_embedding":{"value":1409286144,"formula":"input embedding + output head, google/gemma-4-31B@5bbc2fb safetensors headers"},"params.counted_non_embedding":{"value":29288059196,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, google/gemma-4-31B@5bbc2fb safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_max_layer":{"value":534009857,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, google/gemma-4-31B@5bbc2fb safetensors headers"},"params.counted_active_non_embedding":{"value":29288059196.0,"formula":"all weights except embedding and output head, google/gemma-4-31B@5bbc2fb safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"shape.aspect":{"value":89.6,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"attn.swa_ratio":{"value":5.0,"formula":"attn.local_layers / attn.global_layers"},"kv.growth_per_token":{"value":81920.0,"formula":"81920 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":11576279040.0,"formula":"81920 B × 131072 (Global KV) + 819200 B × min(131072, 1024) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":88320.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2876.8729641693812,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":377077493.15960914,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":81920.0,"windows":[[819200.0,1024]],"groups":[{"name":"Global KV","bytes_per_entry":81920.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":819200.0,"tokens_per_entry":1,"window":1024}],"formula":"Global KV: 10 × (4096 × 16/8) B/entry; SWA KV: 50 × (8192 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"glm53-flash","name":"GLM-5.3-Flash","org":"Z.ai","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; the card says the model starts from a newly trained base, not the GLM-5.2/5.3 base. Architecture from config and the pinned transformers implementation.","sources":{"card":{"url":"https://huggingface.co/zai-org/GLM-5.3-Flash/resolve/46a050bec13002ad29ee795c16e61cf4fa3c4a9c/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/zai-org/GLM-5.3-Flash/resolve/46a050bec13002ad29ee795c16e61cf4fa3c4a9c/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/zai-org/GLM-5.3-Flash/resolve/46a050bec13002ad29ee795c16e61cf4fa3c4a9c/model.safetensors.index.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/glm5_next/modeling_glm5_next.py","kind":"code"},"hf-configuration":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/glm5_next/configuration_glm5_next.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20261019095936id_/https://z.ai/blog/assets/glm-5.3-flash-nnmBGEIY.js","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019092930/https://artificialanalysis.ai/models/glm-5-3-flash","kind":"leaderboard"},"deepswe":{"url":"https://web.archive.org/web/20261009070156id_/https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-26","source":"blog","at":"GLM-5.3-Flash blog; post date","quote":"2026-08-26","note":"The post states the weights are publicly available."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextVisionModel","quote":"input_modalities = (\"image\", \"video\")","note":"The card calls the model natively multimodal without listing modalities; the transformers class declares image and video inputs alongside text."},"params.total":{"value":320000000000,"source":"card","at":"README.md; Introduction","quote":"With 320B total parameters and just 18B active parameters"},"params.active":{"value":18000000000,"source":"card","at":"README.md; Introduction","quote":"With 320B total parameters and just 18B active parameters"},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":45,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":154880,"source":"config","at":"/text_config/vocab_size"},"attn.kind":{"value":"KDA, MLA, DSA","source":"card","at":"README.md; Introduction","quote":"we introduce a hybrid architecture combining sparse and linear attention","note":"Linear layers are Kimi Delta Attention (Glm5NextTextLinearAttention); sparse layers are MLA with a DSA indexer (Glm5NextTextAttention)."},"attn.layout":{"value":"(3 KDA : 1 MLA+DSA) × 11 + 1 KDA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"deepseek_sparse_attention\", \"linear_attention\" ]","basis":"inferred","note":"0-indexed MLA+DSA layers of 45; the rest (kda_layers), including the last (44), are KDA. Ordered 45-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","sequence":["KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA","KDA","KDA","DSA","KDA"]},"attn.global_layers":{"value":11,"source":"config","at":"/text_config/layer_types | count deepseek_sparse_attention"},"attn.linear_layers":{"value":34,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/qk_nope_head_dim + /text_config/qk_rope_head_dim","note":"256 nope + 0 rope. The config's /text_config/head_dim=0 is the RoPE sub-dim."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/text_config/kv_lora_rank"},"attn.rope_dim":{"value":0,"source":"config","at":"/text_config/qk_rope_head_dim"},"attn.v_head_dim":{"value":256,"source":"config","at":"/text_config/v_head_dim"},"attn.qk_norm":{"value":false,"source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextAttention.forward","quote":"q_resid = self.q_a_layernorm(self.q_a_proj(hidden_states))","note":"MLA layers normalize only the Q and KV latents; KDA layers L2-normalize q/k inside the recurrence."},"attn.indexer_dim":{"value":128,"source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextIndexer.__init__","quote":"self.wk = nn.Linear(self.hidden_size, self.head_dim, bias=False)","note":"One key head; IndexPool pools 4 consecutive keys into one (see attn.kv_groups)."},"attn.kv_groups":{"value":[{"name":"MLA latent","layers":11,"elements":512,"tokens_per_entry":1,"note":"kv_lora_rank 512, no RoPE key.","quote":"k_pass = self.kv_a_layernorm(kv_pass).view(batch_size, 1, seq_length, self.kv_lora_rank)"},{"name":"IndexPool indexer key","layers":11,"elements":128,"tokens_per_entry":4,"note":"Blog: IndexPool compresses four indexer key vectors into one; transformers caches unpooled keys and pools at scoring time.","quote":"# Key difference: Score across pools, not on a per token basis"}],"source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextAttention, Glm5NextTextIndexer.forward"},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/text_config/q_lora_rank"},"attn.position":{"value":"NoPE","source":"hf-configuration","at":"configuration_glm5_next.py; Glm5NextTextConfig.__post_init__","quote":"Expecting NoPE for the DSA attention layers","note":"qk_rope_head_dim 0 (config mla_use_nope true); KDA layers carry no positional encoding."},"attn.sparse":{"value":"DSA indexer (32 heads × 128), top-2048 tokens, indexer keys pooled 4:1","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextIndexer docstring","quote":"DeepSeek Sparse Attention (DSA) indexer with k-pool compression for GLM-5.3-Flash.","note":"Blog calls the 4:1 key pooling IndexPool. /text_config/index_topk 2048, /text_config/index_kpool 4, /text_config/index_n_heads 32, /text_config/index_head_dim 128; every MLA layer runs its own indexer (/text_config/indexer_types all \"full\"); index_kpool_always_select_tail adds the trailing incomplete pool's tokens."},"attn.linear_kind":{"value":"KDA: 64 heads × 128, short conv 4 on q/k/v, forget-gate lower bound −5, q/k L2-normalized, gated RMSNorm output","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextLinearAttention","quote":"Kimi-style KDA (Kimi Linear Attention) for GLM-5.3-Flash.","note":"/text_config/linear_attn_config: num_heads 64, head_dim 128, short_conv_kernel_size 4, gate_lower_bound -5.0."},"attn.linear_heads":{"value":64,"source":"config","at":"/text_config/linear_attn_config/num_heads"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_attn_config/head_dim"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_attn_config/short_conv_kernel_size"},"attn.gate":{"value":"KDA layers: gated RMSNorm with low-rank gate (g_a_proj → g_b_proj) before o_proj; none on MLA layers","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextLinearAttention.forward","quote":"output = self.o_norm(core_attn_out, gate).reshape(batch_size, seq_len, -1)"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextMLP.forward","quote":"return self.down_proj(self.act_fn(gate) * up)","note":"act_fn is /text_config/hidden_act silu; clamps in ffn.clamp."},"ffn.dim":{"value":12288,"source":"config","at":"/text_config/intermediate_size"},"ffn.clamp":{"value":"gate ≤ 10, up ±10; dense, shared and routed experts","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextMLP.forward, Glm5NextTextExperts._apply_gate","quote":"up = up.clamp(min=-self.swiglu_limit, max=self.swiglu_limit)","note":"/text_config/swiglu_limit 10.0; gate is clamped above only."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.dense_layers":{"value":3,"source":"config","at":"/text_config/mlp_layer_types | count dense","note":"Layers 0–2; matches first_k_dense_replace."},"moe.experts":{"value":288,"source":"config","at":"/text_config/n_routed_experts"},"moe.top_k":{"value":8,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.shared":{"value":1,"source":"config","at":"/text_config/n_shared_experts"},"moe.expert_dim":{"value":2048,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.shared_dim":{"value":2048,"source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextMoE.__init__","quote":"intermediate_size=config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"2048 × 1 shared expert."},"moe.router":{"value":"sigmoid","source":"config","at":"/text_config/scoring_func","note":"Router logits computed in FP32 (moe_router_dtype float32)."},"moe.balancing":{"value":"expert-score correction bias on top-k selection","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextTopkRouter.forward","quote":"scores_for_choice = scores + self.e_score_correction_bias","note":"Bias affects selection only; its update rule and any auxiliary loss are not disclosed."},"moe.groups":{"value":"none","source":"config","at":"config.json n_group, topk_group","quote":"\"n_group\": 1,","note":"n_group = topk_group = 1."},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/text_config/routed_scaling_factor","note":"Applied after top-k renormalization (Glm5NextTextTopkRouter.forward)."},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.language_model.layers.45.eh_proj.weight\"","note":"Its use in pretraining is not disclosed."},"mtp.modules":{"value":1,"source":"config","at":"/text_config/num_nextn_predict_layers"},"mtp.block":{"value":"enorm + hnorm, eh_proj, one MLA+DSA layer with its own IndexPool indexer, MoE FFN, shared_head.norm; no hyper-connection weights","source":"hf-index","at":"model.safetensors.index.json; model.language_model.layers.45.* weight names","quote":"\"model.language_model.layers.45.self_attn.indexer.index_kpool_compress_gate\"","note":"No embedding or output-head weights under layer 45."},"mtp.ffn":{"value":"MoE","source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.language_model.layers.45.mlp.gate.e_score_correction_bias\""},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextDecoderLayer.__init__","quote":"self.input_layernorm = Glm5NextTextRMSNorm(config.hidden_size, config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_glm5_next.py; Glm5NextTextDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)","note":"Applied to the mHC-collapsed block input."},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"norm.residual":{"value":"mHC: 4 residual streams, Sinkhorn-Knopp (20 iterations) doubly-stochastic stream mixing; streams averaged before the final norm","source":"card","at":"README.md; Introduction","quote":"The model also adopts Manifold-Constrained Hyper-Connections (mHC) to further improve scaling efficiency.","note":"/text_config/hc_mult 4, /text_config/hc_sinkhorn_iters 20, hc_eps 1e-6; Glm5NextTextHyperHead takes an unweighted mean of the streams."},"ctx.max":{"value":1048576,"source":"blog","at":"Serving section","quote":"especially when supporting context lengths of up to one million tokens","note":"Config /text_config/max_position_embeddings is 1048576."},"train.data":{"value":"30T-token multimodal corpus","source":"card","at":"README.md; Introduction","quote":"Together with our latest 30T-token multimodal pre-training corpus","note":"Corpus size; tokens seen in pretraining are not stated."},"eval.aa_index":{"value":42,"source":"aa","at":"Artificial Analysis model page /models/glm-5-3-flash, Wayback snapshot 2026-10-19","quote":"GLM-5.3-Flash scores 42 on the Artificial Analysis Intelligence Index","note":"Variant: GLM-5.3-Flash. Index v4.3.2."},"eval.deepswe":{"value":0.6339285714285714,"source":"deepswe","at":"leaderboard-live.json (generated 2026-09-22), row model glm-5-3-flash, Wayback snapshot 2026-10-09","quote":"\"model\": \"glm-5-3-flash\", \"harness\": \"mini-swe-agent\", \"reasoning_effort\": \"max\", \"config\": \"mini_swe_agent_glm_5_3_flash_max\", \"source\": \"deep-swe\", \"pass_rate\": 0.6339285714285714, \"pass_at_1\": 0.6339285714285714","note":"Reasoning effort max (the only configuration listed), mini-swe-agent, 4 runs, 448 scored attempts."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":313326811696,"formula":"embedding + output head + all experts + other weights, zai-org/GLM-5.3-Flash@46a050b safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"params.counted_embedding":{"value":1268776960,"formula":"input embedding + output head, zai-org/GLM-5.3-Flash@46a050b safetensors headers"},"params.counted_non_embedding":{"value":312058034736,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, zai-org/GLM-5.3-Flash@46a050b safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"params.counted_max_layer":{"value":7412630032,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, zai-org/GLM-5.3-Flash@46a050b safetensors headers"},"params.counted_active_non_embedding":{"value":16107944496.0,"formula":"non-expert weights + routed experts × 8/288, excluding embedding and output head, zai-org/GLM-5.3-Flash@46a050b safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"shape.aspect":{"value":91.02222222222223,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"ffn.mult":{"value":3.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.090909090909091,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":36.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.05625,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":11968.0,"formula":"11264 B / 1 tokens (MLA latent) + 2816 B / 4 tokens (IndexPool indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1568669696.0,"formula":"11264 B × 131072 (MLA latent) + 2816 B × floor(131072 / 4) (IndexPool indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":11968.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":664.8888888888889,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":87148316.44444445,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":11968.0,"windows":[],"groups":[{"name":"MLA latent","bytes_per_entry":11264.0,"tokens_per_entry":1},{"name":"IndexPool indexer key","bytes_per_entry":2816.0,"tokens_per_entry":4}],"formula":"MLA latent: 11 × (512 × 16/8) B/entry; IndexPool indexer key: 11 × (128 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"glm53","name":"GLM-5.3","org":"Z.ai","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint, which uses the same base model as GLM-5.2.","sources":{"card":{"url":"https://huggingface.co/zai-org/GLM-5.3/resolve/ba32f8c19c8738e2c57c71798c601d477a7bf844/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/zai-org/GLM-5.3/resolve/ba32f8c19c8738e2c57c71798c601d477a7bf844/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/zai-org/GLM-5.3/resolve/ba32f8c19c8738e2c57c71798c601d477a7bf844/model.safetensors.index.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py","kind":"code"},"github":{"url":"https://raw.githubusercontent.com/zai-org/GLM-5/83c03d317378f86f0a012f7c4cd48b6243c7e69b/README.md","kind":"model-card"},"blog-52":{"url":"https://web.archive.org/web/20261019100129id_/https://z.ai/blog/assets/glm-5.2-DXVc0kvW.js","kind":"blog"},"hf-discussion-1":{"url":"https://huggingface.co/api/models/zai-org/GLM-5.3/discussions/1","kind":"api"},"aa":{"url":"https://web.archive.org/web/20261019092921/https://artificialanalysis.ai/models/glm-5-3","kind":"leaderboard"},"deepswe":{"url":"https://web.archive.org/web/20261009070156id_/https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-28","source":"hf-discussion-1","at":"/createdAt (first public discussion on the weights repo)","quote":"\"createdAt\":\"2026-08-28T15:03:23","note":"No dated publisher page for the weights: the 2026-08-14 blog deferred them by two weeks; the repo was created empty on 2026-08-25 and the weights committed 2026-08-27 17:16 UTC (\"Initial commit 0828\")."},"meta.license":{"value":"other (glm-5.3)","source":"card","at":"README.md; YAML front matter","quote":"license_name: glm-5.3"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; YAML front matter","quote":"pipeline_tag: text-generation","note":"The card states no modality in words; the config (GlmMoeDsaForCausalLM) has no vision tower."},"params.total":{"value":744000000000,"source":"github","at":"GLM-5 README; Download Model table, GLM-5.3 row","quote":"(https://modelscope.cn/models/ZhipuAI/GLM-5.3) | 744B-A40B"},"params.active":{"value":40000000000,"source":"github","at":"GLM-5 README; Download Model table, GLM-5.3 row","quote":"(https://modelscope.cn/models/ZhipuAI/GLM-5.3) | 744B-A40B"},"params.note":{"value":"Model Size column of the GitHub README; inclusion of the MTP layer is not stated.","source":"card","at":"README.md; intro","quote":"GLM-5.3 uses the same base model as GLM-5.2 — every gain comes from post-training."},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":78,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":154880,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias","note":"Read by GlmMoeDsaAttention for the q_a, kv_a and o projections; the others are bias-free."},"attn.global_layers":{"value":78,"source":"config","at":"/num_hidden_layers","note":"Every layer is MLA with DSA top-k selection."},"attn.head_dim":{"value":256,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim","note":"192 nope + 64 rope (= /qk_head_dim). The config's /head_dim=192 is not the attention head size."},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA, DSA","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaIndexer docstring","quote":"DeepSeek Sparse Attention (DSA) indexer for selecting top-k tokens."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"78 DSA","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaAttention.forward","quote":"# DSA: select this layer's top-k tokens, or reuse the previous full layer's on `\"shared\"` layers.","note":"/indexer_types: 21 \"full\" layers (0, 1, 2, then every 4th from 6) and 57 \"shared\" layers."},"attn.position":{"value":"partial RoPE (MLA decoupled, interleaved, 64 of 256 QK dims)","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaAttention.forward","quote":"q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)","note":"Interleaving is hard-coded for MLA and indexer; config /rope_interleave and /indexer_rope_interleave are true but not read."},"attn.qk_norm":{"value":false,"source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaAttention.forward","quote":"q_resid = self.q_a_layernorm(self.q_a_proj(hidden_states))","note":"RMSNorm only on the Q and KV latents; no per-head Q/K norm."},"attn.indexer_dim":{"value":128,"source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaIndexer.__init__","quote":"self.wk = nn.Linear(self.hidden_size, self.head_dim, bias=False)","note":"One 128-dim key (/index_head_dim, LayerNorm k_norm) per token, cached only on the 21 \"full\" indexer layers; see attn.kv_groups."},"attn.kv_groups":{"value":[{"name":"MLA latent + RoPE key","layers":78,"elements":576,"tokens_per_entry":1,"note":"kv_lora_rank 512 + qk_rope_head_dim 64","quote":"kv_pass, k_rot = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)"},{"name":"DSA indexer key","layers":21,"elements":128,"tokens_per_entry":1,"note":"Only \"full\" layers own an indexer; 57 \"shared\" layers reuse its top-k.","quote":"self.indexer = None if self.skip_topk else GlmMoeDsaIndexer(config, layer_idx)"}],"source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaAttention"},"attn.q_lora_rank":{"value":2048,"source":"config","at":"/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":8000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.sparse":{"value":"DSA indexer, top-2048; 21 layers run their own indexer, 57 reuse the previous indexer layer's top-k","source":"config","at":"config.json index_topk","quote":"\"index_topk\": 2048,","note":"Indexer: 32 query heads × 128 dims (/index_n_heads, /index_head_dim), one key head; /indexer_types marks layers 0, 1, 2 and every 4th from 6 as \"full\"."},"attn.v_head_dim":{"value":256,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (config hidden_act)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":12288,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":3,"source":"config","at":"/mlp_layer_types | count dense","note":"Layers 0–2; matches /first_k_dense_replace. The code reads /mlp_layer_types."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor","note":"Applied after top-k renormalization in the MoE router (modeling_glm_moe_dsa.py)."},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func","note":"Router logits computed in FP32 in GlmMoeDsaTopkRouter (/moe_router_dtype float32)."},"moe.shared_dim":{"value":2048,"source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaMoE.__init__","quote":"config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"2048 × 1 shared expert."},"moe.balancing":{"value":"expert-score correction bias on top-k selection","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaTopkRouter.forward","quote":"scores_for_choice = scores + self.e_score_correction_bias","note":"Bias affects selection only; its update rule and any auxiliary loss are not disclosed."},"moe.groups":{"value":"none","source":"config","at":"config.json n_group, topk_group","quote":"\"n_group\": 1,","note":"/n_group = /topk_group = 1."},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.78.eh_proj.weight\""},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers","note":"Multi-step MTP shares this module's parameters across steps (GLM-5.2 blog, MTP with IndexShare and KVShare); the training step count is not stated."},"mtp.block":{"value":"enorm + hnorm, eh_proj, one MLA+DSA layer with its own indexer, MoE FFN, shared_head.norm","source":"hf-index","at":"model.safetensors.index.json; model.layers.78.* weight names","quote":"\"model.layers.78.self_attn.indexer.wk.weight\"","note":"No embedding or output-head weights under layer 78."},"mtp.ffn":{"value":"MoE","source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.78.mlp.gate.e_score_correction_bias\""},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_glm_moe_dsa.py; GlmMoeDsaDecoderLayer.__init__","quote":"self.input_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps)"},"ctx.max":{"value":1048576,"source":"config","at":"/max_position_embeddings","note":"The card's evaluations use up to a 1M-token context window."},"ctx.extension":{"value":"IndexShare introduced in 128K mid-training of the GLM-5.2 base","source":"blog-52","at":"GLM-5.2 blog; IndexShare for DSA","quote":"GLM-5.2 is trained with IndexShare from mid-training with 128K sequence length"},"eval.aa_index":{"value":45,"source":"aa","at":"Artificial Analysis model page /models/glm-5-3, Wayback snapshot 2026-10-19","quote":"GLM-5.3 (Max) scores 45 on the Artificial Analysis Intelligence Index","note":"Variant: GLM-5.3 (Max). Index v4.3.2."},"eval.deepswe":{"value":0.6895787139689579,"source":"deepswe","at":"leaderboard-live.json (generated 2026-09-22), row model glm-5-3, Wayback snapshot 2026-10-09","quote":"\"model\": \"glm-5-3\", \"harness\": \"mini-swe-agent\", \"reasoning_effort\": \"max\", \"config\": \"mini_swe_agent_glm_5_3_max\", \"source\": \"deep-swe\", \"pass_rate\": 0.6895787139689579, \"pass_at_1\": 0.6895787139689579","note":"Reasoning effort max (the only configuration listed), mini-swe-agent, 4 runs, 451 scored attempts."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":743377019904,"formula":"embedding + output head + all experts + other weights, zai-org/GLM-5.3@ba32f8c safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":1903165440,"formula":"input embedding + output head, zai-org/GLM-5.3@ba32f8c safetensors headers"},"params.counted_non_embedding":{"value":741473854464,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, zai-org/GLM-5.3@ba32f8c safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":9877404672,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, zai-org/GLM-5.3@ba32f8c safetensors headers"},"params.counted_active_non_embedding":{"value":39347364864.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, zai-org/GLM-5.3@ba32f8c safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":78.76923076923077,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"ffn.mult":{"value":2.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.053763440860215055,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":95232.0,"formula":"89856 B / 1 tokens (MLA latent + RoPE key) + 5376 B / 1 tokens (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":12482248704.0,"formula":"89856 B × 131072 (MLA latent + RoPE key) + 5376 B × 131072 (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":95232.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2380.8,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":312056217.6,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":95232.0,"windows":[],"groups":[{"name":"MLA latent + RoPE key","bytes_per_entry":89856.0,"tokens_per_entry":1},{"name":"DSA indexer key","bytes_per_entry":5376.0,"tokens_per_entry":1}],"formula":"MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"gpt-oss-120b","name":"gpt-oss-120b","org":"OpenAI","country":"United States","kind":"reasoning","weights":"open","note":"Only the post-trained reasoning checkpoint is public; MoE weights were quantized to MXFP4 during post-training.","sources":{"config":{"url":"https://huggingface.co/openai/gpt-oss-120b/resolve/b5c939de8f754692c1647ca79fbf85e8c1e70f8a/config.json","kind":"config"},"card":{"url":"https://arxiv.org/pdf/2508.10925v1","kind":"paper"},"hf_card":{"url":"https://huggingface.co/openai/gpt-oss-120b/resolve/b5c939de8f754692c1647ca79fbf85e8c1e70f8a/README.md","kind":"model-card"},"modeling_gpt_oss":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/gpt_oss/modeling_gpt_oss.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019100442/https://artificialanalysis.ai/models/gpt-oss-120b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-08-05","source":"card","at":"PDF p.1 title block","quote":"August 5, 2025","note":"Weights published on Hugging Face the same day."},"meta.license":{"value":"apache-2.0","source":"hf_card","at":"README.md YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"§1 Introduction","quote":"these text-only models are compatible with our Responses API"},"params.active":{"value":5130000000,"source":"card","at":"PDF p.4 §2, Table 1","quote":"Active Parameters 5.13B 3.61B"},"params.active_non_embedding":{"value":4550000000,"source":"card","at":"PDF p.4 §2, Table 1 caption","quote":"Unembedding parameters are counted towards active, but not embeddings.","basis":"inferred","note":"5.13B − 201,088 × 2,880 (unembedding, counted as active; input embedding not counted) = ~4.55B."},"params.note":{"value":"Unembedding counted as active, input embedding not; 1.16B embed + unembed.","source":"card","at":"PDF p.4 §2, Table 1 caption","quote":"Unembedding parameters are counted towards active, but not embeddings."},"params.total":{"value":116830000000,"source":"card","at":"PDF p.4 §2, Table 1","quote":"Total Parameters 116.83B 20.91B"},"params.embedding":{"value":1160000000,"source":"card","at":"PDF p.4 §2, Table 1","quote":"Embed + Unembed 1.16B 1.16B"},"shape.d_model":{"value":2880,"source":"config","at":"/hidden_size"},"shape.layers":{"value":36,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":201088,"source":"config","at":"/vocab_size"},"shape.tokenizer":{"value":"o200k_harmony: BPE extending o200k (GPT-4o, o4-mini) with harmony chat-format tokens","source":"card","at":"PDF p.5 §2.3 Tokenizer","quote":"This is a Byte Pair Encoding (BPE) which extends the o200k tokenizer used for other OpenAI models such as GPT-4o and OpenAI o4-mini"},"attn.global_layers":{"value":18,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":64,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"PDF p.4 §2.2 Attention","quote":"Following GPT-3, attention blocks alternate between banded window and fully dense patterns [10][11], where the bandwidth is 128 tokens."},"attn.sinks":{"value":"learned per-head sink logit in the softmax denominator, all layers","source":"card","at":"PDF p.4 §2.2 Attention","quote":"Each attention head has a learned bias in the denominator of the softmax, similar to off-by-one attention and attention sinks"},"attn.bias":{"value":true,"source":"config","at":"/attention_bias"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"1 SWA : 1 global","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Config /layer_types alternates strictly, starting with a sliding-window layer. Ordered 36-layer backbone from /layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA","SWA","GQA"]},"attn.local_layers":{"value":18,"source":"config","at":"/layer_types | count sliding_attention"},"attn.local_heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"GptOssAttention reads the same head counts, head_dim and rotary embedding for both layer types; only sliding_window differs."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads","note":"Same attribute as the full-attention layers (GptOssAttention.__init__)."},"attn.local_head_dim":{"value":64,"source":"config","at":"/head_dim","note":"Same attribute as the full-attention layers (GptOssAttention.__init__)."},"attn.local_rope_theta":{"value":150000,"source":"config","at":"/rope_theta","note":"GptOssModel builds one rotary embedding shared by both layer types."},"attn.qk_norm":{"value":false,"source":"modeling_gpt_oss","at":"modeling_gpt_oss.py; GptOssAttention.forward","quote":"query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)","note":"Queries and keys go from projection straight to RoPE; no norm module."},"attn.position":{"value":"RoPE","source":"card","at":"PDF p.4 §2.2 Attention","quote":"We apply rotary position embeddings [14] and extend the context length of dense layers to 131,072 tokens using YaRN [15]."},"attn.rope_theta":{"value":150000,"source":"config","at":"/rope_theta"},"attn.window":{"value":128,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"card","at":"PDF p.4 §2.2 Mixture-of-Experts","quote":"The MoE blocks use the gated SwiGLU [9] activation function"},"ffn.clamp":{"value":"gate clamped to ≤7, up clamped to [-7, 7]; output (up + 1) · gate · σ(1.702 · gate); all MoE layers","source":"modeling_gpt_oss","at":"modeling_gpt_oss.py; GptOssExperts._apply_gate","quote":"up = up.clamp(min=-self.limit, max=self.limit)","note":"Limit 7.0 from config /swiglu_limit, α 1.702 from GptOssConfig default; the card calls this clamping plus a residual connection (footnote 1)."},"ffn.bias":{"value":true,"source":"modeling_gpt_oss","at":"modeling_gpt_oss.py; GptOssExperts.__init__","quote":"self.gate_up_proj_bias = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_size))"},"moe.expert_dim":{"value":2880,"source":"config","at":"/intermediate_size"},"moe.experts":{"value":128,"source":"config","at":"/num_local_experts"},"moe.router":{"value":"softmax (over the top-k selected logits)","source":"card","at":"PDF p.4 §2.2 Mixture-of-Experts","quote":"weight the output of each expert by the softmax of the router projection over only the selected experts.","note":"Router is a linear layer with bias (GptOssTopKRouter)."},"moe.norm_topk":{"value":true,"source":"card","at":"PDF p.4 §2.2 Mixture-of-Experts","quote":"weight the output of each expert by the softmax of the router projection over only the selected experts.","note":"Softmax after top-k, so selected weights sum to 1."},"moe.shared":{"value":0,"source":"modeling_gpt_oss","at":"modeling_gpt_oss.py; GptOssMLP.__init__","quote":"self.router = GptOssTopKRouter(config) self.experts = GptOssExperts(config)","note":"The MoE block has only a router and routed experts."},"moe.dense_layers":{"value":0,"source":"modeling_gpt_oss","at":"modeling_gpt_oss.py; GptOssDecoderLayer.__init__","quote":"self.mlp = GptOssMLP(config)","note":"Every decoder layer uses the MoE block."},"moe.top_k":{"value":4,"source":"config","at":"/num_experts_per_tok"},"norm.placement":{"value":"pre","source":"card","at":"PDF p.4 §2.2","quote":"Similar to GPT-2 we use Pre-LN placement"},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.type":{"value":"RMSNorm","source":"card","at":"PDF p.4 §2.2","quote":"applying root mean square normalization [6] on the activations before each attention and MoE block."},"ctx.extension":{"value":"YaRN on the dense layers to 131,072","source":"card","at":"PDF p.4 §2.2 Attention","quote":"extend the context length of dense layers to 131,072 tokens using YaRN [15].","note":"Config rope_scaling: rope_type yarn, factor 32, original_max_position_embeddings 4096, beta_fast 32, beta_slow 1; HF code applies it to both layer types."},"ctx.max":{"value":131072,"source":"card","at":"PDF p.4 §2.2 Attention","quote":"extend the context length of dense layers to 131,072 tokens using YaRN [15].","note":"Equals config max_position_embeddings."},"train.data":{"value":"text-only, trillions of tokens; STEM, coding, general knowledge; CBRN-filtered with GPT-4o pretraining filters","source":"card","at":"PDF p.5 §2.4 Pretraining","quote":"We train the models on a text-only dataset with trillions of tokens, with a focus on STEM, coding, and general knowledge.","note":"Next sentence: harmful content filtered by reusing the CBRN pre-training filters from GPT-4o."},"train.cutoff":{"value":"June 2024","source":"card","at":"PDF p.5 §2.4 Pretraining","quote":"Our model has a knowledge cutoff of June 2024."},"train.hardware":{"value":"NVIDIA H100","source":"card","at":"PDF p.5 §2.4 Pretraining","quote":"The gpt-oss models trained on NVIDIA H100 GPUs using the PyTorch framework"},"train.accelerators":{"value":"H100","source":"card","at":"PDF p.5 §2.4 Pretraining","quote":"The gpt-oss models trained on NVIDIA H100 GPUs using the PyTorch framework"},"train.compute":{"value":"2.1M H100-hours","source":"card","at":"PDF p.5 §2.4 Pretraining","quote":"The training run for gpt-oss-120b required 2.1 million H100-hours to complete","note":"Stated under §2.4 Pretraining as \"the training run\"; whether post-training is included is not stated."},"eval.aa_index":{"value":12,"source":"aa","at":"Artificial Analysis model page /models/gpt-oss-120b, Wayback snapshot 2026-10-19","quote":"gpt-oss-120b (High) scores 12 on the Artificial Analysis Intelligence Index","note":"Variant: gpt-oss-120b (High). Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":116829156672,"formula":"embedding + output head + all experts + other weights, openai/gpt-oss-120b@b5c939d safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":1158266880,"formula":"input embedding + output head, openai/gpt-oss-120b@b5c939d safetensors headers"},"params.counted_non_embedding":{"value":115670889792,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, openai/gpt-oss-120b@b5c939d safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":3213080192,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, openai/gpt-oss-120b@b5c939d safetensors headers"},"params.counted_active_non_embedding":{"value":4553716032.0,"formula":"non-expert weights + routed experts × 4/128, excluding embedding and output head, openai/gpt-oss-120b@b5c939d safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":80.0,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":1.0,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":1.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04390995463493966,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":36864.0,"formula":"36864 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":4836556800.0,"formula":"36864 B × 131072 (Global KV) + 36864 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":36900.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":7192.982456140351,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":942798596.4912281,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":36864.0,"windows":[[36864.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":36864.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":36864.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 18 × (1024 × 16/8) B/entry; SWA KV: 18 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"granite-4-1-30b-base","name":"Granite-4.1-30B-Base","org":"IBM","country":"United States","kind":"base","weights":"open","note":null,"sources":{"blog":{"url":"https://web.archive.org/web/20260430151008/https://huggingface.co/blog/ibm-granite/granite-4-1","kind":"blog"},"disclosure":{"url":"https://raw.githubusercontent.com/ibm-granite/granite-4.1-language-models/1f82511b399b633ee9368b4d637401e5f53f43d0/disclosures/granite_4.1_30b_base_disclosure.json","kind":"model-card"},"card":{"url":"https://huggingface.co/ibm-granite/granite-4.1-30b-base/resolve/9bb22a92d17f4a51788c4750d32764ed068d94f2/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/ibm-granite/granite-4.1-30b-base/resolve/9bb22a92d17f4a51788c4750d32764ed068d94f2/config.json","kind":"config"},"modeling_granite":{"url":"https://raw.githubusercontent.com/huggingface/transformers/e0eb2810ac1b2caf6d082f247d8d37162ecf67a0/src/transformers/models/granite/modeling_granite.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019100630/https://artificialanalysis.ai/models/granite-4-1-30b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-29","source":"card","at":"README.md; Model Summary","quote":"Release Date: April 29th, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"README.md; opening summary","quote":"designed to support a broad range of text‑to‑text generation tasks"},"params.total":{"value":28865728512,"source":"disclosure","at":"granite_4.1_30b_base_disclosure.json; model.parameter_count","quote":"\"parameter_count\": 28865728512,","note":"Card rounds to 30B; tied embedding counted once."},"params.active":{"value":28865728512,"source":"disclosure","at":"granite_4.1_30b_base_disclosure.json; model.parameter_count","quote":"\"parameter_count\": 28865728512,","basis":"inferred","note":"Dense model; active equals the stated total."},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":64,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":true,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100352,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.global_layers":{"value":64,"source":"card","at":"README.md; Model Architecture table","quote":"Number of layers 40 40 64","note":"Dense transformer; every layer is full GQA."},"attn.head_dim":{"value":128,"source":"card","at":"README.md; Model Architecture table","quote":"Attention head size 64 128 128"},"attn.heads":{"value":32,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA","source":"card","at":"README.md; Model Architecture","quote":"Core components of this architecture are: GQA, RoPE, MLP with SwiGLU, RMSNorm, and shared input/output embeddings.","note":"Dense attention-only model, not the Mamba-2 hybrid used in Granite 4.0."},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"64 GQA","source":"card","at":"README.md; Model Architecture","quote":"Granite-4.1-30B-Base is based on a decoder-only dense transformer architecture."},"attn.position":{"value":"RoPE","source":"card","at":"README.md; Model Architecture","quote":"Core components of this architecture are: GQA, RoPE, MLP with SwiGLU, RMSNorm, and shared input/output embeddings."},"attn.qk_norm":{"value":false,"source":"modeling_granite","at":"modeling_granite.py; GraniteAttention.forward","quote":"query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)"},"attn.rope_theta":{"value":50000000,"source":"config","at":"/rope_theta","note":"This card does not state θ. The card of Granite-4.2-30B (post-trained from this base) says θ = 10,000,000, but its config.json, like this one, sets 50,000,000, which modeling_granite.py uses."},"attn.scale":{"value":"0.0078125 (1/head_dim) instead of 1/sqrt(head_dim)","source":"config","at":"config.json; attention_multiplier","quote":"\"attention_multiplier\": 0.0078125,","note":"modeling_granite.py sets self.scaling = config.attention_multiplier."},"ffn.activation":{"value":"SwiGLU","source":"card","at":"README.md; Model Architecture","quote":"Core components of this architecture are: GQA, RoPE, MLP with SwiGLU, RMSNorm, and shared input/output embeddings.","note":"Config hidden_act silu; GraniteMLP computes down_proj(act_fn(gate_proj(x)) * up_proj(x))."},"ffn.bias":{"value":false,"source":"config","at":"/mlp_bias"},"ffn.dim":{"value":32768,"source":"config","at":"/intermediate_size"},"norm.embedding":{"value":"× 12 (embedding_multiplier)","source":"config","at":"config.json; embedding_multiplier","quote":"\"embedding_multiplier\": 12.0,","note":"modeling_granite.py: inputs_embeds = inputs_embeds * self.embedding_multiplier."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.output_scale":{"value":"logits ÷ 16 (logits_scaling)","source":"config","at":"config.json; logits_scaling","quote":"\"logits_scaling\": 16.0,","note":"modeling_granite.py: logits = logits / self.config.logits_scaling."},"norm.placement":{"value":"pre","source":"modeling_granite","at":"modeling_granite.py; GraniteDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.residual":{"value":"branch outputs × 0.175 before the residual add (residual_multiplier)","source":"config","at":"config.json; residual_multiplier","quote":"\"residual_multiplier\": 0.175,","note":"modeling_granite.py: hidden_states = residual + hidden_states * self.residual_multiplier, after attention and after the MLP."},"norm.type":{"value":"RMSNorm","source":"card","at":"README.md; Model Architecture","quote":"Core components of this architecture are: GQA, RoPE, MLP with SwiGLU, RMSNorm, and shared input/output embeddings."},"ctx.extension":{"value":"Staged 4K → 32K → 128K → 512K (phase 5, 396B tokens); exponential LR decay from 1e-4 to 0; model merge after each stage","source":"blog","at":"Pre-Training; Phase 5: Long Context Training (LCE)","quote":"The LCE phase uses an exponential learning rate schedule starting at 1e-4 and decaying to 0","note":"32K and 128K stages reuse the phase-4 mix; the 512K stage uses 80% books + 20% code repositories (8B and 30B only)."},"ctx.final":{"value":524288,"source":"card","at":"README.md; Model Summary","quote":"long‑context extension is applied to expand the model’s context window to 512K tokens."},"ctx.max":{"value":524288,"source":"card","at":"README.md; Model Summary","quote":"long‑context extension is applied to expand the model’s context window to 512K tokens.","note":"Released config sets max_position_embeddings 131072 and no rope_scaling."},"ctx.pretrain":{"value":4096,"source":"blog","at":"Pre-Training; Phase 5: Long Context Training (LCE)","quote":"extends the context window from 4K to 512K"},"train.averaging":{"value":"model merge after each long-context stage","source":"blog","at":"Pre-Training; Phase 5: Long Context Training (LCE)","quote":"we do a model merge after each LCE stage"},"train.data":{"value":"Phase 1: CommonCrawl ~59%, code ~20%, technical ~10.5%, math ~7%, multilingual ~2%, domain-specific ~1.5%. Later phases raise math/code and add synthetic, long-CoT and instruction data.","source":"blog","at":"Pre-Training; Phase 1: General Pre-Training (10T tokens)","quote":"CommonCrawl ~59% — general web data"},"train.hardware":{"value":"NVIDIA GB200 NVL72 cluster (CoreWeave)","source":"card","at":"README.md; Infrastructure","quote":"We trained the Granite 4.1 Language Models utilizing an NVIDIA GB200 NVL72 cluster hosted in CoreWeave."},"train.accelerators":{"value":"GB200","source":"card","at":"README.md; Infrastructure","quote":"We trained the Granite 4.1 Language Models utilizing an NVIDIA GB200 NVL72 cluster hosted in CoreWeave.","note":"The card gives the count only as \"thousands of GPUs\"."},"train.objective":{"value":"FIM with prefix and suffix tokens","source":"card","at":"README.md; Model Summary","quote":"code completion through specialized prefix and suffix tokens"},"train.lr_min":{"value":0,"source":"blog","at":"Pre-Training; Phase 5: Long Context Training (LCE)","quote":"The LCE phase uses an exponential learning rate schedule starting at 1e-4 and decaying to 0","note":"Phase 4 also decays linearly to zero before LCE."},"train.schedule":{"value":"Warmup + power schedule (phases 1-2), exponential decay (phase 3), linear decay to zero (phase 4), exponential decay from 1e-4 to 0 (phase 5, long context)","source":"card","at":"README.md; Training Data table","quote":"General mixture of training data, warmup, and power scheduler for learning rate."},"train.stages":{"value":[{"name":"phase-1","tokens":10000000000000,"note":"General mixture; warmup and power LR schedule.","quote":"10 trillion tokens in phase one"},{"name":"phase-2","tokens":2000000000000,"note":"Higher code and math share; power LR schedule.","quote":"2 trillion tokens each in phases two and three"},{"name":"phase-3","tokens":2000000000000,"note":"High-quality data; exponential LR decay.","quote":"High quality training data, exponential decay of learning rate."},{"name":"phase-4","tokens":500000000000,"note":"High-quality data; linear LR decay to zero.","quote":"0.5 trillion tokens in phase four"},{"name":"phase-5-long-context","tokens":396000000000,"note":"Sub-stages at 32K, 128K and 512K; exponential LR decay from 1e-4 to 0. Tokens from the card's Training Data table (396B).","quote":"Long Context Extension with exponential learning rate schedule."}],"source":"card","at":"README.md; Model Summary and Training Data table"},"train.tokens":{"value":15000000000000,"source":"card","at":"README.md; Model Summary","quote":"The model is trained from scratch on approximately 15 trillion tokens using a five‑phase training strategy","note":"Approximate; phases sum to 14.896T (10T + 2T + 2T + 0.5T + 396B)."},"eval.aa_index":{"value":7,"source":"aa","at":"Artificial Analysis model page /models/granite-4-1-30b, Wayback snapshot 2026-10-19","quote":"Granite 4.1 30B scores 7 on the Artificial Analysis Intelligence Index","note":"Score of Granite-4.1-30B, post-trained from this base. Variant: Granite 4.1 30B. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","train.optimizer","train.lr_peak","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":28865728512,"formula":"embedding + output head + all experts + other weights, ibm-granite/granite-4.1-30b-base@9bb22a9 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_embedding":{"value":411041792,"formula":"input embedding + output head, ibm-granite/granite-4.1-30b-base@9bb22a9 safetensors headers"},"params.counted_non_embedding":{"value":28454686720,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, ibm-granite/granite-4.1-30b-base@9bb22a9 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_max_layer":{"value":444604416,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, ibm-granite/granite-4.1-30b-base@9bb22a9 safetensors headers"},"params.counted_active_non_embedding":{"value":28454686720.0,"formula":"all weights except embedding and output head, ibm-granite/granite-4.1-30b-base@9bb22a9 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"shape.aspect":{"value":64.0,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":4.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":8.0,"formula":"ffn.dim / d_model"},"scaling.tpp":{"value":519.6473733120656,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":527.1539324120171,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":262144.0,"formula":"262144 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":34359738368.0,"formula":"262144 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":262144.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":9081.496068634542,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1190329852.7080667,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":262144.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":262144.0,"tokens_per_entry":1}],"formula":"Global KV: 64 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"hy3-preview-base","name":"Hy3-preview-Base","org":"Tencent","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/tencent/Hy3-preview-Base/resolve/54a62bb00a50195423bffb6b55e91aa28b6a8ce2/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/tencent/Hy3-preview-Base/resolve/54a62bb00a50195423bffb6b55e91aa28b6a8ce2/config.json","kind":"config"},"index":{"url":"https://huggingface.co/tencent/Hy3-preview-Base/resolve/54a62bb00a50195423bffb6b55e91aa28b6a8ce2/model.safetensors.index.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/hy_v3/modeling_hy_v3.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019100827/https://artificialanalysis.ai/models/hy3-preview","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-23","source":"card","at":"README.md; News","quote":"[2026-04-23] 🔥 We open-source Hy3 preview model weights"},"meta.license":{"value":"other (Tencent Hy Community License Agreement)","source":"card","at":"README.md; License","quote":"Hy3 preview is released under the Tencent Hy Community License Agreement"},"meta.modality":{"value":"text","basis":"inferred","source":"config","at":"config.json architectures","quote":"HYV3ForCausalLM","note":"The card states no modality; the config class HYV3ForCausalLM has no vision or audio tower."},"params.active":{"value":21000000000,"source":"card","at":"README.md; Model Introduction","quote":"is a 295B-parameter Mixture-of-Experts (MoE) model with 21B active parameters and 3.8B MTP layer parameters"},"params.note":{"value":"Excludes the 3.8B MTP layer, which the card lists separately (checkpoint index total_size 597.6 GB in BF16 ≈ 298.8B).","source":"card","at":"README.md; Model Introduction","quote":"is a 295B-parameter Mixture-of-Experts (MoE) model with 21B active parameters and 3.8B MTP layer parameters"},"params.total":{"value":295000000000,"source":"card","at":"README.md; Model Introduction","quote":"is a 295B-parameter Mixture-of-Experts (MoE) model with 21B active parameters and 3.8B MTP layer parameters"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":80,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":120832,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"index","at":"/weight_map | has self_attn.q_proj.bias","note":"No QKV bias tensors; config omits attention_bias (transformers default false)."},"attn.global_layers":{"value":80,"source":"config","at":"/num_hidden_layers","note":"Every layer is full GQA; the config has no sliding-window or linear layers."},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA","source":"card","at":"README.md; Model Introduction table","quote":"Attention Heads 64 (GQA, 8 KV heads, head dim 128)"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.position":{"value":"RoPE","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3Attention.forward","quote":"query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)"},"attn.qk_norm":{"value":true,"source":"config","at":"/qk_norm"},"attn.rope_theta":{"value":11158840,"source":"config","at":"/rope_parameters/rope_theta"},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is config /hidden_act silu; routed experts compute act_fn(gate) * up."},"ffn.dim":{"value":13312,"source":"config","at":"/intermediate_size","note":"Dense FFN in layer 0 only."},"moe.dense_layers":{"value":1,"source":"config","at":"/first_k_dense_replace"},"moe.expert_dim":{"value":1536,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":192,"source":"config","at":"/num_experts"},"moe.router":{"value":"sigmoid","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3TopKRouter.forward","quote":"routing_weights = torch.sigmoid(router_logits)","note":"Config /moe_router_use_sigmoid true."},"moe.balancing":{"value":"expert bias added to selection scores (e_score_correction_bias); update rule and any aux loss not disclosed","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3TopKRouter.forward; config /moe_router_enable_expert_bias","quote":"scores_for_choice = routing_weights + e_score_correction_bias"},"moe.routed_scale":{"value":2.826,"source":"config","at":"/router_scaling_factor","note":"Applied to the renormalized top-k weights (modeling_hy_v3.py HYV3TopKRouter.forward)."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_hy_v3.py; HYV3TopKRouter.forward","quote":"top_k_weights = top_k_weights / (top_k_weights.sum(dim=-1, keepdim=True) + 1e-20)","note":"Config /route_norm true."},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.shared_dim":{"value":1536,"source":"hf-transformers","at":"modeling_hy_v3.py; HYV3MoE.__init__","quote":"shared_intermediate = config.moe_intermediate_size * config.num_shared_experts","basis":"inferred","note":"moe_intermediate_size 1536 × num_shared_experts 1 (config)."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"card","at":"README.md; Model Introduction table","quote":"Number of MTP Layers 1","note":"The pre-trained base checkpoint ships the trained MTP layer."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.params":{"value":3800000000,"source":"card","at":"README.md; Model Introduction","quote":"3.8B MTP layer parameters"},"mtp.block":{"value":"enorm(embedding) and hnorm(hidden) concatenated, eh_proj 2d→d, then one GQA (QK norm) + MoE block with final RMSNorm","source":"index","at":"model.safetensors.index.json; model.layers.80.* weights","quote":"\"model.layers.80.eh_proj.weight\"","basis":"inferred","note":"From the layer-80 tensor names (enorm, hnorm, eh_proj, q/k_norm, router + 192 experts + shared_mlp, final_layernorm); transformers modeling_hy_v3.py does not implement the MTP layer."},"mtp.ffn":{"value":"MoE","source":"index","at":"model.safetensors.index.json; model.layers.80.mlp","quote":"\"model.layers.80.mlp.router.gate.weight\"","note":"192 routed experts and a shared expert, as in the backbone."},"mtp.shares_embedding":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.80.eh_proj.weight\"","basis":"inferred","note":"No embed_tokens or shared_head tensors under model.layers.80; the module reuses model.embed_tokens and lm_head."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.80.eh_proj.weight\"","note":"Stored as layer 80."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_hy_v3.py; HYV3DecoderLayer.__init__","quote":"self.input_layernorm = HYV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.max":{"value":262144,"source":"card","at":"README.md; Model Introduction table; config /max_position_embeddings","quote":"Context Length 256K"},"eval.aa_index":{"value":23,"source":"aa","at":"Artificial Analysis model page /models/hy3-preview, Wayback snapshot 2026-10-19","quote":"Hy3-preview (Reasoning) scores 23 on the Artificial Analysis Intelligence Index","note":"Score of Hy3-preview, post-trained from this base. Variant: Hy3-preview (Reasoning). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.loss_weight","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":295033543488,"formula":"embedding + output head + all experts + other weights, tencent/Hy3-preview-Base@54a62bb safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":989855744,"formula":"input embedding + output head, tencent/Hy3-preview-Base@54a62bb safetensors headers"},"params.counted_non_embedding":{"value":294043687744,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, tencent/Hy3-preview-Base@54a62bb safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":3719045568,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, tencent/Hy3-preview-Base@54a62bb safetensors headers"},"params.counted_active_non_embedding":{"value":19685874496.0,"formula":"non-expert weights + routed experts × 8/192, excluding embedding and output head, tencent/Hy3-preview-Base@54a62bb safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":51.2,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":3.25,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.375,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":24.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.0711864406779661,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":327680.0,"formula":"327680 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":42949672960.0,"formula":"327680 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":327680.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":15603.809523809523,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":2045222521.9047618,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":327680.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":327680.0,"tokens_per_entry":1}],"formula":"Global KV: 80 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"hy4-preview","name":"Hy4-preview","org":"Tencent","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/tencent/Hy4-preview/resolve/705d81ee51566a186d645b74c974d642ef2828fe/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/tencent/Hy4-preview/resolve/705d81ee51566a186d645b74c974d642ef2828fe/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/hy_v4/modeling_hy_v4.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20260926093529/https://www.tencent.com/tencent-releases-and-open-sources-tencent-hy4-preview/","kind":"blog"}},"facts":{"meta.released":{"value":"2026-08-28","source":"blog","at":"tencent.com; article header","quote":"Tencent Releases and Open-Sources Tencent Hy4 preview August 28, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower (HYV4ForCausalLM)."},"params.total":{"value":770000000000,"source":"card","at":"README.md; Model Introduction","quote":"The model comprises 770B total parameters, of which 49B are activated per token."},"params.active":{"value":49000000000,"source":"card","at":"README.md; Model Introduction","quote":"The model comprises 770B total parameters, of which 49B are activated per token."},"params.note":{"value":"Backbone only, excluding the MTP layer (10B total, 0.7B active).","source":"card","at":"README.md; Model Specifications","quote":"The table below lists backbone parameters only, excluding the MTP layer."},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":78,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":120832,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":78,"source":"config","at":"/layer_types | count deepseek_sparse_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/qk_head_dim","note":"qk_nope_head_dim 192 + qk_rope_head_dim 64. /head_dim=64 only sizes the rotary embedding."},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA, DSA","source":"config","at":"config.json; use_dsa, use_mla","quote":"\"use_dsa\": true, \"use_mla\": true"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"78 DSA","source":"config","at":"config.json; layer_types","quote":"\"layer_types\": [ \"deepseek_sparse_attention\", \"deepseek_sparse_attention\",","basis":"inferred","note":"Uniform: /layer_types | count deepseek_sparse_attention = 78 = /num_hidden_layers."},"attn.qk_norm":{"value":false,"source":"hf-transformers","at":"modeling_hy_v4.py; HYV4Attention.forward","quote":"k_pass = self.kv_a_layernorm(kv_pass).view(batch_size, 1, seq_length, self.kv_lora_rank)","note":"RMSNorm only on the Q and KV latents; no per-head Q/K norm. The DSA indexer LayerNorms its key."},"attn.indexer_dim":{"value":128,"source":"config","at":"/index_head_dim","note":"One indexer key head (HYV4Indexer.wk outputs index_head_dim); only the 21 'full' indexer layers cache it (see attn.kv_groups)."},"attn.kv_groups":{"value":[{"name":"MLA latent + RoPE key","layers":78,"elements":576,"tokens_per_entry":1,"note":"kv_lora_rank 512 + qk_rope_head_dim 64.","quote":"k_pass, k_rot = past_key_values.update(k_pass, k_rot, self.layer_idx)"},{"name":"DSA indexer key","layers":21,"elements":128,"tokens_per_entry":1,"note":"Only layers with indexer_types 'full' own an indexer; 'shared' layers reuse the previous top-k.","quote":"self.indexer = None if self.skip_topk else HYV4Indexer(config, layer_idx)"}],"source":"hf-transformers","at":"modeling_hy_v4.py; HYV4Attention.__init__, HYV4Attention.forward"},"attn.q_lora_rank":{"value":2048,"source":"config","at":"/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.position":{"value":"partial RoPE (MLA decoupled, 64 of 256 QK dims)","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4Attention.forward","quote":"q_pass, q_rot = torch.split(q_states, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)"},"attn.gate":{"value":"elementwise sigmoid gate computed from the block input, before o_proj","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4Attention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate_states)","note":"Config /gated_mla true, /gating_type elementwise."},"attn.sparse":{"value":"DSA indexer (32 heads × 128, top-2048); IndexCache: 21 layers run the indexer, 57 reuse the previous indexer layer's top-k","source":"card","at":"README.md; Model Introduction","quote":"(Gated DSA) with [IndexCache](https://arxiv.org/abs/2603.12201) for cross-layer sparse index reuse","note":"Indexer dims from the card table (config /index_n_heads, /index_head_dim, /index_topk); layer split from config /indexer_types (21 full, 57 shared)."},"attn.v_head_dim":{"value":256,"source":"config","at":"/v_head_dim"},"attn.sinks":{"value":"learned per-head sink logit, all 78 layers","source":"config","at":"config.json; learnable_sink","quote":"\"learnable_sink\": true, \"learnable_sink_init\": 0.0","note":"modeling_hy_v4.py HYV4Attention holds one sink per head and concatenates it to the attention logits before softmax."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is config /hidden_act silu; routed experts compute F.silu(gate) * up."},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"ffn.clamp":{"value":"SwiGLU limit 10 (gate ≤ 10, up in [-10, 10]), routed experts only","source":"config","at":"config.json; swiglu_limit","quote":"\"swiglu_limit\": 10.0","note":"Clamp applied in modeling_hy_v4.py HYV4Experts._apply_gate; HYV4MLP has none."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_hy_v4.py; HYV4MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Routed experts (HYV4Experts) hold only gate_up_proj and down_proj weights."},"moe.dense_layers":{"value":1,"source":"config","at":"/mlp_layer_types | count dense"},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"hf-transformers","at":"modeling_hy_v4.py; HYV4MoE.__init__","quote":"intermediate_size=config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"moe_intermediate_size 2048 × n_shared_experts 1 (config)."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.balancing":{"value":"expert bias added to selection scores (e_score_correction_bias); update rule and any aux loss not disclosed","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4TopkRouter.forward","quote":"scores_for_choice = scores + self.e_score_correction_bias"},"moe.routed_scale":{"value":2.827,"source":"config","at":"/routed_scaling_factor"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.router":{"value":"sigmoid","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4TopkRouter.forward","quote":"scores = router_logits.sigmoid()"},"mtp.released":{"value":true,"source":"card","at":"README.md; Model Introduction","quote":"1 native MTP layer (10B total parameters, 0.7B activated) is built in for speculative decoding.","note":"Config /num_nextn_predict_layers 1; config mtp_loss_factor 0.1 is not read by modeling_hy_v4.py, and the training stage of the MTP layer is not stated."},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)","note":"Pre-norm on the hyper-connection-mixed input (see norm.residual)."},"norm.residual":{"value":"iHC (identity Hyper-Connections), 4 streams: per sublayer, input-dependent sigmoid read weights and 2·sigmoid write weights, identity stream mixing (no Sinkhorn); streams merged by a sigmoid-weighted sum before the final norm","source":"card","at":"README.md; Model Specifications","quote":"Residual Streams | 4","note":"Card: 'The residual pathway uses iHC (identity Hyper-Connections)'. Details from modeling_hy_v4.py HYV4HyperConnection / HYV4HyperHead and config /enable_ihc, /hc_mult 4, /hc_magnitude 2.0, /hc_eps 1e-6 (added to the sigmoid weights)."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.type":{"value":"RMSNorm","source":"hf-transformers","at":"modeling_hy_v4.py; HYV4DecoderLayer.__init__","quote":"self.input_layernorm = HYV4RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.max":{"value":1048576,"source":"config","at":"/max_position_embeddings","note":"Card table: Context Length 1M."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":769907408484,"formula":"embedding + output head + all experts + other weights, tencent/Hy4-preview@705d81e safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1484783616,"formula":"input embedding + output head, tencent/Hy4-preview@705d81e safetensors headers"},"params.counted_non_embedding":{"value":768422624868,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, tencent/Hy4-preview@705d81e safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":9978461264,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, tencent/Hy4-preview@705d81e safetensors headers"},"params.counted_active_non_embedding":{"value":47572762212.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, tencent/Hy4-preview@705d81e safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":78.76923076923077,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"ffn.mult":{"value":3.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.06363636363636363,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":95232.0,"formula":"89856 B / 1 tokens (MLA latent + RoPE key) + 5376 B / 1 tokens (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":12482248704.0,"formula":"89856 B × 131072 (MLA latent + RoPE key) + 5376 B × 131072 (DSA indexer key); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":95232.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1943.5102040816328,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":254739769.46938777,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":95232.0,"windows":[],"groups":[{"name":"MLA latent + RoPE key","bytes_per_entry":89856.0,"tokens_per_entry":1},{"name":"DSA indexer key","bytes_per_entry":5376.0,"tokens_per_entry":1}],"formula":"MLA latent + RoPE key: 78 × (576 × 16/8) B/entry; DSA indexer key: 21 × (128 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"inkling-small","name":"Inkling-Small","org":"Thinking Machines Lab","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"announcement":{"url":"https://web.archive.org/web/20261009101710/https://thinkingmachines.ai/news/inkling-small/","kind":"blog"},"card":{"url":"https://huggingface.co/thinkingmachines/Inkling-Small/resolve/8cc5877b44d343f88b92086aa1fb72897950f06a/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/thinkingmachines/Inkling-Small/resolve/8cc5877b44d343f88b92086aa1fb72897950f06a/config.json","kind":"config"},"index":{"url":"https://huggingface.co/thinkingmachines/Inkling-Small/resolve/8cc5877b44d343f88b92086aa1fb72897950f06a/model.safetensors.index.json","kind":"config"},"configuration_inkling":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/inkling/configuration_inkling.py","kind":"code"},"transformers-code":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/inkling/modeling_inkling.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019100954/https://artificialanalysis.ai/models/inkling-small","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-30","source":"announcement","at":"article header","quote":"Introducing Inkling-Small Jul 30, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + audio","source":"card","at":"§1 General Information","quote":"Inkling-Small is a general-purpose multimodal model that accepts text, image and audio inputs and generates text outputs."},"params.total":{"value":276000000000,"source":"card","at":"README.md; §3 Model Properties, Parameters","quote":"276B total, 12B active"},"params.active":{"value":12000000000,"source":"card","at":"README.md; §3 Model Properties, Parameters","quote":"276B total, 12B active"},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":42,"source":"config","at":"/text_config/num_hidden_layers"},"shape.vocab":{"value":201024,"source":"config","at":"/text_config/vocab_size","note":"/text_config/unpadded_vocab_size is 200058."},"shape.tied_embeddings":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingForConditionalGeneration","quote":"are separate tensors in the checkpoints, never tied"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; §3 Model Properties, Architecture type","quote":"Attention is a hybrid of local and global layers."},"attn.layout":{"value":"5 SWA : 1 global","source":"config","at":"config.json; /text_config/local_layer_ids","quote":"\"local_layer_ids\": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40 ]","basis":"inferred","note":"Config /text_config/local_layer_ids omits 5, 11, 17, 23, 29, 35, 41: global at every 6th layer. /text_config/local_layer_ids gives zero-based SWA indices; other backbone layers use global attention.","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":7,"source":"card","at":"README.md; §3 Model Properties, Architecture type","quote":"A 42-layer decoder-only transformer","basis":"inferred","note":"42 layers minus the 35 SWA layers in /text_config/local_layer_ids."},"attn.local_layers":{"value":35,"source":"config","at":"/text_config/local_layer_ids | len"},"attn.heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads","note":"Global layers; SWA layers read /text_config/swa_num_key_value_heads (InklingAttention)."},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.local_heads":{"value":32,"source":"config","at":"/text_config/swa_num_attention_heads","note":"InklingAttention reads the swa_* head counts and head dim for hybrid_sliding layers, the global ones otherwise."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/text_config/swa_num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"config","at":"/text_config/swa_head_dim"},"attn.window":{"value":512,"source":"config","at":"/text_config/sliding_window_size"},"attn.position":{"value":"relative position bias (learned, hidden-state-conditioned, all layers)","source":"transformers-code","at":"modeling_inkling.py; InklingAttention.forward","quote":"position_bias = self.rel_logits_proj(relative_states, q_positions, kv_positions)","note":"No rotary embedding. InklingRelativeLogits: hidden-state-conditioned bias (d_rel 16), zero beyond rel_extent (1024 on global layers, /text_config/rel_extent; 512 on SWA)."},"attn.qk_norm":{"value":true,"source":"transformers-code","at":"modeling_inkling.py; InklingAttention.__init__","quote":"self.q_norm = InklingRMSNorm(self.head_dim, eps=config.rms_norm_eps)"},"attn.scale":{"value":"1/head_dim; global layers scale queries and position bias by 1 + 0.1·ln(max((pos+1)/128000, 1))","source":"transformers-code","at":"modeling_inkling.py; InklingAttention.forward","quote":"tau = 1.0 + self.config.log_scaling_alpha * torch.log(","note":"self.scaling = 1.0 / self.head_dim (q/k are RMS-normalized); /text_config/log_scaling_alpha 0.1, /text_config/log_scaling_n_floor 128000."},"attn.bias":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingAttention.__init__","quote":"self.k_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)"},"attn.conv":{"value":4,"source":"config","at":"/text_config/sconv_kernel_size","note":"Causal depthwise convolutions on the K and V projections and on the attention and MLP block outputs (InklingShortConvolution; /text_config/use_sconv)."},"ffn.activation":{"value":"SwiGLU","source":"configuration_inkling","at":"configuration_inkling.py; InklingTextConfig","quote":"hidden_act: str = \"silu\"","note":"config.json does not set hidden_act, so the silu default applies; modeling_inkling.py computes act_fn(gate) * up in dense, shared and routed experts."},"ffn.dim":{"value":16384,"source":"config","at":"/text_config/dense_intermediate_size"},"ffn.bias":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.dense_layers":{"value":2,"source":"config","at":"/text_config/dense_mlp_idx","note":"Layers i < dense_mlp_idx are dense (configuration_inkling.py __post_init__)."},"moe.expert_dim":{"value":2048,"source":"config","at":"/text_config/intermediate_size","note":"With dense_intermediate_size set, intermediate_size is the routed-expert width (configuration_inkling.py __post_init__)."},"moe.experts":{"value":256,"source":"config","at":"/text_config/n_routed_experts"},"moe.shared":{"value":2,"source":"config","at":"/text_config/n_shared_experts"},"moe.shared_dim":{"value":4096,"source":"transformers-code","at":"modeling_inkling.py; InklingSharedExperts.__init__","quote":"intermediate_dim = config.moe_intermediate_size","basis":"inferred","note":"2 shared experts × 2048."},"moe.top_k":{"value":6,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.router":{"value":"sigmoid","source":"transformers-code","at":"modeling_inkling.py; InklingTopkRouter.forward","quote":"scores = router_logits.sigmoid()","note":"Config /text_config/gate_activation is not read by the modeling code."},"moe.norm_topk":{"value":true,"source":"transformers-code","at":"modeling_inkling.py; InklingTopkRouter.forward","quote":"topk_weights = torch.exp(topk_log_probs - torch.logsumexp(topk_log_probs, dim=-1, keepdim=True))","note":"Sigmoid scores of the 6 selected routed experts and the 2 shared experts are normalized jointly (shared_expert_sink, /text_config/norm_after_topk)."},"moe.routed_scale":{"value":8.0,"source":"config","at":"/text_config/route_scale","note":"Also multiplied by a learned scalar (global_scale, /text_config/use_global_scale); applies to the shared-expert weights as well."},"moe.balancing":{"value":"expert-score correction bias on top-k selection","source":"transformers-code","at":"modeling_inkling.py; InklingTopkRouter.forward","quote":"scores_for_choice = routed_scores + self.e_score_correction_bias","note":"Bias (/text_config/use_gate_bias) affects selection only; its update rule and any auxiliary loss are not disclosed."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.7.input_proj.weight\"","note":"8 MTP layers (/mtp_config/num_nextn_predict_layers); their use in pretraining is not disclosed."},"norm.type":{"value":"RMSNorm","source":"transformers-code","at":"modeling_inkling.py; InklingDecoderLayer.__init__","quote":"self.input_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"transformers-code","at":"modeling_inkling.py; InklingDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)","note":"Short convolutions on attention and MLP outputs before the residual add."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.embedding":{"value":"RMSNorm after embedding","source":"transformers-code","at":"modeling_inkling.py; InklingNormedEmbedding.__init__; config /text_config/use_embed_norm","quote":"self.embed_norm = InklingRMSNorm(embedding_dim, eps=norm_eps)"},"norm.output_scale":{"value":"final hidden state ÷ 16 before the LM head","source":"transformers-code","at":"modeling_inkling.py; InklingForConditionalGeneration.forward","quote":"hidden_states = outputs[0] / self.config.text_config.logits_mup_width_multiplier","note":"/text_config/logits_mup_width_multiplier is 16.0."},"ctx.max":{"value":1048576,"source":"announcement","at":"Introduction","quote":"a context window of up to 1M tokens","note":"Config /text_config/model_max_length is 1048576."},"train.data":{"value":"text, images, audio, video","source":"card","at":"README.md; §4 Training","quote":"Training data includes a broad variety of content types, including text, images, audio, video.","note":"The announcement says the pre-training data mix and recipe were changed from Inkling's."},"train.hardware":{"value":"NVIDIA GB300 NVL72","source":"announcement","at":"Introduction","quote":"trained on NVIDIA GB300 NVL72 systems"},"train.accelerators":{"value":"GB300","source":"announcement","at":"Introduction","quote":"trained on NVIDIA GB300 NVL72 systems","note":"Count not stated."},"scaling.parametrization":{"value":"μP (final hidden state ÷ 16 width multiplier before the LM head)","source":"configuration_inkling","at":"configuration_inkling.py; InklingTextConfig docstring, logits_mup_width_multiplier","quote":"muP width multiplier the final hidden states are divided by before the language modeling head.","note":"/text_config/logits_mup_width_multiplier is 16.0."},"eval.aa_index":{"value":26,"source":"aa","at":"Artificial Analysis model page /models/inkling-small, Wayback snapshot 2026-10-19","quote":"Inkling Small scores 26 on the Artificial Analysis Intelligence Index","note":"Variant: Inkling Small. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":263655199232,"formula":"embedding + output head + all experts + other weights, thinkingmachines/Inkling-Small@8cc5877 safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"params.counted_embedding":{"value":1646788608,"formula":"input embedding + output head, thinkingmachines/Inkling-Small@8cc5877 safetensors headers"},"params.counted_non_embedding":{"value":262008410624,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, thinkingmachines/Inkling-Small@8cc5877 safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"params.counted_max_layer":{"value":6537945600,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, thinkingmachines/Inkling-Small@8cc5877 safetensors headers"},"params.counted_active_non_embedding":{"value":10350170624.0,"formula":"non-expert weights + routed experts × 6/256, excluding embedding and output head, thinkingmachines/Inkling-Small@8cc5877 safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"shape.aspect":{"value":97.52380952380952,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":4.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":5.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":42.666666666666664,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.043478260869565216,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":28672.0,"formula":"28672 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3831496704.0,"formula":"28672 B × 131072 (Global KV) + 143360 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":29232.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2436.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":319291392.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":28672.0,"windows":[[143360.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":28672.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":143360.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 7 × (2048 × 16/8) B/entry; SWA KV: 35 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"inkling","name":"Inkling","org":"Thinking Machines Lab","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"announcement":{"url":"https://web.archive.org/web/20261007195943/https://thinkingmachines.ai/news/introducing-inkling/","kind":"blog"},"config":{"url":"https://huggingface.co/thinkingmachines/Inkling/resolve/828496eeae4c243ff1a22f7f28ff83694f2f7bc9/config.json","kind":"config"},"configuration_inkling":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/inkling/configuration_inkling.py","kind":"code"},"transformers-code":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/inkling/modeling_inkling.py","kind":"code"},"weight_index":{"url":"https://huggingface.co/thinkingmachines/Inkling/resolve/828496eeae4c243ff1a22f7f28ff83694f2f7bc9/model.safetensors.index.json","kind":"config"},"card":{"url":"https://huggingface.co/thinkingmachines/Inkling/resolve/828496eeae4c243ff1a22f7f28ff83694f2f7bc9/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019101130/https://artificialanalysis.ai/models/inkling","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-15","source":"announcement","at":"article header","quote":"Inkling: Our Open-Weights Model Jul 15, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + audio","source":"card","at":"§1 General Information","quote":"Inkling is a general-purpose multimodal model that accepts text, image and audio inputs and generates text outputs.","note":"§3 says images and video share the patch encoder, but the card's input modalities list only text, image and audio."},"params.total":{"value":975000000000,"source":"announcement","at":"Introduction","quote":"a Mixture-of-Experts transformer with 975B total parameters, 41B active"},"params.active":{"value":41000000000,"source":"announcement","at":"Introduction","quote":"a Mixture-of-Experts transformer with 975B total parameters, 41B active"},"shape.d_model":{"value":6144,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":66,"source":"config","at":"/text_config/num_hidden_layers"},"shape.vocab":{"value":201024,"source":"config","at":"/text_config/vocab_size","note":"/text_config/unpadded_vocab_size is 200058."},"shape.tied_embeddings":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingForCausalLM","quote":"`embed` and `unembed` are separate tensors in the checkpoints, never tied"},"attn.kind":{"value":"GQA, SWA","source":"announcement","at":"The making of Inkling; Architecture","quote":"we interleave sliding-window and global layers at a 5:1 ratio with 8 KV heads"},"attn.layout":{"value":"5 SWA : 1 global","source":"config","at":"config.json; /text_config/local_layer_ids","quote":"\"local_layer_ids\": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 60, 61, 62, 63, 64 ]","note":"Config /text_config/local_layer_ids omits 5, 11, ..., 65: global at every 6th layer. /text_config/local_layer_ids gives zero-based SWA indices; other backbone layers use global attention.","basis":"inferred","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":11,"source":"configuration_inkling","at":"configuration_inkling.py; InklingTextConfig.__post_init__","quote":"\"hybrid_sliding\" if i in local_layer_ids else \"hybrid\" for i in range(self.num_hidden_layers)","basis":"inferred","note":"66 layers (/text_config/num_hidden_layers) minus 55 /text_config/local_layer_ids; the rest are full-attention \"hybrid\" layers."},"attn.local_layers":{"value":55,"source":"config","at":"/text_config/local_layer_ids | len"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads","note":"Global layers only; SWA layers use /text_config/swa_num_key_value_heads (InklingAttention)."},"attn.local_kv_heads":{"value":16,"source":"config","at":"/text_config/swa_num_key_value_heads"},"attn.local_heads":{"value":64,"source":"config","at":"/text_config/swa_num_attention_heads","note":"InklingAttention reads the swa_* head counts and head dim for hybrid_sliding layers, the global ones otherwise."},"attn.local_head_dim":{"value":128,"source":"config","at":"/text_config/swa_head_dim"},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.window":{"value":512,"source":"config","at":"/text_config/sliding_window_size"},"attn.position":{"value":"relative position bias (hidden-state-conditioned, all layers)","source":"announcement","at":"The making of Inkling; Architecture","quote":"We find that encoding position with a relative positional embedding","note":"No rotary embedding. InklingRelativeLogits: per-head d_rel=16 feature from r_proj, bias zero beyond rel_extent (1024 on global layers, /text_config/rel_extent; 512 on SWA)."},"attn.scale":{"value":"1/head_dim; global layers also multiply q and the position bias by 1 + 0.1·ln(n/128000) at query position n > 128000","source":"transformers-code","at":"modeling_inkling.py; InklingAttention.forward","quote":"tau = 1.0 + self.config.log_scaling_alpha * torch.log( (effective_n / self.config.log_scaling_n_floor).clamp(min=1.0)","note":"log_scaling_alpha 0.1 and log_scaling_n_floor 128000 from /text_config; base scaling is self.scaling = 1.0 / self.head_dim."},"attn.bias":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingAttention.__init__","quote":"self.k_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)","note":"q_proj, v_proj and o_proj are also bias-free."},"attn.conv":{"value":4,"source":"config","at":"/text_config/sconv_kernel_size","note":"Depthwise causal convs after k_proj and v_proj in every attention layer, and on the attention and MLP branch outputs (InklingShortConvolution; /text_config/use_sconv)."},"attn.qk_norm":{"value":true,"source":"transformers-code","at":"modeling_inkling.py; InklingAttention.__init__","quote":"self.q_norm = InklingRMSNorm(self.head_dim, eps=config.rms_norm_eps)"},"ffn.activation":{"value":"SwiGLU","source":"transformers-code","at":"modeling_inkling.py; InklingMLP.forward","quote":"hidden_states = self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))","note":"config.json omits hidden_act, so the InklingTextConfig default silu applies."},"ffn.dim":{"value":24576,"source":"config","at":"/text_config/dense_intermediate_size"},"ffn.bias":{"value":false,"source":"transformers-code","at":"modeling_inkling.py; InklingMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Routed and shared expert weights are bias-free 3D parameters."},"moe.dense_layers":{"value":2,"source":"config","at":"/text_config/dense_mlp_idx","note":"Layers i < dense_mlp_idx are dense (configuration_inkling.py __post_init__)."},"moe.expert_dim":{"value":3072,"source":"config","at":"/text_config/intermediate_size","note":"With dense_intermediate_size set, intermediate_size is the routed-expert width (configuration_inkling.py __post_init__)."},"moe.experts":{"value":256,"source":"config","at":"/text_config/n_routed_experts"},"moe.shared":{"value":2,"source":"config","at":"/text_config/n_shared_experts"},"moe.shared_dim":{"value":6144,"source":"transformers-code","at":"modeling_inkling.py; InklingSharedExperts.__init__","quote":"intermediate_dim = config.moe_intermediate_size","basis":"inferred","note":"2 shared experts (/text_config/n_shared_experts) × 3072 each."},"moe.top_k":{"value":6,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.router":{"value":"sigmoid","source":"transformers-code","at":"modeling_inkling.py; InklingTopkRouter.forward","quote":"scores = router_logits.sigmoid()","note":"Config /text_config/gate_activation is not read by the modeling code."},"moe.balancing":{"value":"auxiliary-loss-free bias","source":"announcement","at":"The making of Inkling; Architecture","quote":"Inkling uses a sigmoid-based router with an auxiliary-loss-free load-balancing bias.","note":"Bias e_score_correction_bias (/text_config/use_gate_bias) affects routed-expert selection only; selected routed and shared expert scores are normalized jointly (/text_config/shared_expert_sink)."},"moe.norm_topk":{"value":true,"source":"announcement","at":"The making of Inkling; Architecture; config /text_config/norm_after_topk","quote":"The scores of the selected routed experts and the shared experts are normalized jointly and used to weight their combined outputs.","note":"Normalized over the 6 routed plus 2 shared experts together."},"moe.routed_scale":{"value":8.0,"source":"config","at":"/text_config/route_scale","note":"Multiplies the jointly normalized routed and shared weights, together with a learned scalar global_scale (InklingTopkRouter.forward; /text_config/use_global_scale)."},"mtp.released":{"value":true,"source":"weight_index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.7.input_proj.weight\": \"mtp.safetensors\"","note":"8 MTP layers (/mtp_config/num_nextn_predict_layers) in mtp.safetensors; their use in pretraining is not stated."},"norm.placement":{"value":"pre","source":"transformers-code","at":"modeling_inkling.py; InklingDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"transformers-code","at":"modeling_inkling.py; InklingDecoderLayer.__init__","quote":"self.input_layernorm = InklingRMSNorm(config.hidden_size, config.rms_norm_eps)"},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.residual":{"value":"attention and MLP branch outputs y become y + conv(y) (depthwise causal conv, kernel 4) before the residual add","source":"announcement","at":"The making of Inkling; Architecture","quote":"on the attention and MLP residual branch outputs before they rejoin the main residual stream"},"norm.embedding":{"value":"RMSNorm after embedding lookup","source":"transformers-code","at":"modeling_inkling.py; InklingNormedEmbedding.forward; config /text_config/use_embed_norm","quote":"return self.embed_norm(super().forward(input_ids))"},"norm.output_scale":{"value":"final hidden states ÷ 24 before the LM head","source":"transformers-code","at":"modeling_inkling.py; InklingForConditionalGeneration.forward","quote":"hidden_states = outputs[0] / self.config.text_config.logits_mup_width_multiplier","note":"/text_config/logits_mup_width_multiplier is 24.0."},"ctx.max":{"value":1048576,"source":"config","at":"/text_config/model_max_length","note":"Announcement: up to 1M tokens."},"train.tokens":{"value":45000000000000,"source":"announcement","at":"The making of Inkling; Training","quote":"Inkling was pretrained on 45 trillion tokens from a variety of content types, including text, images, audio and video.","note":"Multimodal token count."},"train.optimizer":{"value":"Muon + Adam (Muon for large matrix weights)","source":"announcement","at":"The making of Inkling; Training","quote":"Muon for large matrix weights, Adam for other parameters","note":"Weight decay coupled to the square of the learning rate."},"train.data":{"value":"text, images, audio and video","source":"announcement","at":"The making of Inkling; Training","quote":"Inkling was pretrained on 45 trillion tokens from a variety of content types, including text, images, audio and video."},"train.hardware":{"value":"NVIDIA GB300 NVL72","source":"announcement","at":"The making of Inkling; Training","quote":"was trained on NVIDIA GB300 NVL72 systems"},"train.accelerators":{"value":"GB300","source":"announcement","at":"The making of Inkling; Training","quote":"was trained on NVIDIA GB300 NVL72 systems","note":"Count not stated."},"scaling.parametrization":{"value":"μP (final hidden state ÷ 24 width multiplier before the LM head)","source":"configuration_inkling","at":"configuration_inkling.py; InklingTextConfig docstring, logits_mup_width_multiplier","quote":"muP width multiplier the final hidden states are divided by before the language modeling head.","note":"/text_config/logits_mup_width_multiplier is 24.0."},"eval.aa_index":{"value":25,"source":"aa","at":"Artificial Analysis model page /models/inkling, Wayback snapshot 2026-10-19","quote":"Inkling (Xhigh) scores 25 on the Artificial Analysis Intelligence Index","note":"Variant: Inkling (Xhigh). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":947025564160,"formula":"embedding + output head + all experts + other weights, thinkingmachines/Inkling@828496e safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"params.counted_embedding":{"value":2470182912,"formula":"input embedding + output head, thinkingmachines/Inkling@828496e safetensors headers"},"params.counted_non_embedding":{"value":944555381248,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, thinkingmachines/Inkling@828496e safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"params.counted_max_layer":{"value":14742553088,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, thinkingmachines/Inkling@828496e safetensors headers"},"params.counted_active_non_embedding":{"value":38585717248.0,"formula":"non-expert weights + routed experts × 6/256, excluding embedding and output head, thinkingmachines/Inkling@828496e safetensors headers; reproduces stated total (MTP+other modalities) and active (with embedding and output head)"},"shape.aspect":{"value":93.0909090909091,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":5.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":42.666666666666664,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04205128205128205,"formula":"params.active / params.total"},"scaling.tpp":{"value":46.15384615384615,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1166.234638344904,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":45056.0,"formula":"45056 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6136266752.0,"formula":"45056 B × 131072 (Global KV) + 450560 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":46816.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1141.8536585365853,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":149665042.7317073,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":45056.0,"windows":[[450560.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":45056.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":450560.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 11 × (2048 × 16/8) B/entry; SWA KV: 55 × (4096 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"k-exaone-2","name":"K-EXAONE 2.0","org":"LG AI Research","country":"South Korea","kind":"reasoning","weights":"open","note":"Post-trained checkpoint (SFT and preference learning); pretraining facts are from the technical report.","sources":{"card":{"url":"https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/resolve/fec0c8dd25fdd20ffd09869d5af5277966d5523d/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/resolve/fec0c8dd25fdd20ffd09869d5af5277966d5523d/config.json","kind":"config"},"weights-index":{"url":"https://huggingface.co/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B/resolve/fec0c8dd25fdd20ffd09869d5af5277966d5523d/model.safetensors.index.json","kind":"config"},"modeling":{"url":"https://raw.githubusercontent.com/nuxlear/transformers/969f8bdfc73d7d8bb0df1b426e057d80a1adf501/src/transformers/models/exaone_moe/modeling_exaone_moe.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2608.04505v1","kind":"paper"},"news":{"url":"https://web.archive.org/web/20261011182321/https://www.lgresearch.ai/news/view?seq=678","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019101312/https://artificialanalysis.ai/models/k-exaone-2-0-0803","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-31","source":"news","at":"LG AI Research news, 2026.07.31","quote":"on the open-source platform Hugging Face on July 31","note":"News post dated 2026.07.31."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower (ExaoneMoeForCausalLM)."},"params.active":{"value":37000000000,"source":"report","at":"p.2 §1","quote":"The resulting model contains 750B total parameters-more than three times the total parameter count of its predecessor-with approximately 37B parameters activated per token."},"params.total":{"value":750000000000,"source":"report","at":"p.2 §1","quote":"The resulting model contains 750B total parameters-more than three times the total parameter count of its predecessor-with approximately 37B parameters activated per token."},"params.note":{"value":"Main-block count; the MTP block adds 0.52B and the separately trained DSpark drafter 2.53B","source":"report","at":"p.4 Table 1","quote":"MTP Block","note":"Table 1 lists 0.52B for the MTP block and 2.53B for the 5-layer DSpark block."},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":78,"source":"config","at":"/num_hidden_layers"},"shape.tokenizer":{"value":"inherited unchanged from K-EXAONE","source":"report","at":"p.4 §2.2","quote":"The number of activated experts (top-8 plus one shared expert), the sliding-window size, the routing policy, and the tokenizer are inherited unchanged."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":153600,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"weights-index","at":"/weight_map | has q_proj.bias"},"attn.global_layers":{"value":20,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.3 §2.1","quote":"The hybrid attention architecture, QK Norm, and SWA-only RoPE are retained"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"1 global + 1 SWA (4096 window) + 19 × (3 SWA : 1 global)","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Depth expansion repeated whole LLLG blocks, 12 → 19 (report p.4 §2.2). Ordered 78-layer backbone from /layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["GQA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.local_layers":{"value":58,"source":"config","at":"/layer_types | count sliding_attention"},"attn.position":{"value":"RoPE (SWA), NoPE (global)","source":"report","at":"p.3 §2.1","quote":"The hybrid attention architecture, QK Norm, and SWA-only RoPE are retained","note":"Card lists the first layer as '1 x Global (NoPE)'."},"attn.qk_norm":{"value":true,"source":"modeling","at":"modeling_exaone_moe.py; ExaoneMoeAttention.__init__","quote":"self.q_norm = ExaoneMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K, applied before RoPE; per-layer q_norm/k_norm weights are in the weight map. Report p.3 §2.1 confirms QK Norm."},"attn.local_rope_theta":{"value":1000000,"source":"config","at":"/rope_parameters/rope_theta","note":"Only the sliding-window layers apply RoPE; global layers are NoPE."},"attn.local_heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"ExaoneMoeAttention uses the same num_attention_heads, num_key_value_heads and head_dim for both layer types; only the window and RoPE differ."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.window":{"value":128,"source":"config","at":"/sliding_windows/2","note":"Layer 1 uses a 4096 window (attn.long_window); every other SWA layer uses 128."},"attn.long_window":{"value":4096,"source":"config","at":"/sliding_windows/1"},"attn.long_window_layers":{"value":1,"source":"config","at":"/sliding_windows | count 4096"},"ffn.activation":{"value":"SwiGLU","source":"modeling","at":"modeling_exaone_moe.py; ExaoneMoeMLP.forward","quote":"return self.down_proj(self.act_fn(gate) * up)","note":"act_fn is SiLU (/hidden_act)."},"ffn.bias":{"value":false,"source":"weights-index","at":"/weight_map | has up_proj.bias"},"ffn.clamp":{"value":"Clamped SwiGLU, limit 7.0 on the last 16 MoE layers: gate clamped above, up clamped both sides","source":"report","at":"p.4 Figure 2 caption","quote":"the last 16 layers of the main model apply Clamped SwiGLU with a limit value of 7.0 to the experts","note":"Config /swiglu_limits is 7.0 on layers 62-77; released code clamps gate (max 7) and up (±7) before the product in routed and shared experts of those layers."},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"moe.balancing":{"value":"sequence-level load balancing plus DeepSeek-style per-expert score bias; dropless","source":"report","at":"p.3 §2.1","quote":"employing sigmoid-based scoring, sequence-level load balancing, and a dropless routing policy","note":"Per-expert bias from p.4 §2.2: 'K-EXAONE inherits the DeepSeek-style router with per-expert score biases'. Coefficients not disclosed."},"moe.dense_layers":{"value":2,"source":"config","at":"/mlp_layer_types | count dense"},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/num_shared_experts) of the routed-expert width."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"config","at":"/num_nextn_predict_layers","note":"Report p.4 §2.3: the MTP head is trained jointly with the backbone from pre-training onward."},"mtp.modules":{"value":1,"source":"config","at":"/mtp_layer_types | len"},"mtp.block":{"value":"RMSNorm on embedding and on previous hidden state, concatenated and projected (fc 12288 → 6144); one SWA layer (window 128, 64/8 heads, QK norm) with dense SwiGLU FFN; final RMSNorm; shared LM head","source":"weights-index","at":"model.safetensors.index.json; mtp.* keys","quote":"\"mtp.pre_fc_norm_embedding.weight\"","note":"Keys mtp.pre_fc_norm_hidden, mtp.fc, mtp.layers.0.{self_attn,mlp}, mtp.norm; config mtp_layer_types sliding_attention, mtp_sliding_windows 128; widths from the shard headers."},"mtp.ffn":{"value":"dense","source":"weights-index","at":"model.safetensors.index.json; mtp.layers.0.mlp","quote":"\"mtp.layers.0.mlp.gate_proj.weight\"","note":"No expert or router tensors in the MTP layer; gate_proj is 18432 × 6144 (same width as ffn.dim)."},"mtp.shares_embedding":{"value":true,"source":"report","at":"p.3 Figure 2","quote":"Embedding (Shared)","note":"Figure 2 also labels the MTP 'LM Head (Shared)'; the weight map has no MTP embedding or head."},"mtp.params":{"value":520000000,"source":"report","at":"p.4 Table 1; MTP Block","quote":"Parameters 0.52B 0.52B"},"mtp.released":{"value":true,"source":"weights-index","at":"/weight_map | has mtp."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling","at":"modeling_exaone_moe.py; ExaoneMoeDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling","at":"modeling_exaone_moe.py; ExaoneMoeRMSNorm.forward","quote":"return self.weight * hidden_states.to(input_dtype)"},"ctx.extension":{"value":"mid-training in two stages: 8K → 64K (400B tokens) → 256K (400B tokens)","source":"report","at":"p.7 §4","quote":"In the first stage (Mid-Stage 1), the model is trained on 400B tokens with a 64K context window, followed by training on an additional 400B tokens with context windows exceeding 64K tokens in the second stage (Mid-Stage 2)."},"ctx.final":{"value":262144,"source":"report","at":"p.7 §4","quote":"progressively extending the context window from 8K to 64K and then to 256K tokens"},"ctx.max":{"value":262144,"source":"config","at":"/max_position_embeddings"},"ctx.pretrain":{"value":8192,"source":"report","at":"p.7 §4","quote":"progressively extending the context window from 8K to 64K and then to 256K tokens","note":"Stated as the context mid-training starts from."},"train.cutoff":{"value":"2025 2Q","source":"card","at":"README.md; Model Configurations; Knowledge Cutoff","quote":"2025 2Q"},"train.data":{"value":"K-EXAONE mixture plus Active Reading synthetic data, institution-sourced Korean culture and history corpora, and FineWeb2 for four added languages","source":"report","at":"p.5 §3","quote":"We also extend the data recipe with two new synthetic data generation methods to improve knowledge acquisition and reduce training loss.","note":"Second method extends thinking-augmented pre-training with latent intermediate reasoning."},"train.init":{"value":"upcycled from K-EXAONE: depth 48 → 78 layers (12 → 19 LLLG blocks, repeating blocks from the middle of the stack); width 128 → 256 experts (each expert duplicated with its router row, plus norm-preserving random rotation noise)","source":"report","at":"p.4 §2.2","quote":"K-EXAONE 2.0 is upcycled from K-EXAONE by expanding the model along both the depth axis (48 → 78 layers) and the width axis (128 → 256 experts), followed by continued pre-training.","note":"Block and noise details from the Depth/Width up-scaling paragraphs (p.4 §2.2); per-expert dimension unchanged."},"train.stability":{"value":"Clamped SwiGLU (limit 7.0) on the last 16 layers","source":"report","at":"p.4 §2.2","quote":"bounding the gate branch from above and the linear branch on both sides at a threshold of 7.0 and leaving the routing decision untouched","note":"With the clamp disabled at inference, peak SwiGLU activation reaches 6862 at the last layer; the clamp holds it at 48.96 (p.5 Figure 3)."},"train.stages":{"value":[{"name":"healing","quote":"To maintain training stability following this expansion, we first perform a healing stage using a portion of the later-stage K-EXAONE training data."},{"name":"continual pre-training","tokens":8000000000000,"quote":"After healing, the model is further trained on an additional 8T tokens using the K-EXAONE 2.0 pre-training data mixture."},{"name":"mid-stage 1","tokens":400000000000,"seq_len":65536,"quote":"In the first stage (Mid-Stage 1), the model is trained on 400B tokens with a 64K context window"},{"name":"mid-stage 2","tokens":400000000000,"seq_len":262144,"quote":"followed by training on an additional 400B tokens with context windows exceeding 64K tokens in the second stage (Mid-Stage 2).","note":"Context windows above 64K, up to 256K."}],"source":"report","at":"p.4-7 §2.2, §3, §4","note":"Starts from upcycled K-EXAONE weights (train.init); the report says depth up-scaling is applied during the early stage of pre-training (p.5 §3). Healing-stage token count is not disclosed; mid-training extends the context from 8K (p.7 §4)."},"train.tokens":{"value":8000000000000,"source":"report","at":"p.5 §3","quote":"the model is further trained on an additional 8T tokens using the K-EXAONE 2.0 pre-training data mixture","note":"Continual pre-training after upcycling; excludes the K-EXAONE parent run, the healing stage and the 800B mid-training tokens."},"scaling.ladder":{"value":"small-scale K-EXAONE model (size not disclosed) for mid-training stage ablations","source":"report","at":"p.8 §4.2","quote":"We evaluate the contribution of Mid-Stage 1 and Mid-Stage 2 using the small-scale model."},"eval.aa_index":{"value":20,"source":"aa","at":"Artificial Analysis model page /models/k-exaone-2-0-0803, Wayback snapshot 2026-10-19","quote":"K-EXAONE 2.0 0803 scores 20 on the Artificial Analysis Intelligence Index","note":"Variant: K-EXAONE 2.0 0803. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.loss_weight","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":748828971520,"formula":"embedding + output head + all experts + other weights, LGAI-EXAONE/K-EXAONE-2.0-750B-A37B@fec0c8d safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"params.counted_embedding":{"value":1887436800,"formula":"input embedding + output head, LGAI-EXAONE/K-EXAONE-2.0-750B-A37B@fec0c8d safetensors headers"},"params.counted_non_embedding":{"value":746941534720,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, LGAI-EXAONE/K-EXAONE-2.0-750B-A37B@fec0c8d safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"params.counted_max_layer":{"value":9816257024,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, LGAI-EXAONE/K-EXAONE-2.0-750B-A37B@fec0c8d safetensors headers"},"params.counted_active_non_embedding":{"value":35453358592.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, LGAI-EXAONE/K-EXAONE-2.0-750B-A37B@fec0c8d safetensors headers; reproduces stated total (MTP) and active (with embedding and output head)"},"shape.aspect":{"value":78.76923076923077,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":3.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":2.9,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04933333333333333,"formula":"params.active / params.total"},"scaling.tpp":{"value":10.666666666666666,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":225.64857936492336,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":81920.0,"formula":"81920 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":10784079872.0,"formula":"81920 B × 131072 (Global KV) + 233472 B × min(131072, 128) (SWA KV) + 4096 B × min(131072, 4096) (Long-window KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":82276.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2223.675675675676,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":291461618.1621622,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":81920.0,"windows":[[233472.0,128],[4096.0,4096]],"groups":[{"name":"Global KV","bytes_per_entry":81920.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":233472.0,"tokens_per_entry":1,"window":128},{"name":"Long-window KV","bytes_per_entry":4096.0,"tokens_per_entry":1,"window":4096}],"formula":"Global KV: 20 × (2048 × 16/8) B/entry; SWA KV: 57 × (2048 × 16/8) B/entry; Long-window KV: 1 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"k2-horizon-mova","name":"K2-Horizon-MoVA-36B-A4B (mid_4_10000)","org":"MBZUAI IFM","country":"United Arab Emirates","kind":"base","weights":"open","note":"Describes tag mid_4_10000, the final Midtraining Stage 4 checkpoint (before SFT). Stage 4 trains on a mix shifted toward agentic and reasoning SFT data, so this is not a pure pretraining checkpoint (that is pretrain_1100000, at 8K context).","sources":{"config":{"url":"https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B/resolve/35c37ffac1422cdc9efffa6f9da1136951e9cda0/config.json","kind":"config"},"modeling_k2_horizon":{"url":"https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B/resolve/35c37ffac1422cdc9efffa6f9da1136951e9cda0/modeling_k2_horizon.py","kind":"code"},"card":{"url":"https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B/resolve/e9ce717642b2074082875be233c02c729ba47960/README.md","kind":"model-card"},"blog":{"url":"http://web.archive.org/web/20260903154432/https://ifm.ai/blog/k2/","kind":"blog"},"xllm_transformer":{"url":"https://raw.githubusercontent.com/ifm-ai/xllm/3578359ad320e4d18efaca071854ba2019201f31/xllm/models/transformer.py","kind":"code"},"xllm_launch":{"url":"https://raw.githubusercontent.com/ifm-ai/xllm/3578359ad320e4d18efaca071854ba2019201f31/examples/k2-horizon-mova-36B-a4B.sh","kind":"code"},"xllm_config":{"url":"https://raw.githubusercontent.com/ifm-ai/xllm/3578359ad320e4d18efaca071854ba2019201f31/xllm/config.py","kind":"code"},"xllm_optimizer":{"url":"https://raw.githubusercontent.com/ifm-ai/xllm/3578359ad320e4d18efaca071854ba2019201f31/xllm/optim/optimizer.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019134332/https://artificialanalysis.ai/models/k2-horizon-mova-36b-a4b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-09-03","source":"blog","at":"ifm.ai/blog/k2 (Wayback snapshot 2026-09-03); header","quote":"September 3, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower (K2HorizonForCausalLM)."},"params.total":{"value":36000000000,"source":"card","at":"README.md; introduction","quote":"a Mixture-of-Experts model with Mixture-of-Values attention (MoVA) that stores 36B parameters and runs 4B per token"},"params.active":{"value":4000000000,"source":"card","at":"README.md; introduction","quote":"a Mixture-of-Experts model with Mixture-of-Values attention (MoVA) that stores 36B parameters and runs 4B per token"},"params.note":{"value":"Scope of the 36B and 4B counts is not stated. xLLM's num_parameters formula on this config gives 36.2B total and 4.7B active excluding the embedding and output head (1.28B together).","source":"xllm_transformer","at":"xllm/models/transformer.py; num_parameters","quote":"activated_value_params = self.model_dim * self.num_kv_heads * self.head_dim * self.num_activated_values","basis":"inferred","note":"Per MoVA+MoE layer: 34.1M Q/K/O/gate + 64 × 2.62M value experts (4 active) + 100 × 5.9M routed experts (8 active) + 5.9M shared + routers; × 45 layers, plus 3 dense layers of 83.9M."},"shape.d_model":{"value":2560,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.vocab":{"value":250624,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"attn.kind":{"value":"GQA (MoVA: routed value projections in 45 of 48 layers)","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonMoVAAttention","quote":"MoVA attention with routed value experts and optional post-attention gate.","note":"MoVA (Mixture-of-Value Attention) replaces v_proj with 64 value experts (/mova_num_experts), each a 2560→8×128 linear map; a sigmoid router picks 4 per token (/mova_num_experts_per_tok), and the SiLU'd expert outputs are summed with renormalized weights × 2.5. Q and K are ordinary GQA, every token attends to all earlier tokens, and the mixed V is cached like a normal V, so the KV cache is the standard 8 × (128 + 128) per layer. sliding_window null, use_sliding_window false."},"attn.layout":{"value":"3 GQA, then 45 GQA with MoVA values","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonDecoderLayer.__init__","quote":"if is_sparse_layer and config.mova_num_experts > 0:","note":"MoVA layers are exactly the MoE-FFN layers; layers 0-2 (/mlp_only_layers) use a plain v_proj and a dense FFN."},"attn.global_layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"attn.heads":{"value":32,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.position":{"value":"RoPE","source":"config","at":"config.json; rope_head_dim, head_dim","quote":"\"rope_head_dim\": 128","note":"rope_head_dim equals head_dim, so all 128 QK dims rotate (unlike the 375B sibling's 64)."},"attn.rope_fraction":{"value":1.0,"source":"config","at":"config.json; rope_head_dim, head_dim","quote":"\"rope_head_dim\": 128","basis":"inferred","note":"rope_head_dim 128 divided by head_dim 128."},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.qk_norm":{"value":false,"source":"config","at":"/query_key_norm"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.gate":{"value":"softplus (β = ln 2) gate per head dimension on the attention output, before o_proj, in all layers","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonAttention.forward and K2HorizonMoVAAttention.forward","quote":"gate = F.softplus(gate, beta=math.log(2))","note":"Gate is gate_proj(x), 2560 → 32 × 128, selected by /attention_gate_func."},"ffn.activation":{"value":"SwiGLU","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (/hidden_act); dense FFN, routed and shared experts all use K2HorizonMLP."},"ffn.bias":{"value":false,"source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":6144,"source":"config","at":"/intermediate_size"},"moe.experts":{"value":100,"source":"config","at":"/num_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.expert_dim":{"value":768,"source":"config","at":"/moe_intermediate_size"},"moe.shared_dim":{"value":768,"source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonSparseMoeBlock.__init__","quote":"intermediate_size=config.moe_intermediate_size * config.num_shared_experts)","note":"768 × 1 shared expert."},"moe.dense_layers":{"value":3,"source":"config","at":"/mlp_only_layers | len","note":"Layers 0, 1, 2 use the 6144-wide dense FFN; decoder_sparse_step 1 makes every later layer MoE."},"moe.router":{"value":"sigmoid","source":"config","at":"/router_score_func"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/router_scaling_factor"},"moe.balancing":{"value":"per-expert bias for top-k selection only (update rate 1e-3) plus 'dot' auxiliary loss, coefficient 1e-4","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; TRAINING_CFGS","quote":"--moe_aux_loss_coeff 1e-4","note":"Bias update rate from ConfStore[\"k2-horizon-mova-36B-a4B\"] (moe_router_bias_update_rate=1e-3; HF config moe_gate_bias, router_aux_loss_coef 1e-4). The MoVA value routers use the same router settings; xLLM averages their aux loss with the FFN router's."},"norm.type":{"value":"RMSNorm (2 groups of 1280)","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonRMSNorm.forward","quote":"hidden_states = hidden_states.reshape(*hidden_states.shape[:-1], self.n_groups, -1)","note":"layernorm_num_groups 2: each half of d_model is normalized separately, in block and final norms."},"norm.placement":{"value":"pre","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.pretrain":{"value":8192,"source":"card","at":"README.md; Training Overview table","quote":"Pretraining | 1100000 | 22.9T | 8K | Pretraining."},"ctx.final":{"value":524288,"source":"card","at":"README.md; Training Overview table","quote":"Midtraining — Stage 3 | 5500 | 110B | 512K | Context extension.","note":"512K = 524,288 tokens (Highlights)."},"ctx.max":{"value":524288,"source":"card","at":"README.md; Highlights","quote":"Native 524,288-token context from the midtraining stages onward.","note":"config max_position_embeddings 524288."},"ctx.extension":{"value":"8K → 32K (1.1T) → 128K (498B) → 512K (110B, then 199B with agentic/reasoning SFT data)","source":"card","at":"README.md; Training Overview table","quote":"Midtraining — Stage 1 | 55000 | 1.1T | 32K | Context extension.","note":"RoPE base raised with length: config rope_theta 500000 at pretrain_1100000 (commit 3cc4264), 1e6 at mid_1_55000 (632ec2f), 1e7 from mid_2_25000 (358bc8f); rope_type default, no scaling."},"mtp.depth":{"value":0,"source":"xllm_transformer","at":"xllm/models/transformer.py; TransformerOutputLayer.forward","quote":"loss = vocab_parallel_cross_entropy(logits, y)","basis":"inferred","note":"xLLM, the training code, has no MTP implementation; the loss is next-token cross-entropy, and model.safetensors.index.json has no MTP weights."},"train.tokens":{"value":24807000000000,"source":"card","at":"README.md; Training Overview","quote":"Token budgets cover only the additional training in that stage or phase.","basis":"inferred","note":"Sum of the per-stage budgets in train.stages (22.9T + 1.1T + 498B + 110B + 199B) through mid_4_10000; includes the 199B Stage 4 tokens with SFT data in the mix."},"train.stages":{"value":[{"name":"pretraining","tokens":22900000000000,"seq_len":8192,"steps":1100000,"quote":"Pretraining | 1100000 | 22.9T | 8K | Pretraining."},{"name":"midtraining-1","tokens":1100000000000,"seq_len":32768,"steps":55000,"note":"Context extension.","quote":"Midtraining — Stage 1 | 55000 | 1.1T | 32K | Context extension."},{"name":"midtraining-2","tokens":498000000000,"seq_len":131072,"steps":25000,"note":"Context extension.","quote":"Midtraining — Stage 2 | 25000 | 498B | 128K | Context extension."},{"name":"midtraining-3","tokens":110000000000,"seq_len":524288,"steps":5500,"note":"Context extension.","quote":"Midtraining — Stage 3 | 5500 | 110B | 512K | Context extension."},{"name":"midtraining-4","tokens":199000000000,"seq_len":524288,"steps":10000,"note":"Mix shifted toward agentic and reasoning SFT data; final checkpoint is tag mid_4_10000.","quote":"Midtraining — Stage 4 | 10000 | 199B | 512K | Continued context extension from Stage 3, with the data mix shifted toward agentic and reasoning SFT data."}],"source":"card","at":"README.md; Training Overview table"},"train.data":{"value":"Same pretraining tokens as K2 Horizon 3.7B, 7B and 32B; Midtraining Stage 4 shifts the mix toward agentic and reasoning SFT data","source":"blog","at":"ifm.ai/blog/k2; Pre-training Dynamics","quote":"and 36B-A4B were trained on exactly the same 22 trillion tokens.","note":"The blog says 22 trillion; the card's pretraining stage says 22.9T."},"train.optimizer":{"value":"AdamW","source":"xllm_optimizer","at":"xllm/optim/optimizer.py; build_optimizer","quote":"optimizer = AdamW("},"train.lr_peak":{"value":0.0004,"source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; variables","quote":"LR=4e-4","note":"IFM's launch script for this model; its --steps 1100000 matches the card's pretraining stage."},"train.schedule":{"value":"WSD with no decay phase in pretraining: linear warmup, then constant 4e-4 to step 1,100,000","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; OPTIM_CFGS","quote":"--optim.wsd_decay_duration_steps 0","note":"xLLM lr_wsd holds the peak through total_steps when the decay duration is 0; the LR used in midtraining is not in these sources."},"train.warmup":{"value":"3,000 steps, linear from 1e-4 × peak","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; OPTIM_CFGS","quote":"--optim.warmup 3000"},"train.batch":{"value":"16 × 8,192 per data-parallel rank; global batch not stated","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; variables and TRAINING_CFGS","quote":"BSZ=16","note":"xLLM batch_size is per DP rank. The script's 38 nodes × 8 GPUs with model_parallel_size 2 give 152 ranks (~19.9M tokens/step), but 22.9T / 1.1M steps is ~20.8M, so the global batch is left unstated."},"train.weight_decay":{"value":0.06,"source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; variables","quote":"WD=0.06"},"train.betas":{"value":"(0.9, 0.95)","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; OPTIM_CFGS","quote":"--optim.beta1 0.9 --optim.beta2 0.95"},"train.grad_clip":{"value":1.0,"source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; OPTIM_CFGS","quote":"--optim.clip 1.0"},"train.precision":{"value":"BF16","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; TRAINING_CFGS","quote":"--dtype \"bf16\""},"train.init":{"value":"Gaussian, std 0.02","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; MODEL_CFGS","quote":"--model.init_mode \"gaussian\" --model.init_std 0.02","note":"Matches config initializer_range 0.02, which by itself is a loader value."},"train.masking":{"value":"no attention across document boundaries (BOS-delimited segments)","source":"xllm_config","at":"xllm/config.py; TrainerConf.multi_segments","quote":"This will prevent attention across document boundaries.","note":"The launch script sets --multi_segments true."},"train.parallelism":{"value":"TP2 · FSDP","source":"xllm_launch","at":"examples/k2-horizon-mova-36B-a4B.sh; DISTRIBUTED_CFGS","quote":"--model_parallel_size 2","note":"xllm/config.py defines model_parallel_size as model/tensor parallel; the script sets --context_parallel_size 1 and --model.ddp_backend fsdp2."},"eval.aa_index":{"value":25,"source":"aa","at":"Artificial Analysis model page /models/k2-horizon-mova-36b-a4b, Wayback snapshot 2026-10-19","quote":"K2 Horizon MoVA 36B A4B scores 25 on the Artificial Analysis Intelligence Index","note":"Score of K2-Horizon-MoVA-36B-A4B (SFT checkpoint sft_2_2500 on main, from the final Midtraining Stage 4 checkpoint), post-trained from this base. Variant: K2 Horizon MoVA 36B A4B. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.lr_min"],"derived":{"params.counted_total":{"value":null,"formula":"checkpoint count (37.4B backbone) does not reproduce the stated total and active counts"},"params.counted_embedding":{"value":null,"formula":"checkpoint count (37.4B backbone) does not reproduce the stated total and active counts"},"params.counted_non_embedding":{"value":null,"formula":"checkpoint count (37.4B backbone) does not reproduce the stated total and active counts"},"params.counted_active_non_embedding":{"value":null,"formula":"checkpoint count (37.4B backbone) does not reproduce the stated total and active counts"},"shape.aspect":{"value":53.333333333333336,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":4.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.4,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":12.5,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.1111111111111111,"formula":"params.active / params.total"},"scaling.tpp":{"value":689.0833333333334,"formula":"train.tokens / params.total"},"kv.growth_per_token":{"value":196608.0,"formula":"196608 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":25769803776.0,"formula":"196608 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":196608.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":49152.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":6442450944.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":196608.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":196608.0,"tokens_per_entry":1}],"formula":"Global KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"k2-horizon","name":"K2-Horizon-375B-A23B (mid_4_6000)","org":"MBZUAI IFM","country":"United Arab Emirates","kind":"base","weights":"open","note":"Describes tag mid_4_6000, the final Midtraining Stage 4 checkpoint (before RL and SFT). Stage 4 trains on a mix shifted toward agentic and reasoning SFT data, so this is not a pure pretraining checkpoint (that is pretrain_ph2_240000, at 8K context).","sources":{"card":{"url":"https://huggingface.co/IFM/K2-Horizon-375B-A23B/resolve/78272deca7adc2f90d154288b5161172283c80b3/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/IFM/K2-Horizon-375B-A23B/resolve/7d22cc0ac26a91003d19482efa71070244239b11/config.json","kind":"config"},"modeling_k2_horizon":{"url":"https://huggingface.co/IFM/K2-Horizon-375B-A23B/resolve/7d22cc0ac26a91003d19482efa71070244239b11/modeling_k2_horizon.py","kind":"code"},"blog":{"url":"http://web.archive.org/web/20260903154432/https://ifm.ai/blog/k2/","kind":"blog"},"xllm_transformer":{"url":"https://raw.githubusercontent.com/ifm-ai/xllm/3578359ad320e4d18efaca071854ba2019201f31/xllm/models/transformer.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019101438/https://artificialanalysis.ai/models/k2-horizon-375b-a23b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-09-03","source":"blog","at":"ifm.ai/blog/k2 (Wayback snapshot 2026-09-03); header","quote":"September 3, 2026"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower (K2HorizonForCausalLM)."},"params.total":{"value":375000000000,"source":"card","at":"README.md; introduction","quote":"a sparse Mixture-of-Experts model that stores 375B parameters and runs 23B per token"},"params.active":{"value":23000000000,"source":"card","at":"README.md; introduction","quote":"a sparse Mixture-of-Experts model that stores 375B parameters and runs 23B per token"},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.vocab":{"value":250624,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"attn.kind":{"value":"GQA","source":"config","at":"config.json; num_attention_heads, num_key_value_heads","quote":"\"num_key_value_heads\": 8","note":"48 query heads share 8 KV heads; sliding_window null, no MoVA experts (/mova_num_experts 0)."},"attn.layout":{"value":"61 GQA","source":"config","at":"config.json; num_hidden_layers, use_sliding_window","quote":"\"use_sliding_window\": false"},"attn.global_layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.position":{"value":"partial RoPE (64 of 128 QK dims)","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonAttention.forward","quote":"split_size_or_sections=[self.rope_head_dim, self.head_dim - self.rope_head_dim],","note":"rope_head_dim is config /rope_head_dim (64); the remaining 64 dims per head carry no position."},"attn.rope_fraction":{"value":0.5,"source":"config","at":"config.json; rope_head_dim, head_dim","quote":"\"rope_head_dim\": 64","basis":"inferred","note":"rope_head_dim 64 divided by head_dim 128."},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.qk_norm":{"value":false,"source":"config","at":"/query_key_norm"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"ffn.activation":{"value":"SwiGLU","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (/hidden_act); dense FFN, routed and shared experts all use K2HorizonMLP."},"ffn.bias":{"value":false,"source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":16384,"source":"config","at":"/intermediate_size"},"moe.experts":{"value":192,"source":"config","at":"/num_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.expert_dim":{"value":1792,"source":"config","at":"/moe_intermediate_size"},"moe.shared_dim":{"value":1792,"source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonSparseMoeBlock.__init__","quote":"intermediate_size=config.moe_intermediate_size * config.num_shared_experts)","note":"1792 × 1 shared expert."},"moe.dense_layers":{"value":3,"source":"config","at":"/mlp_only_layers | len","note":"Layers 0, 1, 2 use the 16384-wide dense FFN; decoder_sparse_step 1 makes every later layer MoE."},"moe.router":{"value":"sigmoid","source":"config","at":"/router_score_func"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/router_scaling_factor"},"moe.balancing":{"value":"per-expert bias for top-k selection only (update rate 1e-3)","source":"xllm_transformer","at":"xllm/models/transformer.py; ConfStore[\"k2-horizon-375B-a23B\"]","quote":"num_experts=192, num_activated_experts=8, num_shared_experts=1, expert_inter_dim=1792, moe_router_score_func='sigmoid', moe_router_bias=True, moe_router_bias_update_rate=1e-3,","note":"HF code adds gate.bias (config moe_gate_bias) to scores for selection only; whether the 375B run also used an auxiliary loss is not stated (config router_aux_loss_coef 1e-4 is a loader value)."},"norm.type":{"value":"RMSNorm","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonDecoderLayer.__init__","quote":"self.input_layernorm = K2HorizonRMSNorm(","note":"layernorm_num_groups 1: one normalization group over d_model."},"norm.placement":{"value":"pre","source":"modeling_k2_horizon","at":"modeling_k2_horizon.py; K2HorizonDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.pretrain":{"value":8192,"source":"card","at":"README.md; Training Overview table","quote":"Pretraining — Phase 1 | 211000 | 7.08T | 8K"},"ctx.final":{"value":524288,"source":"card","at":"README.md; Training Overview table","quote":"Midtraining — Stage 3 | 3500 | 117B | 512K | Context extension.","note":"512K = 524,288 tokens (Highlights)."},"ctx.max":{"value":524288,"source":"card","at":"README.md; Highlights","quote":"Native 524,288-token context from the midtraining stages onward.","note":"config max_position_embeddings 524288."},"ctx.extension":{"value":"8K → 32K (1.09T) → 128K (503B) → 512K (117B, then 201B with agentic/reasoning SFT data)","source":"card","at":"README.md; Training Overview table","quote":"Midtraining — Stage 1 | 32500 | 1.09T | 32K | Context extension.","note":"RoPE base raised with length: config rope_theta 500000 at branch pretrain_ph2_240000, 1e6 at mid_1_32500, 1e7 from mid_2_15000; no RoPE scaling (rope_type default)."},"mtp.depth":{"value":0,"source":"xllm_transformer","at":"xllm/models/transformer.py; TransformerOutputLayer.forward","quote":"loss = vocab_parallel_cross_entropy(logits, y)","basis":"inferred","note":"xLLM, the training code, has no MTP implementation; the loss is next-token cross-entropy, and the checkpoint has no MTP weights."},"train.batch":{"value":"~33.5M tokens","source":"card","at":"README.md; Training Overview table","quote":"Pretraining — Phase 2 | 240000 | 8.05T | 8K","basis":"inferred","note":"8.05T / 240,000 steps (Phase 1: 7.08T / 211,000 steps gives the same); sequences per batch and any ramp not stated."},"train.batch_tokens":{"value":33500000,"source":"card","at":"README.md; Training Overview table","quote":"Pretraining — Phase 2 | 240000 | 8.05T | 8K","basis":"inferred","note":"8.05T tokens / 240,000 steps, to the card's 3-significant-figure precision."},"train.tokens":{"value":17041000000000,"source":"card","at":"README.md; Training Overview","quote":"Token budgets cover only the additional training in that stage or phase.","basis":"inferred","note":"Sum of the per-stage budgets quoted in train.stages (7.08T + 8.05T + 1.09T + 503B + 117B + 201B) through mid_4_6000; includes the 201B Stage 4 tokens with SFT data in the mix."},"train.data":{"value":"Phase 2 continues on newer and better data; Midtraining Stage 4 shifts the mix toward agentic and reasoning SFT data","source":"card","at":"README.md; Training Overview table","quote":"Continued context extension from Stage 3, with the data mix shifted toward agentic and reasoning SFT data."},"train.stages":{"value":[{"name":"pretraining-phase-1","tokens":7080000000000,"seq_len":8192,"steps":211000,"quote":"Pretraining — Phase 1 | 211000 | 7.08T | 8K | Pretraining."},{"name":"pretraining-phase-2","tokens":8050000000000,"seq_len":8192,"steps":240000,"note":"Continued from Phase 1 with newer and better data.","quote":"Pretraining — Phase 2 | 240000 | 8.05T | 8K | Continued pretraining from Phase 1, with newer and better data."},{"name":"midtraining-1","tokens":1090000000000,"seq_len":32768,"steps":32500,"note":"Context extension.","quote":"Midtraining — Stage 1 | 32500 | 1.09T | 32K | Context extension."},{"name":"midtraining-2","tokens":503000000000,"seq_len":131072,"steps":15000,"note":"Context extension.","quote":"Midtraining — Stage 2 | 15000 | 503B | 128K | Context extension."},{"name":"midtraining-3","tokens":117000000000,"seq_len":524288,"steps":3500,"note":"Context extension.","quote":"Midtraining — Stage 3 | 3500 | 117B | 512K | Context extension."},{"name":"midtraining-4","tokens":201000000000,"seq_len":524288,"steps":6000,"note":"Mix shifted toward agentic and reasoning SFT data; final checkpoint is tag mid_4_6000.","quote":"Midtraining — Stage 4 | 6000 | 201B | 512K | Continued context extension from Stage 3, with the data mix shifted toward agentic and reasoning SFT data."}],"source":"card","at":"README.md; Training Overview table"},"eval.aa_index":{"value":31,"source":"aa","at":"Artificial Analysis model page /models/k2-horizon-375b-a23b, Wayback snapshot 2026-10-19","quote":"K2 Horizon 375B A23B scores 31 on the Artificial Analysis Intelligence Index","note":"Score of K2-Horizon-375B-A23B (RL and SFT from the final Midtraining Stage 4 checkpoint), post-trained from this base. Variant: K2 Horizon 375B A23B. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":379167159168,"formula":"embedding + output head + all experts + other weights, IFM/K2-Horizon-375B-A23B@7d22cc0 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":3079667712,"formula":"input embedding + output head, IFM/K2-Horizon-375B-A23B@7d22cc0 safetensors headers"},"params.counted_non_embedding":{"value":376087491456,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, IFM/K2-Horizon-375B-A23B@7d22cc0 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":6464090304,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, IFM/K2-Horizon-375B-A23B@7d22cc0 safetensors headers"},"params.counted_active_non_embedding":{"value":23589794688.0,"formula":"non-expert weights + routed experts × 8/192, excluding embedding and output head, IFM/K2-Horizon-375B-A23B@7d22cc0 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":100.72131147540983,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.6666666666666665,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2916666666666667,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":24.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.06133333333333333,"formula":"params.active / params.total"},"scaling.tpp":{"value":45.44266666666667,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":722.3886526095398,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":249856.0,"formula":"249856 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":32749125632.0,"formula":"249856 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":249856.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":10863.304347826086,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1423875027.4782608,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":249856.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":249856.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"kimi-k2-5","name":"Kimi-K2.5","org":"Moonshot AI","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/4d01dfe0332d63057c186e0b262165819efb6611/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/4d01dfe0332d63057c186e0b262165819efb6611/config.json","kind":"config"},"modeling_deepseek":{"url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/4d01dfe0332d63057c186e0b262165819efb6611/modeling_deepseek.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2602.02276v2","kind":"paper"},"weight-index":{"url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/4d01dfe0332d63057c186e0b262165819efb6611/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20260817080036/https://www.kimi.com/blog/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019101612/https://artificialanalysis.ai/models/kimi-k2-5","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-01-27","source":"blog","at":"kimi.com blog index (Wayback 2026-08-17); Kimi K2.5 entry","quote":"Kimi K2.5 2026-01-27"},"meta.license":{"value":"other (modified-mit)","source":"card","at":"README.md; YAML front matter","quote":"license_name: modified-mit"},"meta.modality":{"value":"text + image + video","source":"card","at":"§6 Model Usage; Chat Completion with visual content","quote":"K2.5 supports Image and Video input."},"params.active":{"value":32000000000,"source":"card","at":"README.md; 2. Model Summary","quote":"Activated Parameters | 32B"},"params.note":{"value":"Rounded card figures; whether they include the 400M MoonViT encoder is not stated.","source":"card","at":"README.md; 2. Model Summary","quote":"Parameters of Vision Encoder | 400M"},"params.total":{"value":1000000000000,"source":"card","at":"README.md; 2. Model Summary","quote":"Total Parameters | 1T"},"shape.d_model":{"value":7168,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":163840,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":61,"source":"config","at":"/text_config/num_hidden_layers"},"attn.head_dim":{"value":192,"source":"config","at":"/text_config/qk_nope_head_dim + /text_config/qk_rope_head_dim"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"MLA","source":"card","at":"README.md; 2. Model Summary","quote":"Attention Mechanism | MLA"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/text_config/kv_lora_rank"},"attn.layout":{"value":"61 MLA","source":"card","at":"README.md; 2. Model Summary","quote":"Number of Layers (Dense layer included) | 61"},"attn.position":{"value":"partial RoPE (MLA decoupled, 64 of 192 QK dims)","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3Attention.forward","quote":"q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids)"},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/text_config/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3Attention.forward","quote":"q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states)))","note":"RMSNorm only on the Q and KV latents before up-projection; no per-head Q/K norm."},"attn.rope_dim":{"value":64,"source":"config","at":"/text_config/qk_rope_head_dim"},"attn.rope_theta":{"value":50000,"source":"config","at":"/text_config/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/text_config/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3MLP.forward","quote":"self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (config /text_config/hidden_act)."},"ffn.dim":{"value":18432,"source":"config","at":"/text_config/intermediate_size"},"moe.balancing":{"value":"aux-loss-free selection bias (e_score_correction_bias added to sigmoid scores for top-k only)","source":"modeling_deepseek","at":"modeling_deepseek.py; MoEGate.forward","quote":"bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)","note":"config topk_method = noaux_tc; the report does not describe the bias update rule or any auxiliary loss."},"moe.dense_layers":{"value":1,"source":"config","at":"/text_config/first_k_dense_replace","note":"moe_layer_freq = 1."},"moe.groups":{"value":"none","source":"config","at":"config.json text_config.n_group, topk_group","quote":"\"n_group\": 1,","note":"n_group = topk_group = 1."},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_topk_prob"},"moe.routed_scale":{"value":2.827,"source":"config","at":"/text_config/routed_scaling_factor"},"moe.shared_dim":{"value":2048,"source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3MoE.__init__","quote":"intermediate_size = config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"2048 × 1 shared expert."},"moe.expert_dim":{"value":2048,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/text_config/n_routed_experts"},"moe.router":{"value":"sigmoid","source":"config","at":"/text_config/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/text_config/n_shared_experts"},"moe.top_k":{"value":8,"source":"config","at":"/text_config/num_experts_per_tok"},"mtp.released":{"value":false,"source":"weight-index","at":"/weight_map | has eh_proj","note":"Backbone layers 0–60 only; config /text_config/num_nextn_predict_layers 0."},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3DecoderLayer.__init__","quote":"self.input_layernorm = DeepseekV3RMSNorm("},"ctx.extension":{"value":"YaRN, 32K → 256K during long-context mid-training (500B → 200B tokens)","source":"report","at":"p.8 §4.3","quote":"sequentially extending context length via YaRN","note":"Release rope_scaling: type yarn, factor 64 over original_max_position_embeddings 4096, beta_fast 32, beta_slow 1, mscale = mscale_all_dim = 1."},"ctx.final":{"value":262144,"source":"report","at":"p.7 Table 3","quote":"32768→262144"},"ctx.max":{"value":262144,"source":"config","at":"/text_config/max_position_embeddings"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.8 §4.3","quote":"The joint pre-training stage continues from a near-end Kimi K2 checkpoint over additional 15T vision-text tokens at 4K sequence length."},"train.data":{"value":"Kimi K2 distribution extended with new unique tokens, higher coding weight and per-source epoch caps, plus interleaved image, video and OS-screenshot data","source":"report","at":"p.8 §4.3","quote":"The data recipe extends Kimi K2’s pre-training distribution by introducing unique tokens, adjusting data proportions with increased weight on coding-related content, and controlling maximum epochs per data source.","note":"Joint pre-training data types from Table 3 (p.7)."},"train.hardware":{"value":"NVIDIA H800, 8×400 Gbps RoCE between nodes","source":"report","at":"App. C","quote":"Kimi K2.5 is trained on NVIDIA H800 GPU clusters with 8×400 Gbps RoCE interconnects across nodes."},"train.accelerators":{"value":"H800","source":"report","at":"App. C","quote":"Kimi K2.5 is trained on NVIDIA H800 GPU clusters with 8×400 Gbps RoCE interconnects across nodes.","note":"Count not stated."},"train.parallelism":{"value":"PP16 · VPP · EP16 · ZeRO-1 DP","source":"report","at":"p.10 §4.5.1","quote":"K2.5 seamlessly inherits the parallel strategy of K2","basis":"inferred","note":"Degrees are Kimi K2's (K2 report §2.4.2: 16-way PP with virtual stages, 16-way EP, ZeRO-1 DP); the vision encoder is replicated on all GPUs."},"train.init":{"value":"Language model continued from a near-end Kimi K2 checkpoint; MoonViT-3D from SigLIP-SO-400M","source":"report","at":"p.8 §4.3","quote":"The joint pre-training stage continues from a near-end Kimi K2 checkpoint"},"train.objective":{"value":"Joint vision-text next-token prediction with a constant vision ratio from the start of continued pretraining","source":"report","at":"p.2 §1","quote":"K2.5 mixes text and vision tokens with a constant ratio throughout the entire training process.","note":"The production vision:text ratio is not disclosed."},"train.stages":{"value":[{"name":"vit","tokens":1000000000000,"seq_len":4096,"note":"MoonViT-3D only, caption loss aligned to Moonlight-16B-A3B; then a short projector-only stage.","quote":"consuming about 1T tokens with very few training FLOPs."},{"name":"joint","tokens":15000000000000,"seq_len":4096,"note":"ViT and LLM trained.","quote":"over additional 15T vision-text tokens at 4K sequence length."},{"name":"long-context","seq_len":262144,"note":"Two phases, 500B tokens at 32K then 200B at 256K; high-quality text, multimodal, long text, long video, reasoning and long-CoT data.","quote":"500B→200B"}],"source":"report","at":"p.7 Table 3; p.7–8 §4.3"},"train.tokens":{"value":15000000000000,"source":"report","at":"p.8 §4.3","quote":"over additional 15T vision-text tokens at 4K sequence length.","note":"Joint continued-pretraining tokens on top of Kimi K2; excludes the 1T ViT-only stage and the 500B→200B long-context mid-training."},"eval.aa_index":{"value":23,"source":"aa","at":"Artificial Analysis model page /models/kimi-k2-5, Wayback snapshot 2026-10-19","quote":"Kimi K2.5 (Reasoning) scores 23 on the Artificial Analysis Intelligence Index","note":"Variant: Kimi K2.5 (Reasoning). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":1026408232448,"formula":"embedding + output head + all experts + other weights, moonshotai/Kimi-K2.5@4d01dfe safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":2348810240,"formula":"input embedding + output head, moonshotai/Kimi-K2.5@4d01dfe safetensors headers"},"params.counted_non_embedding":{"value":1024059422208,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, moonshotai/Kimi-K2.5@4d01dfe safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":17059365248,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, moonshotai/Kimi-K2.5@4d01dfe safetensors headers"},"params.counted_active_non_embedding":{"value":30512690688.0,"formula":"non-expert weights + routed experts × 8/384, excluding embedding and output head, moonshotai/Kimi-K2.5@4d01dfe safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2857142857142857,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":48.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.032,"formula":"params.active / params.total"},"scaling.tpp":{"value":15.0,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":491.5987302915631,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":70272.0,"formula":"70272 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9210691584.0,"formula":"70272 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":70272.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2196.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":287834112.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":70272.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":70272.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"kimi-k2","name":"Kimi-K2-Base","org":"Moonshot AI","country":"China","kind":"base","weights":"open","note":null,"sources":{"config":{"url":"https://huggingface.co/moonshotai/Kimi-K2-Base/resolve/ce72df012259dcc55d945e890f815fe7ef69159c/config.json","kind":"config"},"modeling_deepseek":{"url":"https://huggingface.co/moonshotai/Kimi-K2-Base/resolve/ce72df012259dcc55d945e890f815fe7ef69159c/modeling_deepseek.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2507.20534v1","kind":"paper"},"k3_report":{"url":"https://raw.githubusercontent.com/MoonshotAI/Kimi-K3/3cb39dfd32e51c3328e2e4b4af21341247d06c43/k3_tech_report.pdf","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20260817080036/https://www.kimi.com/blog/","kind":"blog"},"card":{"url":"https://huggingface.co/moonshotai/Kimi-K2-Base/resolve/ce72df012259dcc55d945e890f815fe7ef69159c/README.md","kind":"model-card"},"weight-index":{"url":"https://huggingface.co/moonshotai/Kimi-K2-Base/resolve/ce72df012259dcc55d945e890f815fe7ef69159c/model.safetensors.index.json","kind":"config"},"aa":{"url":"https://web.archive.org/web/20261019101736/https://artificialanalysis.ai/models/kimi-k2","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-07-11","source":"blog","at":"kimi.com blog index (Wayback 2026-08-17); Kimi K2 entry","quote":"Kimi K2 2025-07-11"},"meta.license":{"value":"other (modified-mit)","source":"card","at":"README.md; YAML front matter","quote":"license_name: modified-mit"},"params.active":{"value":32600000000,"source":"report","at":"p.6 §2.3, Table 2","quote":"Activated Parameters 37B 32.6B","note":"Abstract and card round to 32B."},"params.note":{"value":"Table 2 figures; abstract rounds to 1T total / 32B active.","source":"report","at":"p.1 Abstract","quote":"32 billion activated parameters and 1 trillion total parameters"},"params.total":{"value":1040000000000,"source":"report","at":"p.6 §2.3, Table 2","quote":"Kimi K2 is a 1.04 trillion-parameter Mixture-of-Experts (MoE) transformer model","note":"Abstract and card round to 1T."},"shape.d_model":{"value":7168,"source":"config","at":"/hidden_size"},"shape.layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":163840,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.global_layers":{"value":61,"source":"config","at":"/num_hidden_layers"},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA","source":"report","at":"p.6 §2.3","quote":"employing Multi-head Latent Attention (MLA) [44] as the attention mechanism"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"61 MLA","source":"report","at":"p.6 §2.3, Table 2","quote":"#Layers 61 61"},"attn.position":{"value":"partial RoPE (MLA decoupled, 64 of 192 QK dims)","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3Attention.forward","quote":"q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids)"},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3Attention.forward","quote":"torch.matmul(query_states, key_states.transpose(2, 3)) * self.softmax_scale","note":"RMSNorm only on the Q and KV latents before up-projection; no per-head Q/K norm. Training bounded logits with QK-Clip weight clipping instead (see train.optimizer)."},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":50000,"source":"config","at":"/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (config hidden_act)."},"ffn.bias":{"value":false,"source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":18432,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":1,"source":"report","at":"p.6 §2.3, Table 2","quote":"Number of Dense Layers 3 1","note":"config first_k_dense_replace = 1, moe_layer_freq = 1."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/n_routed_experts"},"moe.balancing":{"value":"aux-loss-free selection bias (e_score_correction_bias added to sigmoid scores for top-k only)","source":"modeling_deepseek","at":"modeling_deepseek.py; MoEGate.forward","quote":"scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)","note":"config topk_method = noaux_tc; the K2 report does not describe the bias update rule or any auxiliary loss."},"moe.groups":{"value":"none","source":"report","at":"p.6 §2.3, Table 2","quote":"Expert Grouping Yes No","note":"DeepSeek-V3 column says Yes, Kimi K2 says No; config n_group = topk_group = 1."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob","note":"Applied in MoEGate.forward before the routed scaling factor."},"moe.routed_scale":{"value":2.827,"source":"config","at":"/routed_scaling_factor","note":"MoEGate.forward multiplies the renormalized top-k weights by it."},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3MoE.__init__","quote":"intermediate_size = config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"2048 × 1 shared expert."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"k3_report","at":"K3 report p.11 §3.2, Table 1 (Kimi K2 column)","quote":"Number of MTP Layers 1 layer 1 layer","note":"The K2 report does not mention MTP."},"mtp.modules":{"value":1,"source":"k3_report","at":"K3 report p.11 §3.2, Table 1 (Kimi K2 column)","quote":"Number of MTP Layers 1 layer 1 layer"},"mtp.released":{"value":false,"source":"weight-index","at":"/weight_map | has eh_proj","note":"Backbone layers 0–60 only; config /num_nextn_predict_layers 0."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_deepseek","at":"modeling_deepseek.py; DeepseekV3DecoderLayer.__init__","quote":"self.input_layernorm = DeepseekV3RMSNorm("},"ctx.extension":{"value":"4K → 32K (60B tokens), then YaRN to 128K","source":"report","at":"p.9 §2.5","quote":"followed by an additional 60 billion tokens with a 32k sequence length. To extend the context window to 128k, we employed the YaRN method [55].","note":"Release rope_scaling: type yarn, factor 32 over original_max_position_embeddings 4096, beta_fast = beta_slow = 1, mscale = mscale_all_dim = 1 (softmax scale × (0.1·ln 32 + 1)² in DeepseekV3Attention)."},"ctx.final":{"value":32768,"source":"report","at":"p.9 §2.5","quote":"followed by an additional 60 billion tokens with a 32k sequence length","note":"Longest length the K2 report states was trained; 128K is reached by YaRN. The K3 report (Table 1) lists K2 training context as 128K."},"ctx.max":{"value":131072,"source":"report","at":"p.9 §2.5","quote":"To extend the context window to 128k, we employed the YaRN method","note":"config max_position_embeddings = 131072."},"ctx.pretrain":{"value":4096,"source":"report","at":"p.9 §2.5","quote":"We pre-trained the model with a 4,096-token context window"},"train.batch":{"value":"67M tokens, constant","source":"report","at":"p.9 §2.5","quote":"the global batch size was held at 67M tokens."},"train.batch_tokens":{"value":67000000,"source":"report","at":"p.9 §2.5","quote":"the global batch size was held at 67M tokens."},"train.data":{"value":"15.5T tokens of Web Text, Code, Mathematics, Knowledge; knowledge corpora rephrased by LLM (each at most twice), math rewritten as learning notes and translated to English","source":"report","at":"p.5–6 §2.2","quote":"We extended this method to other large-scale knowledge corpora and observed similarly encouraging results, and each corpora is rephrased at most twice.","note":"Domains: p.6 §2.2 \"Pre-training Data Overall\"; math rewriting follows SwallowMath."},"train.hardware":{"value":"NVIDIA H800, 8 GPUs per node, 8×400 Gbps RoCE between nodes","source":"report","at":"p.7 §2.4.1","quote":"Kimi K2 was trained on a cluster equipped with NVIDIA H800 GPUs."},"train.accelerators":{"value":"H800","source":"report","at":"p.7 §2.4.1","quote":"Kimi K2 was trained on a cluster equipped with NVIDIA H800 GPUs.","note":"Count not stated."},"train.parallelism":{"value":"PP16 · VPP · EP16 · ZeRO-1 DP","source":"report","at":"p.8 §2.4.2","quote":"Pipeline Parallelism (PP) with virtual stages [28, 53, 38, 57, 47, 21], 16-way Expert Parallelism (EP) [39], and ZeRO-1 Data Parallelism [60].","note":"The PP degree ends the previous line, which Figure 7 separates from the quote: 'a combination of 16-way'; VPP degree not stated."},"train.lr_min":{"value":7e-06,"source":"report","at":"p.9 §2.5","quote":"the learning rate was decayed from 2e-5 to 7e-6."},"train.lr_peak":{"value":0.0002,"source":"report","at":"p.9 §2.5","quote":"The first 10T tokens were trained with a constant learning rate of 2e-4 after a 500-step warm-up, followed by 5.5T tokens with a cosine decay from 2e-4 to 2e-5."},"train.optimizer":{"value":"MuonClip (Muon + weight decay + RMS matching + QK-Clip, τ=100)","source":"report","at":"p.4 §2.1","quote":"we train Kimi K2, a large-scale MoE model, using MuonClip with τ = 100"},"train.precision":{"value":"No FP8 compute; MoE up-projection and SwiGLU inputs stored as FP8-E4M3 (1×128 tiles, FP32 scales)","source":"report","at":"p.8 §2.4.3","quote":"Inputs of MoE up-projections and SwiGLU are compressed to FP8-E4M3 in 1× 128 tiles with FP32 scales. Small-scale experiments show no measurable loss increase. Due to potential risks of performance degradation that we observed during preliminary study, we do not apply FP8 in computation."},"train.schedule":{"value":"WSD: 500-step warmup, constant 2e-4 for 10T, cosine to 2e-5 over 5.5T, then 2e-5 → 7e-6 over the 400B (4K) + 60B (32K) annealing/long-context phase","source":"report","at":"p.9 §2.5","quote":"the WSD learning rate schedule [25], processing a total of 15.5T tokens."},"train.stability":{"value":"QK-Clip (MuonClip), τ=100, per head; MLA: head-specific q/k content weights × √γ, q rope weights × γ, shared k rope untouched; 12.7% of heads triggered at least once in the first 70k steps, inactive afterwards","source":"report","at":"p.30 App. D","quote":"Initial 70000 steps: 12.7% of attention heads triggered QK-Clip for at least once, clamping Smax to 100.","note":"Per-head weight rescaling is Algorithm 1 (p.4 §2.1); no loss spikes over the run (Figure 3)."},"train.stages":{"value":[{"name":"main","tokens":15500000000000,"seq_len":4096,"lr":0.0002,"note":"Constant 2e-4 for 10T tokens, then cosine to 2e-5 over 5.5T.","quote":"The first 10T tokens were trained with a constant learning rate of 2e-4 after a 500-step warm-up, followed by 5.5T tokens with a cosine decay from 2e-4 to 2e-5."},{"name":"anneal","tokens":400000000000,"seq_len":4096,"lr":2e-05,"note":"LR decays from 2e-5 to 7e-6 across the anneal and long-context stages.","quote":"the model was trained on 400 billion tokens with a 4k sequence length"},{"name":"long-context","tokens":60000000000,"seq_len":32768,"quote":"followed by an additional 60 billion tokens with a 32k sequence length"}],"source":"report","at":"p.9 §2.5"},"train.tokens":{"value":15500000000000,"source":"report","at":"p.9 §2.5","quote":"We pre-trained the model with a 4,096-token context window using the MuonClip optimizer (Algorithm 1) and the WSD learning rate schedule [25], processing a total of 15.5T tokens.","note":"15.5T = 10T constant + 5.5T cosine; the 400B anneal and 60B long-context tokens are reported separately."},"train.warmup":{"value":"500 steps","source":"report","at":"p.9 §2.5","quote":"after a 500-step warm-up"},"train.weight_decay":{"value":0.1,"source":"report","at":"p.9 §2.5","quote":"Weight decay was set to 0.1 throughout"},"scaling.ladder":{"value":"small-scale Muon MoE runs: sparsity 8–64 at 8 active + 1 shared expert; heads = layers vs 2× layers at 1.2e20–9.0e20 FLOPs","source":"report","at":"p.6–7 §2.3, Figures 5–6","quote":"Through carefully controlled small-scale experiments, we observe that — under a fixed number of activated parameters (i.e., constant FLOPs) — increasing the total number of experts (i.e., increasing sparsity) consistently lowers both the training and validation loss","note":"Sparsity levels and FLOPs budgets from the Figure 5–6 legends; model sizes are not given."},"eval.aa_index":{"value":13,"source":"aa","at":"Artificial Analysis model page /models/kimi-k2, Wayback snapshot 2026-10-19","quote":"Kimi K2 scores 13 on the Artificial Analysis Intelligence Index","note":"Score of Kimi-K2-Instruct, post-trained from this base. Variant: Kimi K2. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.block","mtp.ffn","mtp.shares_embedding","mtp.loss_weight","train.betas","train.grad_clip"],"derived":{"params.counted_total":{"value":1026408235864,"formula":"embedding + output head + all experts + other weights, moonshotai/Kimi-K2-Base@ce72df0 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":2348810240,"formula":"input embedding + output head, moonshotai/Kimi-K2-Base@ce72df0 safetensors headers"},"params.counted_non_embedding":{"value":1024059425624,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, moonshotai/Kimi-K2-Base@ce72df0 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":17059365304,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, moonshotai/Kimi-K2-Base@ce72df0 safetensors headers"},"params.counted_active_non_embedding":{"value":30512694104.0,"formula":"non-expert weights + routed experts × 8/384, excluding embedding and output head, moonshotai/Kimi-K2-Base@ce72df0 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.2857142857142857,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":48.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.03134615384615384,"formula":"params.active / params.total"},"scaling.tpp":{"value":14.903846153846153,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":507.985297763925,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":70272.0,"formula":"70272 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9210691584.0,"formula":"70272 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":70272.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2155.5828220858893,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":282536551.6564417,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":70272.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":70272.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"kimi-k3","name":"Kimi-K3","org":"Moonshot AI","country":"China","kind":"reasoning","weights":"open","note":"Post-trained always-thinking checkpoint (MoE expert weights QAT-quantized to MXFP4); no base checkpoint is public.","sources":{"card":{"url":"https://huggingface.co/moonshotai/Kimi-K3/resolve/f831ab66814297da540d832a5235f8e904f29d06/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/moonshotai/Kimi-K3/resolve/f831ab66814297da540d832a5235f8e904f29d06/config.json","kind":"config"},"modeling_kimi_linear":{"url":"https://huggingface.co/moonshotai/Kimi-K3/resolve/f831ab66814297da540d832a5235f8e904f29d06/modeling_kimi_linear.py","kind":"code"},"report":{"url":"https://raw.githubusercontent.com/MoonshotAI/Kimi-K3/3cb39dfd32e51c3328e2e4b4af21341247d06c43/k3_tech_report.pdf","kind":"paper"},"weight-index":{"url":"https://huggingface.co/moonshotai/Kimi-K3/resolve/f831ab66814297da540d832a5235f8e904f29d06/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20260817080036/https://www.kimi.com/blog/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019102042/https://artificialanalysis.ai/models/kimi-k3","kind":"leaderboard"},"deepswe":{"url":"https://web.archive.org/web/20261009070156id_/https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-16","source":"blog","at":"kimi.com blog index (Wayback 2026-08-17); Kimi K3 entry","quote":"Kimi K3 2026-07-16"},"meta.license":{"value":"other (kimi-k3)","source":"card","at":"README.md; YAML front matter","quote":"license_name: \"kimi-k3\""},"meta.modality":{"value":"text + image + video","source":"card","at":"§1 Model Introduction; Key Features","quote":"Kimi K3 understands text, images, and video within the same model"},"params.active":{"value":104200000000,"source":"report","at":"p.11 §3.2, Table 1","quote":"Activated Parameters 32.6B 104.2B","note":"Card and abstract round to 104B."},"params.note":{"value":"Table 1 figures (card rounds to 2.8T / 104B). The 401M MoonViT-V2 vision encoder is listed separately; whether the totals include it is not stated.","source":"report","at":"p.11 §3.2, Table 1","quote":"Total Parameters of ViT - 401M"},"params.total":{"value":2780000000000,"source":"report","at":"p.11 §3.2, Table 1","quote":"Total Parameters 1.04T 2.78T","note":"Card and abstract round to 2.8T."},"shape.d_model":{"value":7168,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":93,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":163840,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiMLAAttention.__init__, KimiDeltaAttention.__init__","quote":"self.v_proj = nn.Linear(self.hidden_size, projection_size, bias=False)","note":"All MLA and KDA projections are built with bias=False."},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_attn_config/short_conv_kernel_size"},"attn.gate":{"value":"sigmoid, channel-wise full-rank, on MLA (and KDA) output before o_proj","source":"report","at":"p.5 §2.1.2, Eq. 7","quote":"Kimi K3 augments MLA with an input-dependent, channel-wise full-rank output gate.","note":"config mla_use_output_gate = true; KDA gate from linear_attn_config use_full_rank_gate."},"attn.global_layers":{"value":24,"source":"report","at":"p.11 §3.2, Table 1","quote":"Attention-Layer Composition 61 MLA 69 KDA + 24 MLA"},"attn.head_dim":{"value":192,"source":"config","at":"/text_config/qk_nope_head_dim + /text_config/qk_rope_head_dim","note":"MLA layers. KDA layers use head_dim 128 (/text_config/linear_attn_config/head_dim)."},"attn.heads":{"value":96,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"KDA, MLA","source":"report","at":"p.4 §2.1","quote":"Kimi K3 uses a layerwise hybrid of linear and global attention, combining KDA [64] with Gated MLA."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/text_config/kv_lora_rank"},"attn.layout":{"value":"(3 KDA : 1 MLA) × 23 + 1 MLA","source":"config","at":"config.json; /text_config/linear_attn_config/full_attn_layers","quote":"\"full_attn_layers\": [ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 93 ]","note":"linear_attn_config full_attn_layers (1-indexed) = 4, 8, …, 92, 93; kda_layers lists the other 69. full_attn_layers uses one-based indices; kda_layers is the complementary list. Excludes auxiliary modules.","basis":"inferred","sequence":["KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","MLA","MLA"]},"attn.qk_norm":{"value":false,"source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiMLAAttention.forward","quote":"query_states = torch.cat((q_pass, q_rot), dim=-1)","note":"MLA layers: RMSNorm only on the Q and KV latents. KDA layers L2-normalize q and k (attn.linear_kind)."},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_attn_config/head_dim"},"attn.linear_heads":{"value":96,"source":"config","at":"/text_config/linear_attn_config/num_heads"},"attn.linear_kind":{"value":"KDA: 96 heads × 128, short conv 4 + Swish on q/k/v, L2-normed q/k, channel-wise decay with log-decay bounded below at −5 (gmin·sigmoid), head-wise RMSNorm + full-rank sigmoid output gate","source":"report","at":"p.5 §2.1.1, Eq. 5","quote":"where Ah is a learnable per-head log-scale and gmin = −5 is fixed.","note":"Heads, head dim and kernel from /text_config/linear_attn_config; gate_lower_bound = -5; Ah initialized to 0."},"attn.linear_layers":{"value":69,"source":"report","at":"p.11 §3.2, Table 1","quote":"Attention-Layer Composition 61 MLA 69 KDA + 24 MLA"},"attn.position":{"value":"NoPE (all MLA layers)","source":"report","at":"p.5 §2.1.2","quote":"Kimi K3 follows the hybrid design of Kimi Linear [64] and applies No Position Encoding (NoPE) to all MLA layers."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/text_config/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"config","at":"/text_config/qk_rope_head_dim","note":"Head-shared key slice cached next to the latent but never rotated: mla_use_nope=true and the code asserts use_nope with rotary_emb=None."},"attn.v_head_dim":{"value":128,"source":"config","at":"/text_config/v_head_dim"},"ffn.activation":{"value":"SiTU-GLU (β1=4 gate, β2=25 up)","source":"report","at":"p.11 §3.2, Table 1; β values p.8 §2.3.2","quote":"Activation Function SwiGLU SiTU-GLU","note":"β1 = 4 and β2 = 25 per §2.3.2 and config activation_situ_beta / activation_situ_linear_beta."},"ffn.bias":{"value":false,"source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiMLP.__init__","quote":"self.intermediate_size, self.hidden_size, bias=False)"},"ffn.clamp":{"value":"SiTU-GLU soft caps: gate 4·tanh(x/4)·σ(x), up 25·tanh(x/25), |f| ≤ 100; all dense, shared and routed FFNs","source":"report","at":"p.7 §2.3.2, Figure 4","quote":"SiTU-GLU, shown in red with β1 = 4 and β2 = 25, closely follows SwiGLU near the origin and approaches the bound |f (x)| ≤ β1 β2 = 100 for large positive inputs","note":"Smooth tanh caps, not hard clamps; config hidden_act = situ applies to every MLP in modeling_kimi_linear.py."},"ffn.dim":{"value":33792,"source":"config","at":"/text_config/intermediate_size"},"moe.balancing":{"value":"Quantile Balancing: aux-loss-free selection bias; each expert's bias recomputed every step from the (1−k/n)-quantile of its routing margins (global-batch histogram estimate), applied from the next step","source":"report","at":"p.8–9 §2.3.3; p.11 §3.3","quote":"the (1 − k/n)-quantile of the margins across tokens, giving the QB update","note":"config topk_method = noaux_tc (e_score_correction_bias added to scores for selection)."},"moe.dense_layers":{"value":1,"source":"config","at":"/text_config/first_k_dense_replace","note":"moe_layer_freq = 1."},"moe.groups":{"value":"none","source":"config","at":"config.json text_config.num_expert_group, topk_group","quote":"\"num_expert_group\": 1,","note":"num_expert_group = topk_group = 1, so use_grouped_topk has no effect."},"moe.expert_dim":{"value":3072,"source":"config","at":"/text_config/moe_intermediate_size","note":"Routed experts run in a 3584-wide latent (LatentMoE, /text_config/routed_expert_hidden_size)."},"moe.experts":{"value":896,"source":"config","at":"/text_config/num_experts"},"moe.router":{"value":"sigmoid","source":"config","at":"/text_config/moe_router_activation_func"},"moe.shared":{"value":2,"source":"config","at":"/text_config/num_shared_experts"},"moe.shared_dim":{"value":6144,"source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiSparseMoeBlock.__init__","quote":"intermediate_size = config.moe_intermediate_size * config.num_shared_experts","basis":"inferred","note":"3072 × 2 shared experts, fused into one full-width MLP."},"moe.latent_dim":{"value":3584,"source":"config","at":"/text_config/routed_expert_hidden_size","note":"0.5 × d_model (Table 1); RMSNorm on the aggregated routed output before the up-projection (Eq. 11; config latent_moe_use_norm)."},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/moe_renormalize"},"moe.routed_scale":{"value":1.0,"source":"config","at":"/text_config/routed_scaling_factor"},"moe.top_k":{"value":16,"source":"config","at":"/text_config/num_experts_per_token"},"mtp.depth":{"value":1,"source":"report","at":"p.14 §4.1.4","quote":"Kimi K3 is pre-trained with a multi-token-prediction (MTP) layer that mirrors the structure of a backbone block."},"mtp.modules":{"value":1,"source":"report","at":"p.11 §3.2, Table 1","quote":"Number of MTP Layers 1 layer 1 layer"},"mtp.block":{"value":"one layer with the structure of a backbone block","source":"report","at":"p.14 §4.1.4","quote":"multi-token-prediction (MTP) layer that mirrors the structure of a backbone block","note":"Mixer type (KDA or MLA), FFN type and how hidden state and embedding are combined are not stated."},"mtp.released":{"value":false,"source":"weight-index","at":"/weight_map | has eh_proj","note":"Backbone layers 0–92 only; config /text_config/num_nextn_predict_layers 0. The pretrained MTP layer was fine-tuned into an EAGLE-3 drafter (report p.14), not shipped here."},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)","note":"Residual path is Block Attention Residuals (see norm.residual)."},"norm.residual":{"value":"Block Attention Residuals: layer outputs summed within 12-layer blocks (8 blocks + embedding = 9 sources); each attention and FFN sublayer input is a softmax-weighted mix of block sums and the current partial sum (learned query, RMSNorm'd keys)","source":"report","at":"p.6 §2.2","quote":"for Kimi K3, we partition its layers into 8 blocks with 12-layer size, giving a partial final block and 9 total blocks when counting the embedding layer.","note":"/text_config/attn_res_block_size = 12; separate res queries for attention and FFN in KimiDecoderLayer._forward_attn_residual; the final output also aggregates all blocks."},"norm.type":{"value":"RMSNorm","source":"modeling_kimi_linear","at":"modeling_kimi_linear.py; KimiDecoderLayer.__init__","quote":"self.input_layernorm = KimiRMSNorm("},"ctx.extension":{"value":"8K → 64K (pre-training) → 256K → 1M (cooldown); no positional-encoding change (NoPE)","source":"report","at":"p.12 §3.4","quote":"following a four-stage curriculum. The window grows from 8K to 64K tokens during pre-training, and from 256K to 1M tokens during the cooldown phase."},"ctx.final":{"value":1048576,"source":"report","at":"p.12 §3.4","quote":"The window grows from 8K to 64K tokens during pre-training, and from 256K to 1M tokens during the cooldown phase."},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Model Summary","quote":"Context Length 1048576","note":"config max_position_embeddings = 1048576."},"ctx.pretrain":{"value":8192,"source":"report","at":"p.12 §3.3","quote":"Our pre-training begins with a context length of 8k tokens, which is later extended to 64k tokens"},"train.data":{"value":"Web Text, Code, Mathematics, Knowledge plus vision (captions, interleaved image–text, OCR, perception, video, visual coding); K2-style rephrasing of knowledge and math; long documents and videos upsampled and synthetic long-context data in cooldown","source":"report","at":"p.10 §3.1","quote":"Following the rephrasing recipe of Kimi K2 [59], we rephrase knowledge and mathematics corpora with style and perspective-diverse prompting, chunk-wise autoregressive generation, and fidelity verification against the source documents.","note":"Domain list p.10 §3.1; long-context data p.12 §3.4. No token counts or shares disclosed."},"train.objective":{"value":"native multimodal: text and vision tokens interleaved in one next-token objective from the start; vision encoder trained from scratch","source":"report","at":"p.11 §3.3","quote":"Under this paradigm, visual and textual tokens are interleaved within a single next-token prediction objective","note":"MoonViT-V2 trained from scratch with next-token prediction rather than SigLIP-initialized (p.9 §2.4)."},"train.optimizer":{"value":"Muon (per-head Newton–Schulz on Q/K/V projections; K2 QK-Clip weight clipping)","source":"report","at":"p.11 §3.3","quote":"We optimize the model using the Per-Head Muon optimizer (§ 2.5) together with the weight-clipping mechanism introduced in Kimi K2"},"train.schedule":{"value":"cosine with 1% linear warmup","source":"report","at":"p.11 §3.3","quote":"We use a cosine learning rate schedule with a 1% linear warmup."},"train.stability":{"value":"K2 QK-Clip weight clipping; SiTU-GLU and RMSNorm before the latent up-projection against routed-branch activation explosion","source":"report","at":"p.11 §3.3","quote":"together with the weight-clipping mechanism introduced in Kimi K2","note":"Clipping threshold not stated. SiTU-GLU and RMSNorm: p.7 §2.3."},"train.stages":{"value":[{"name":"pretrain-8k","seq_len":8192,"quote":"Our pre-training begins with a context length of 8k tokens"},{"name":"pretrain-64k","seq_len":65536,"quote":"which is later extended to 64k tokens in a subsequent training phase."},{"name":"cooldown-256k","seq_len":262144,"quote":"and from 256K to 1M tokens during the cooldown phase."},{"name":"cooldown-1m","seq_len":1048576,"quote":"and from 256K to 1M tokens during the cooldown phase."}],"source":"report","at":"p.12 §3.3–3.4","note":"Native multimodal (text + vision) from the first step. Token counts per stage not disclosed."},"train.warmup":{"value":"1% of training, linear","source":"report","at":"p.11 §3.3","quote":"with a 1% linear warmup"},"train.weight_decay":{"value":0.1,"source":"report","at":"p.11 §3.3","quote":"Weight decay is set to 0.1 throughout."},"train.parallelism":{"value":"PP · VPP · EP · CP · ZeRO-1 DP","source":"report","at":"p.18 §5.2","quote":"Kimi K3 pre-training combines Pipeline Parallelism (PP) with virtual stages (VP) [49, 82], Expert Parallelism (EP) [67], ZeRO-1 Data Parallelism [101], Pipeline ZeRO-2 gradient sharding [147], and Context Parallelism (CP, §5.1.2) [51].","note":"Degrees not stated; gradients are also sharded across DP ranks (Pipeline ZeRO-2)."},"scaling.hp_rule":{"value":"batch size, LR, TPP and model shape retuned by scaling-law studies; separate search per schedule, cosine chosen over WSD","source":"report","at":"p.10 §3.2","quote":"we conduct dedicated scaling-law studies to retune key hyperparameters, including the batch size, learning rate, tokens-per-parameter ratio (TPP) and the model shape"},"eval.aa_index":{"value":44,"source":"aa","at":"Artificial Analysis model page /models/kimi-k3, Wayback snapshot 2026-10-19","quote":"Kimi K3 (Max) scores 44 on the Artificial Analysis Intelligence Index","note":"Variant: Kimi K3 (Max). Index v4.3.2."},"eval.deepswe":{"value":0.6851441241685144,"source":"deepswe","at":"leaderboard-live.json (generated 2026-09-22), row model kimi-k3, Wayback snapshot 2026-10-09","quote":"\"model\": \"kimi-k3\", \"harness\": \"mini-swe-agent\", \"reasoning_effort\": \"max\", \"config\": \"mini_swe_agent_kimi_k3_max\", \"source\": \"deep-swe\", \"pass_rate\": 0.6851441241685144, \"pass_at_1\": 0.6851441241685144","note":"Reasoning effort max (the only configuration listed), mini-swe-agent, 4 runs, 451 scored attempts."}},"not_disclosed":["params.active_non_embedding","mtp.ffn","mtp.shares_embedding","mtp.loss_weight","train.tokens","train.lr_peak","train.lr_min","train.batch","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":2779484478208,"formula":"embedding + output head + all experts + other weights, moonshotai/Kimi-K3@f831ab6 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_embedding":{"value":2348810240,"formula":"input embedding + output head, moonshotai/Kimi-K3@f831ab6 safetensors headers"},"params.counted_non_embedding":{"value":2777135667968,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, moonshotai/Kimi-K3@f831ab6 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_max_layer":{"value":30228720256,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, moonshotai/Kimi-K3@f831ab6 safetensors headers"},"params.counted_active_non_embedding":{"value":103015209728.0,"formula":"non-expert weights + routed experts × 16/896, excluding embedding and output head, moonshotai/Kimi-K3@f831ab6 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"shape.aspect":{"value":77.0752688172043,"formula":"d_model / layers"},"attn.qk_width":{"value":18432,"formula":"heads × head_dim"},"ffn.mult":{"value":4.714285714285714,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.42857142857142855,"formula":"moe.expert_dim / d_model"},"moe.latent_ratio":{"value":2.0,"formula":"d_model / moe.latent_dim"},"attn.linear_ratio":{"value":2.875,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":56.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.03748201438848921,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":27648.0,"formula":"27648 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3623878656.0,"formula":"27648 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":27648.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":265.33589251439537,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":34778106.10364683,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":27648.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":27648.0,"tokens_per_entry":1}],"formula":"Global KV: 24 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"laguna-m-1-base","name":"Laguna-M.1-base","org":"Poolside","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/poolside/Laguna-M.1-base/resolve/628037ad548abdf6445ed00d1a691ac36c9dad2f/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/poolside/Laguna-M.1-base/resolve/628037ad548abdf6445ed00d1a691ac36c9dad2f/config.json","kind":"config"},"modeling_laguna":{"url":"https://huggingface.co/poolside/Laguna-M.1/resolve/2bf8a4ab6d2e8e1e0d92a41318532a0fee1640d6/modeling_laguna.py","kind":"code"},"report":{"url":"https://web.archive.org/web/20261004102254id_/https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf","kind":"paper"}},"facts":{"meta.released":{"value":"2026-05-25","source":"report","at":"p.1 title block","quote":"RESEARCH@POOLSIDE.AI MAY 25, 2026","note":"Technical report date; M.1 was API-only from 2026-04-28 and its weights were published after the report."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"Model overview","quote":"Modality: text-to-text (completion)"},"params.active":{"value":23400000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"out of which 23.4B (including embeddings) are activated per token"},"params.active_non_embedding":{"value":22600000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"out of which 23.4B (including embeddings) are activated per token","basis":"inferred","note":"23.4B − 2 × 100,352 × 4,096 (input embedding and untied LM head) = 22.58B, rounded to the 0.1B precision of the stated count."},"params.note":{"value":"Active count includes embeddings.","source":"report","at":"p.5 §3.1 Architecture & Training","quote":"out of which 23.4B (including embeddings) are activated per token"},"params.total":{"value":225800000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"LAGUNA M.1 has 225.8B total parameters"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":70,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100352,"source":"config","at":"/vocab_size"},"shape.tokenizer":{"value":"BPE trained on internal data; shared with XS.2","source":"report","at":"p.5 §3.1","quote":"Both models share the same tokenizer with a vocabulary size of 100,352 tokens, trained with Byte-Pair Encoding [75] on our internal datasets."},"attn.gate":{"value":"softplus, per element, on attention output before o_proj","source":"card","at":"README.md; Model overview","quote":"Attention: 64 Q-heads, 8 KV-heads, head dimension 128, with softplus attention output gating","note":"Per element: config /gating = \"per-element\" (one gate per head channel); XS.2 moved to per-head."},"attn.global_layers":{"value":70,"source":"card","at":"README.md; Model overview","quote":"Layers: 70 layers with global attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA","source":"card","at":"README.md; Highlights","quote":"Laguna M.1 uses global attention across all layers with 64 Q-heads, 8 KV-heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"all global","source":"card","at":"README.md; Highlights","quote":"Laguna M.1 uses global attention across all layers","note":"Config sliding_window 0 means SWA is disabled."},"attn.position":{"value":"RoPE","source":"card","at":"README.md; Model overview","quote":"Positional encoding: RoPE with YaRN"},"attn.qk_norm":{"value":true,"source":"modeling_laguna","at":"Laguna-M.1 modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_norm = LagunaRMSNorm(config.head_dim, eps=config.rms_norm_eps)"},"attn.rope_fraction":{"value":1,"source":"config","at":"/rope_parameters/full_attention/partial_rotary_factor"},"attn.rope_theta":{"value":500000,"source":"config","at":"/rope_parameters/full_attention/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.bias":{"value":false,"source":"modeling_laguna","at":"Laguna-M.1 modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, self.num_heads * config.head_dim, bias=False)"},"ffn.activation":{"value":"SwiGLU","source":"card","at":"README.md; Model overview","quote":"Dense layers: first 3 layers are dense SwiGLU; remaining 67 layers are sparse MoE"},"ffn.dim":{"value":16384,"source":"config","at":"/intermediate_size"},"ffn.bias":{"value":false,"source":"modeling_laguna","at":"Laguna-M.1 modeling_laguna.py; LagunaMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.balancing":{"value":"auxiliary-loss-free","source":"card","at":"README.md; Highlights","quote":"Laguna M.1 uses 67 sparse MoE layers with 256 experts, top-k=16 routing and auxiliary-loss-free load balancing","note":"Report §3.1.3 says M.1 padding tokens were \"included in the load-balancing loss\"; coefficient not disclosed."},"moe.dense_layers":{"value":3,"source":"config","at":"/mlp_only_layers | len","note":"Layers 0-2; decoder_sparse_step 1 makes every other layer MoE (LagunaDecoderLayer.__init__)."},"moe.expert_dim":{"value":1024,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.router":{"value":"sigmoid","source":"modeling_laguna","at":"Laguna-M.1 modeling_laguna.py; LagunaTopKRouter","quote":"Laguna MoE router using sigmoid scoring (not softmax)."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":1.0,"source":"config","at":"/moe_routed_scaling_factor","note":"Routed-expert modulation (2.5) was introduced in XS.2 (report §3.1)."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model overview","quote":"Experts: 256 experts with 1 shared expert; top-k=16 routing"},"moe.shared_dim":{"value":1024,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":16,"source":"config","at":"/num_experts_per_tok"},"norm.placement":{"value":"pre","source":"report","at":"p.5 §3.1","quote":"LAGUNA XS.2 and M.1 are both Mixture-of-Experts (MoE) [77] models using a pre-norm Transformer architecture"},"norm.type":{"value":"RMSNorm","source":"report","at":"p.5 §3.1","quote":"with RMSNorm [103] as the normalization layer"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.extension":{"value":"YaRN factor 64 over original 4,096 (release config)","source":"config","at":"config.json; rope_parameters.full_attention","quote":"\"rope_type\": \"yarn\", \"factor\": 64.0, \"original_max_position_embeddings\": 4096,","note":"M.1's long-context training stages are not disclosed; the report's 32K/128K recipe is stated for XS.2. Same dict: beta_fast 64, beta_slow 1, attention_factor 1.416; max_position_embeddings 262,144."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Model overview","quote":"Context window: 262,144 tokens"},"train.optimizer":{"value":"Muon + AdamW (AdamW for non-matrix parameters)","source":"report","at":"p.8 §3.1.3 Expert Collapse","quote":"To align the effective weight decay between matrix parameters (updated by Muon)","note":"Same sentence: non-matrix parameters updated by AdamW. Muon in M.1 pre-training: §3.1.2 Distributed Muon reports its overhead \"during LAGUNA M.1 pre-training\"."},"train.schedule":{"value":"cosine","source":"report","at":"p.5 §3.1 From Laguna M.1 to Laguna XS.2","quote":"Second, we adopted a Warmup-Stable-Decay (WSD) learning rate schedule [34] instead of cosine.","basis":"inferred","note":"Listed as an M.1 → XS.2 change, implying M.1 used cosine. Phase boundaries not disclosed."},"train.tokens":{"value":">30T","source":"report","at":"p.9 §3.2","quote":"In total, both models were trained on more than 30T tokens."},"train.data":{"value":"sampled from a ~27T unique-token pool (web, code, educational, academic, synthetic); manually designed mixture with a high-precision web pipeline","source":"report","at":"p.9 §3.2","quote":"We trained both LAGUNA M.1 and LAGUNA XS.2 from scratch on a pre-training corpus sampled from a pool of ∼27T unique tokens.","note":"Same section: for M.1 \"we used a high-precision pipeline with a manually designed mixture\"; the Table 4 AutoMixer mix is XS.2's."},"train.hardware":{"value":"6,144 H200","source":"report","at":"p.6 §3.1.1","quote":"LAGUNA M.1 and LAGUNA XS.2 were pre-trained on 6,144 and 2,048 NVIDIA H200 GPUs, respectively."},"train.accelerators":{"value":"6,144 H200","source":"report","at":"p.6 §3.1.1","quote":"LAGUNA M.1 and LAGUNA XS.2 were pre-trained on 6,144 and 2,048 NVIDIA H200 GPUs, respectively."},"train.parallelism":{"value":"EP8","source":"report","at":"p.7 §3.1.2","quote":"For MoE layers, we use Expert Parallelism, sharding expert weights across 8 H200 GPUs interconnected via NVLink.","note":"Titan's mesh is (PP, DDP, FSDP, EGP, ETP) for MoE layers with ETP = 1 for M.1 and XS.2; other degrees not stated."},"train.stability":{"value":"FP32 LM-head input-gradient all-reduce to fix positive logit drift; no z-loss","source":"report","at":"p.9 §3.1.3","quote":"We therefore enforce the LM head input-gradient all-reduce in FP32 while keeping the layer tensor-parallel, resolving this numerical instability.","note":"Introduced after M.1 logit drift; the report does not say at what point in pretraining. Same paragraph: \"no z-loss\"."},"train.masking":{"value":"document masking; position ids reset at document boundaries","source":"report","at":"p.6 §3.1.2 Model Factory component: Titan","quote":"resetting position indices at document boundaries to avoid RoPE-related rounding under document masking in attention"}},"not_disclosed":["mtp.depth","ctx.pretrain","ctx.final","train.lr_peak","train.lr_min","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":225795725568,"formula":"embedding + output head + all experts + other weights, poolside/Laguna-M.1-base@628037a safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":822083584,"formula":"input embedding + output head, poolside/Laguna-M.1-base@628037a safetensors headers"},"params.counted_non_embedding":{"value":224973641984,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, poolside/Laguna-M.1-base@628037a safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":3343917568,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, poolside/Laguna-M.1-base@628037a safetensors headers"},"params.counted_active_non_embedding":{"value":22640400944.0,"formula":"non-expert weights + routed experts × 16/256, excluding embedding and output head, poolside/Laguna-M.1-base@628037a safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":58.51428571428571,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":16.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.10363153232949512,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":286720.0,"formula":"286720 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":37580963840.0,"formula":"286720 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":286720.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":12252.991452991453,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1606024095.7264957,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":286720.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":286720.0,"tokens_per_entry":1}],"formula":"Global KV: 70 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"laguna-s-2-1","name":"Laguna-S-2.1","org":"Poolside","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"blog":{"url":"https://web.archive.org/web/20261009070156/https://poolside.ai/blog/introducing-laguna-s-2-1","kind":"blog"},"card":{"url":"https://huggingface.co/poolside/Laguna-S-2.1/resolve/0f573140834b11cfac0c2af97a101a7a69a13e22/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/poolside/Laguna-S-2.1/resolve/0f573140834b11cfac0c2af97a101a7a69a13e22/config.json","kind":"config"},"configuration_laguna":{"url":"https://huggingface.co/poolside/Laguna-S-2.1/resolve/0f573140834b11cfac0c2af97a101a7a69a13e22/configuration_laguna.py","kind":"code"},"modeling_laguna":{"url":"https://huggingface.co/poolside/Laguna-S-2.1/resolve/0f573140834b11cfac0c2af97a101a7a69a13e22/modeling_laguna.py","kind":"code"}},"facts":{"meta.released":{"value":"2026-07-21","source":"blog","at":"Introducing Laguna S 2.1; post date","quote":"category:\"announcements\",date:\"2026-07-21\""},"meta.license":{"value":"openmdw-1.1","source":"card","at":"README.md; YAML front matter","quote":"license: openmdw-1.1"},"meta.modality":{"value":"text","source":"card","at":"Model overview","quote":"Modality: text-to-text"},"params.active":{"value":8000000000,"source":"blog","at":"Introducing Laguna S 2.1; intro","quote":"Laguna S 2.1 is a 118B total parameter Mixture-of-Experts (MoE) model with 8B activated parameters per token","note":"The card says \"~8B activated\"."},"params.total":{"value":118000000000,"source":"card","at":"README.md; Model overview","quote":"Number of parameters: 118B total, ~8B activated per token"},"shape.d_model":{"value":3072,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100352,"source":"config","at":"/vocab_size"},"shape.tokenizer":{"value":"Laguna family tokenizer","source":"card","at":"README.md; Model overview","quote":"Vocabulary: 100,352 tokens (Laguna family tokenizer)"},"attn.gate":{"value":"softplus, per head, on attention output before o_proj","source":"card","at":"README.md; Model overview","quote":"Attention: grouped-query, 8 KV heads, head dim 128; per-head softplus output gating"},"attn.global_layers":{"value":12,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads","note":"Global layers; SWA layers use 72 (attn.local_heads)."},"attn.local_heads":{"value":72,"source":"config","at":"/num_attention_heads_per_layer/1","note":"Layer 1 is sliding_attention; every sliding_attention entry is 72, every full_attention entry 48. modeling_laguna.py LagunaDecoderLayer.__init__ passes the per-layer count to LagunaAttention."},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; intro","quote":"grouped-query attention, and interleaved full/sliding-window attention"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"1 global : 3 SWA","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\" ]","basis":"inferred","note":"The pinned /layer_types array repeats this four-layer pattern 12 times, beginning with the global layer. Ordered 48-layer backbone from /layer_types; auxiliary modules are excluded.","sequence":["GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA"]},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads","note":"LagunaAttention builds k_proj/v_proj from config.num_key_value_heads and config.head_dim for both layer types."},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim","note":"LagunaAttention uses config.head_dim for both layer types."},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_parameters/sliding_attention/rope_theta","note":"configuration_laguna.py derives swa_rope_parameters from rope_parameters[\"sliding_attention\"]; LagunaModel builds swa_rotary_emb from it."},"attn.local_layers":{"value":36,"source":"config","at":"/layer_types | count sliding_attention"},"attn.position":{"value":"partial RoPE (global, 50%), RoPE (SWA)","source":"config","at":"config.json; rope_parameters.full_attention","quote":"\"partial_rotary_factor\": 0.5","note":"rope_parameters.sliding_attention has partial_rotary_factor 1.0."},"attn.qk_norm":{"value":true,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_norm = LagunaRMSNorm(config.head_dim, eps=config.rms_norm_eps)"},"attn.rope_fraction":{"value":0.5,"source":"config","at":"/rope_parameters/full_attention/partial_rotary_factor","note":"Global layers; SWA layers rotate all dims (/rope_parameters/sliding_attention/partial_rotary_factor = 1.0)."},"attn.rope_theta":{"value":500000,"source":"config","at":"/rope_parameters/full_attention/rope_theta","note":"Global layers; SWA layers use 10,000."},"attn.v_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.bias":{"value":false,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, self.num_heads * config.head_dim, bias=False)"},"attn.window":{"value":512,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"configuration_laguna","at":"configuration_laguna.py; LagunaConfig.__init__","quote":"hidden_act: str = \"silu\"","note":"config.json sets no hidden_act; LagunaMLP computes down_proj(act_fn(gate_proj(x)) * up_proj(x))."},"ffn.bias":{"value":false,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":12288,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":1,"source":"config","at":"/mlp_only_layers | len","note":"Layer 0; decoder_sparse_step 1 makes every other layer MoE (LagunaDecoderLayer.__init__)."},"moe.expert_dim":{"value":1024,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.router":{"value":"sigmoid","source":"modeling_laguna","at":"modeling_laguna.py; LagunaTopKRouter","quote":"Laguna MoE router using sigmoid scoring (not softmax).","note":"The card's \"router with softplus gating\" conflicts with the code."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/moe_routed_scaling_factor"},"moe.shared":{"value":1,"source":"card","at":"README.md; Model overview","quote":"Experts: 256 routed (top-10) plus 1 shared expert"},"moe.shared_dim":{"value":1024,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":10,"source":"config","at":"/num_experts_per_tok"},"norm.placement":{"value":"pre","source":"modeling_laguna","at":"modeling_laguna.py; LagunaDecoderLayer.forward","quote":"residual = hidden_states hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_laguna","at":"modeling_laguna.py; LagunaDecoderLayer.__init__","quote":"self.input_layernorm = LagunaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.extension":{"value":"YaRN factor 128 over original 8,192 on global layers (release config)","source":"config","at":"config.json; rope_parameters.full_attention","quote":"\"rope_type\": \"yarn\", \"factor\": 128.0, \"original_max_position_embeddings\": 8192,","note":"Training-time extension stages are not disclosed. Same dict: beta_fast 32, beta_slow 1, attention_factor 1.485; max_position_embeddings 1,048,576."},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Model overview","quote":"Context window: 1,048,576 tokens"},"train.hardware":{"value":"4,096 H200; pretraining began 2026-05-22","source":"blog","at":"Introducing Laguna S 2.1; The bets we're making","quote":"Laguna S 2.1 began pre-training on 4,096 NVIDIA H200 GPUs on May 22, 2026, 60 days ago."},"train.accelerators":{"value":"4,096 H200","source":"blog","at":"Introducing Laguna S 2.1; The bets we're making","quote":"Laguna S 2.1 began pre-training on 4,096 NVIDIA H200 GPUs on May 22, 2026, 60 days ago."},"train.data":{"value":"same pretraining data as XS 2.1","source":"blog","at":"Introducing Laguna S 2.1; A strong base, then new post-training","quote":"Laguna S 2.1 is a scale-up of the Laguna XS family, trained on exactly the same pre-training data as XS 2.1"},"train.cutoff":{"value":"November 2025","source":"blog","at":"Introducing Laguna S 2.1; Case study 3","quote":"the model has a knowledge cutoff date in November 2025"}},"not_disclosed":["params.active_non_embedding","moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":117561977600,"formula":"embedding + output head + all experts + other weights, poolside/Laguna-S-2.1@0f57314 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":616562688,"formula":"input embedding + output head, poolside/Laguna-S-2.1@0f57314 safetensors headers"},"params.counted_non_embedding":{"value":116945414912,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, poolside/Laguna-S-2.1@0f57314 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":2489285120,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, poolside/Laguna-S-2.1@0f57314 safetensors headers"},"params.counted_active_non_embedding":{"value":7832681942.0,"formula":"non-expert weights + routed experts × 10/256, excluding embedding and output head, poolside/Laguna-S-2.1@0f57314 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":64.0,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":25.6,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.06779661016949153,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":49152.0,"formula":"49152 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6517948416.0,"formula":"49152 B × 131072 (Global KV) + 147456 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":49728.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":6216.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":814743552.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":49152.0,"windows":[[147456.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":49152.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":147456.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 36 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"laguna-xs-2-1","name":"Laguna-XS-2.1","org":"Poolside","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"blog":{"url":"https://web.archive.org/web/20260920153810/https://poolside.ai/blog/introducing-laguna-xs-2-1","kind":"blog"},"card":{"url":"https://huggingface.co/poolside/Laguna-XS-2.1/resolve/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/poolside/Laguna-XS-2.1/resolve/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/config.json","kind":"config"},"configuration_laguna":{"url":"https://huggingface.co/poolside/Laguna-XS-2.1/resolve/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/configuration_laguna.py","kind":"code"},"modeling_laguna":{"url":"https://huggingface.co/poolside/Laguna-XS-2.1/resolve/c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db/modeling_laguna.py","kind":"code"}},"facts":{"meta.released":{"value":"2026-07-02","source":"blog","at":"Introducing Laguna XS 2.1; post date","quote":"category:\"announcements\",date:\"2026-07-02\""},"meta.license":{"value":"openmdw-1.1","source":"card","at":"README.md; YAML front matter","quote":"license: openmdw-1.1"},"meta.modality":{"value":"text","source":"card","at":"Model overview","quote":"Modality: text-to-text"},"params.active":{"value":3000000000,"source":"blog","at":"Introducing Laguna XS 2.1; intro","quote":"Laguna XS 2.1 is a 33B total parameter Mixture-of-Experts model with 3B activated parameters per token"},"params.total":{"value":33000000000,"source":"card","at":"README.md; Model overview","quote":"Number of parameters: 33B total with 3B activated per token"},"shape.d_model":{"value":2048,"source":"config","at":"/hidden_size"},"shape.layers":{"value":40,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100352,"source":"config","at":"/vocab_size"},"attn.gate":{"value":"softplus, per head, on attention output before o_proj","source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.forward","quote":"gate = F.softplus(self.g_proj(hidden_states).float()).to(attn_output.dtype)","note":"Per head: config /gating = \"per-head\". The card says \"sigmoid gating\"; the code uses softplus."},"attn.global_layers":{"value":10,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads","note":"Global layers; SWA layers use 64 (attn.local_heads)."},"attn.local_heads":{"value":64,"source":"config","at":"/num_attention_heads_per_layer/1","note":"Layer 1 is sliding_attention; every sliding_attention entry is 64, every full_attention entry 48. modeling_laguna.py LagunaDecoderLayer.__init__ passes the per-layer count to LagunaAttention."},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; Highlights","quote":"enabling mixed SWA (Sliding Window Attention) and global attention layers in a 3:1 ratio","note":"GQA: 8 KV heads for 48/64 Q-heads (config)."},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"1 global : 3 SWA","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\" ]","basis":"inferred","note":"The pinned /layer_types array repeats this four-layer pattern 10 times, beginning with the global layer. Ordered 40-layer backbone from /layer_types; auxiliary modules are excluded.","sequence":["GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA"]},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads","note":"LagunaAttention builds k_proj/v_proj from config.num_key_value_heads and config.head_dim for both layer types."},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim","note":"LagunaAttention uses config.head_dim for both layer types."},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_parameters/sliding_attention/rope_theta","note":"configuration_laguna.py derives swa_rope_parameters from rope_parameters[\"sliding_attention\"]; LagunaModel builds swa_rotary_emb from it."},"attn.local_layers":{"value":30,"source":"config","at":"/layer_types | count sliding_attention"},"attn.position":{"value":"partial RoPE (global, 50%), RoPE (SWA)","source":"config","at":"config.json; rope_parameters.full_attention","quote":"\"partial_rotary_factor\": 0.5","note":"rope_parameters.sliding_attention has partial_rotary_factor 1.0."},"attn.qk_norm":{"value":true,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_norm = LagunaRMSNorm(config.head_dim, eps=config.rms_norm_eps)"},"attn.rope_fraction":{"value":0.5,"source":"config","at":"/rope_parameters/full_attention/partial_rotary_factor","note":"Global layers; SWA layers rotate all dims (/rope_parameters/sliding_attention/partial_rotary_factor = 1.0)."},"attn.rope_theta":{"value":500000,"source":"config","at":"/rope_parameters/full_attention/rope_theta","note":"Global layers; SWA layers use 10,000."},"attn.v_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.bias":{"value":false,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, self.num_heads * config.head_dim, bias=False)"},"attn.window":{"value":512,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"configuration_laguna","at":"configuration_laguna.py; LagunaConfig.__init__","quote":"hidden_act: str = \"silu\"","note":"config.json sets no hidden_act; LagunaMLP computes down_proj(act_fn(gate_proj(x)) * up_proj(x))."},"ffn.bias":{"value":false,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":8192,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":1,"source":"config","at":"/mlp_only_layers | len","note":"Layer 0; decoder_sparse_step 1 makes every other layer MoE (LagunaDecoderLayer.__init__)."},"moe.expert_dim":{"value":512,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.router":{"value":"sigmoid","source":"modeling_laguna","at":"modeling_laguna.py; LagunaTopKRouter","quote":"Laguna MoE router using sigmoid scoring (not softmax)."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/moe_routed_scaling_factor"},"moe.shared":{"value":1,"source":"card","at":"README.md; Model overview","quote":"Experts: 256 experts with 1 shared expert"},"moe.shared_dim":{"value":512,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"norm.placement":{"value":"pre","source":"modeling_laguna","at":"modeling_laguna.py; LagunaDecoderLayer.forward","quote":"residual = hidden_states hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_laguna","at":"modeling_laguna.py; LagunaDecoderLayer.__init__","quote":"self.input_layernorm = LagunaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.extension":{"value":"YaRN factor 32 over original 8,192 on global layers (release config)","source":"config","at":"config.json; rope_parameters.full_attention","quote":"\"rope_type\": \"yarn\", \"factor\": 32.0, \"original_max_position_embeddings\": 8192,","note":"Training-time extension stages are not disclosed. Same dict: beta_fast 64, beta_slow 1, attention_factor 1.347; max_position_embeddings 262,144."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Model overview","quote":"Context window: 262,144 tokens"},"train.optimizer":{"value":"Muon","source":"card","at":"README.md; Model overview","quote":"Optimizer: Muon"}},"not_disclosed":["params.active_non_embedding","moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":33442617088,"formula":"embedding + output head + all experts + other weights, poolside/Laguna-XS-2.1@c5f3626 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":411041792,"formula":"input embedding + output head, poolside/Laguna-XS-2.1@c5f3626 safetensors headers"},"params.counted_non_embedding":{"value":33031575296,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, poolside/Laguna-XS-2.1@c5f3626 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":846860800,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, poolside/Laguna-XS-2.1@c5f3626 safetensors headers"},"params.counted_active_non_embedding":{"value":2606084408.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, poolside/Laguna-XS-2.1@c5f3626 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":51.2,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.09090909090909091,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":40960.0,"formula":"40960 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":5431623680.0,"formula":"40960 B × 131072 (Global KV) + 122880 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":41440.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":13813.333333333334,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1810541226.6666667,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":40960.0,"windows":[[122880.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":40960.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":122880.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"laguna-xs-2","name":"Laguna-XS.2","org":"Poolside","country":"United States","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/poolside/Laguna-XS.2/resolve/a397bde04501591d0ac3d74a5d9d0a4ec298ffb8/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/poolside/Laguna-XS.2/resolve/a397bde04501591d0ac3d74a5d9d0a4ec298ffb8/config.json","kind":"config"},"configuration_laguna":{"url":"https://huggingface.co/poolside/Laguna-XS.2/resolve/a397bde04501591d0ac3d74a5d9d0a4ec298ffb8/configuration_laguna.py","kind":"code"},"modeling_laguna":{"url":"https://huggingface.co/poolside/Laguna-XS.2/resolve/a397bde04501591d0ac3d74a5d9d0a4ec298ffb8/modeling_laguna.py","kind":"code"},"report":{"url":"https://web.archive.org/web/20261004102254id_/https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20260903084301/https://poolside.ai/blog/introducing-laguna-xs2-m1","kind":"blog"}},"facts":{"meta.released":{"value":"2026-04-28","source":"blog","at":"Introducing Laguna XS.2 and Laguna M.1; post date and Get started","quote":"category:\"announcements\",date:\"2026-04-28\"","note":"Same post: \"Weights are live today under Apache 2.0.\""},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","source":"card","at":"Model overview","quote":"Modality: text-to-text"},"params.active":{"value":3000000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"with 3B parameters (including embeddings) activated per token"},"params.active_non_embedding":{"value":2590000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"with 3B parameters (including embeddings) activated per token","basis":"inferred","note":"~3B − 2 × 100,352 × 2,048 (input embedding and untied LM head) = ~2.59B."},"params.note":{"value":"Active count includes embeddings.","source":"report","at":"p.5 §3.1 Architecture & Training","quote":"with 3B parameters (including embeddings) activated per token"},"params.total":{"value":33400000000,"source":"report","at":"p.5 §3.1 Architecture & Training","quote":"LAGUNA XS.2 has a total parameter count of 33.4B"},"shape.d_model":{"value":2048,"source":"config","at":"/hidden_size"},"shape.layers":{"value":40,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100352,"source":"config","at":"/vocab_size"},"shape.tokenizer":{"value":"BPE trained on internal data; shared with M.1","source":"report","at":"p.5 §3.1","quote":"Both models share the same tokenizer with a vocabulary size of 100,352 tokens, trained with Byte-Pair Encoding [75] on our internal datasets."},"attn.gate":{"value":"softplus, per head, on attention output before o_proj","source":"report","at":"p.5 §3.1","quote":"we use Grouped Query Attention (GQA) [3] with 8 KV heads, a head dimension of 128, and softplus-based per-head gating [67].","note":"The model card says \"sigmoid gating\"; the report and released code (F.softplus on a per-head g_proj) use softplus. Config /gating (true) is not read by this modeling_laguna.py."},"attn.global_layers":{"value":10,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads","note":"Global layers; SWA layers use 64 (attn.local_heads)."},"attn.local_heads":{"value":64,"source":"config","at":"/num_attention_heads_per_layer/1","note":"Layer 1 is sliding_attention; every sliding_attention entry is 64, every full_attention entry 48. modeling_laguna.py LagunaDecoderLayer.__init__ passes the per-layer count to LagunaAttention."},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.5 §3.1","quote":"In both types of layers, we use Grouped Query Attention (GQA)"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"1 global : 3 SWA","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\" ]","basis":"inferred","note":"The pinned /layer_types array repeats this four-layer pattern 10 times, beginning with the global layer. Ordered 40-layer backbone from /layer_types; auxiliary modules are excluded.","sequence":["GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA"]},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads","note":"LagunaAttention builds k_proj/v_proj from config.num_key_value_heads and config.head_dim for both layer types."},"attn.local_head_dim":{"value":128,"source":"report","at":"p.5 §3.1","quote":"In both types of layers, we use Grouped Query Attention (GQA) [3] with 8 KV heads, a head dimension of 128"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_parameters/sliding_attention/rope_theta"},"attn.local_layers":{"value":30,"source":"config","at":"/layer_types | count sliding_attention"},"attn.position":{"value":"partial RoPE (global, first 50%), RoPE (SWA)","source":"report","at":"p.5 §3.1","quote":"GA layers use 48 Q-heads, θ = 500,000, and partial RoPE applied to the first 50% of the head dimension"},"attn.qk_norm":{"value":true,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaAttention.__init__","quote":"self.q_norm = LagunaRMSNorm(self.head_dim, eps=config.rms_norm_eps)"},"attn.rope_fraction":{"value":0.5,"source":"config","at":"/rope_parameters/full_attention/partial_rotary_factor","note":"Global layers; SWA layers rotate all dims (/rope_parameters/sliding_attention/partial_rotary_factor = 1.0)."},"attn.rope_theta":{"value":500000,"source":"config","at":"/rope_parameters/full_attention/rope_theta","note":"Global layers; SWA layers use 10,000."},"attn.v_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.window":{"value":512,"source":"config","at":"/sliding_window"},"ffn.dim":{"value":8192,"source":"config","at":"/intermediate_size"},"ffn.activation":{"value":"SwiGLU","source":"configuration_laguna","at":"configuration_laguna.py; LagunaConfig defaults","quote":"hidden_act: str = \"silu\"","note":"config.json sets no hidden_act; LagunaMLP computes down_proj(act_fn(gate_proj(x)) * up_proj(x))."},"ffn.bias":{"value":false,"source":"modeling_laguna","at":"modeling_laguna.py; LagunaMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.balancing":{"value":"global-batch auxiliary loss (Qiu et al.) over non-padding tokens; padding tokens skip routing","source":"report","at":"p.5 §3.1","quote":"For load-balancing, we use an auxiliary loss from Qiu et al. [66] across the non-padding tokens of the global batch.","note":"Coefficient not disclosed. Padding skip: §3.1.3."},"moe.dense_layers":{"value":1,"source":"config","at":"/mlp_layer_types | count dense"},"moe.expert_dim":{"value":512,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.router":{"value":"sigmoid","source":"report","at":"p.5 §3.1","quote":"We use a linear layer followed by a sigmoid activation as the routing function"},"moe.norm_topk":{"value":true,"source":"report","at":"p.5 §3.1","quote":"apply normalization to the scores after top-k"},"moe.routed_scale":{"value":2.5,"source":"report","at":"p.5 §3.1","quote":"The output of the routed experts is modulated by a coefficient of 2.5 before combining with the output of the shared expert.","note":"Added in XS.2; M.1 has no routed-expert modulation (config /moe_routed_scaling_factor 2.5 here)."},"moe.shared":{"value":1,"source":"report","at":"p.5 §3.1","quote":"plus a shared expert [20] that processes every token"},"moe.shared_dim":{"value":512,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"norm.placement":{"value":"pre","source":"report","at":"p.5 §3.1","quote":"using a pre-norm Transformer architecture"},"norm.type":{"value":"RMSNorm","source":"report","at":"p.5 §3.1","quote":"with RMSNorm [103] as the normalization layer"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.extension":{"value":"YaRN on global layers only; 4K → 32K → 128K (100B tokens each); 256K by doubling the global RoPE scale without training","source":"report","at":"p.6 §3.1.1 Long-Context Training","quote":"Both sub-stages apply YaRN [62] to global attention layers only","note":"Released config /rope_parameters/full_attention: rope_type yarn, factor 64, original_max_position_embeddings 4,096, beta_fast 64, beta_slow 1, attention_factor 1.416; max_position_embeddings 262,144."},"ctx.final":{"value":131072,"source":"report","at":"p.6 §3.1.1 Long-Context Training","quote":"the second extends the context to 128K tokens"},"ctx.max":{"value":262144,"source":"card","at":"README.md; Model overview","quote":"Context window: 262,144 tokens"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.6 §3.1.1","quote":"The pre-training stage uses a context length of 4K tokens"},"train.batch":{"value":"24M tokens","source":"report","at":"p.6 §3.1.1 Peak Learning Rate from a WSD Scaling Law","quote":"trains at B = 24M tokens","note":"Long-context stages also use 24M tokens."},"train.batch_tokens":{"value":24000000,"source":"report","at":"p.6 §3.1.1 Peak Learning Rate from a WSD Scaling Law","quote":"trains at B = 24M tokens"},"train.lr_min":{"value":2.5e-05,"source":"report","at":"p.6 §3.1.1","quote":"ends at 5% of the peak (2.5×10−5 )","note":"End of the main decay; long-context stages then decay to 5e-6."},"train.lr_peak":{"value":0.0005,"source":"report","at":"p.6 §3.1.1","quote":"linear warmup to a peak learning rate of 5×10−4","note":"WSD LR scaling law predicts ~5.5e-4; 5e-4 used for margin."},"train.optimizer":{"value":"Muon + AdamW (Moonlight variant; AdamW for non-matrix parameters)","source":"report","at":"p.6 §3.1.1","quote":"We use the Muon optimizer [39], specifically the Moonlight variant of Liu et al. [49], across all training stages","note":"AdamW for non-matrix parameters with Moonlight-style LR scaling: §3.1.3."},"train.precision":{"value":"BF16 mixed precision, FP32 master weights; selective FP32 in RMSNorm, RoPE and the LM-head input-gradient all-reduce","source":"report","at":"p.6 §3.1.1","quote":"We use BF16 mixed precision training throughout all stages with the master weights in FP32 and most computations in BF16"},"train.schedule":{"value":"WSD: linear warmup, stable at peak, 1−√ decay over the last 30% of steps to 5% of peak","source":"report","at":"p.6 §3.1.1","quote":"a final decay phase covering the last 30% of training steps that follows a 1−√⋅ shape and ends at 5% of the peak"},"train.stages":{"value":[{"name":"main","seq_len":4096,"lr":0.0005,"batch_tokens":24000000,"note":"WSD; tokens not disclosed","quote":"The pre-training stage uses a context length of 4K tokens"},{"name":"extend-32K","tokens":100000000000,"seq_len":32768,"lr":2.5e-05,"batch_tokens":24000000,"note":"cosine 2.5e-5 → 5e-6, no re-warmup; YaRN on global layers","quote":"split it into two equal-token sub-stages of 100B tokens each. The first sub-stage extends the context to 32K tokens"},{"name":"extend-128K","tokens":100000000000,"seq_len":131072,"lr":2.5e-05,"batch_tokens":24000000,"note":"cosine 2.5e-5 → 5e-6; EMA of the last 10 checkpoints is the final base","quote":"the second extends the context to 128K tokens"}],"source":"report","at":"p.6 §3.1.1"},"train.tokens":{"value":">30T","source":"report","at":"p.9 §3.2","quote":"In total, both models were trained on more than 30T tokens."},"train.data":{"value":"AutoMixer mix: raw code 30.6%, web 25.2%, synthetic/code-text 25.4%, math 9.0%, knowledge 6.6%, instruction-like 1.4%, academic papers 1.1%, books 0.7%; sampled from a ~27T unique-token pool","source":"report","at":"p.16 Table 4","quote":"Raw code 30.6% Web 25.2% Synthetic/code-text 25.4% Math 9.0% Knowledge 6.6% Instruction-like 1.4% Academic papers 1.1% Books 0.7%","note":"Pool size: §3.2. Synthetic data is ~13% of the mix across all stages, from ~4.4T generated tokens (§3.2.2)."},"train.hardware":{"value":"2,048 H200","source":"report","at":"p.6 §3.1.1","quote":"LAGUNA M.1 and LAGUNA XS.2 were pre-trained on 6,144 and 2,048 NVIDIA H200 GPUs, respectively."},"train.accelerators":{"value":"2,048 H200","source":"report","at":"p.6 §3.1.1","quote":"LAGUNA M.1 and LAGUNA XS.2 were pre-trained on 6,144 and 2,048 NVIDIA H200 GPUs, respectively."},"train.parallelism":{"value":"EP8","source":"report","at":"p.7 §3.1.2","quote":"For MoE layers, we use Expert Parallelism, sharding expert weights across 8 H200 GPUs interconnected via NVLink.","note":"Titan's mesh is (PP, DDP, FSDP, EGP, ETP) for MoE layers with ETP = 1 for M.1 and XS.2; other degrees not stated."},"train.stability":{"value":"FP32 LM-head input-gradient all-reduce; no z-loss; padding tokens skip routing and load balancing","source":"report","at":"p.9 §3.1.3","quote":"We therefore enforce the LM head input-gradient all-reduce in FP32 while keeping the layer tensor-parallel, resolving this numerical instability.","note":"Same section states \"no z-loss\" and the padding-token routing skip added for XS.2."},"train.masking":{"value":"document masking; position ids reset at document boundaries","source":"report","at":"p.6 §3.1.2 Model Factory component: Titan","quote":"resetting position indices at document boundaries to avoid RoPE-related rounding under document masking in attention"},"train.averaging":{"value":"EMA over the 10 most recent checkpoints at the end of the 128K stage","source":"report","at":"p.6 §3.1.1 Long-Context Training","quote":"we apply an exponential moving average (EMA) over the 10 most recent checkpoints from the end of the 128K stage and use the EMA weights as the final base checkpoint."},"scaling.hp_rule":{"value":"peak LR from WSD scaling law lr⋆(N, D) = 10^4.488 · N^−0.4639 · D^−0.2661 (N active params, D tokens incl. cooldown), scaled by √(B/B0) from B0 = 8M; predicts ~5.5e-4 at N = 3.0B, B = 24M; 5e-4 used","source":"report","at":"p.6 §3.1.1 Peak Learning Rate from a WSD Scaling Law, Eq. 1","quote":"To pick the peak LR for LAGUNA XS.2 at production scale, we therefore fit a WSD-specific scaling law"},"scaling.ladder":{"value":"LR-law sweep: 4 MoE sizes (2B–16B total / 0.3B–2.2B active) × 6 LRs × 5 token budgets (30B–480B) at 8M batch; architecture ablations on a 16.6B-total / 2.3B-active proxy (22 layers, d_model 2048, 128 experts top-8, 335B tokens); data mixture from ~60 × ~0.5B MoE proxies on ~60B tokens","source":"report","at":"p.6 §3.1.1; App. A.1–A.2, Table 8; §3.2.3 AutoMixer","quote":"sweeping four MoE sizes (2B–16B total / 0.3B–2.2B active) across six learning rates and five token budgets","note":"Appendix A.1 instead lists D ∈ {60, 120, 240, 480, 960}B and B0 = 8.4M tokens."}},"not_disclosed":["mtp.depth","train.warmup","train.weight_decay","train.betas","train.grad_clip"],"derived":{"params.counted_total":{"value":33442617088,"formula":"embedding + output head + all experts + other weights, poolside/Laguna-XS.2@a397bde safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":411041792,"formula":"input embedding + output head, poolside/Laguna-XS.2@a397bde safetensors headers"},"params.counted_non_embedding":{"value":33031575296,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, poolside/Laguna-XS.2@a397bde safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":846860800,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, poolside/Laguna-XS.2@a397bde safetensors headers"},"params.counted_active_non_embedding":{"value":2606084408.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, poolside/Laguna-XS.2@a397bde safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":51.2,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.08982035928143713,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":40960.0,"formula":"40960 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":5431623680.0,"formula":"40960 B × 131072 (Global KV) + 122880 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":41440.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":13813.333333333334,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1810541226.6666667,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":40960.0,"windows":[[122880.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":40960.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":122880.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 10 × (2048 × 16/8) B/entry; SWA KV: 30 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"ling-2-6-1t","name":"Ling-2.6-1T-base","org":"inclusionAI","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/inclusionAI/Ling-2.6-1T-base/resolve/64beeb8a048920769d206e52ae303a81779296ec/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/inclusionAI/Ling-2.6-1T-base/resolve/64beeb8a048920769d206e52ae303a81779296ec/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/inclusionAI/Ling-2.6-1T-base/resolve/64beeb8a048920769d206e52ae303a81779296ec/model.safetensors.index.json","kind":"config"},"modeling_bailing_moe_v2_5":{"url":"https://huggingface.co/inclusionAI/Ling-2.6-1T-base/resolve/64beeb8a048920769d206e52ae303a81779296ec/modeling_bailing_moe_v2_5.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2606.15079v1","kind":"paper"},"hf-api":{"url":"https://huggingface.co/api/models/inclusionAI/Ling-2.6-1T-base","kind":"api"},"aa":{"url":"https://web.archive.org/web/20261019102254/https://artificialanalysis.ai/models/ling-2-6-1t","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-06-02","source":"hf-api","at":"/createdAt","quote":"2026-06-02T","note":"Hugging Face repository creation date; no dated publisher page for the base weights (report: arXiv 13 Jun 2026)."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","source":"report","at":"p.33 §5 Conclusion, Limitations, and Future Directions","quote":"An equally important transition is from text-only systems to native multimodal agents.","basis":"inferred","note":"The report names native multimodality as future work; config declares no vision or audio tower and the card tags the checkpoint text-generation."},"params.active":{"value":63000000000,"source":"card","at":"README.md; 3. Model Summary","quote":"Totoal ~1T, Activated ~63B"},"params.note":{"value":"Approximate (~) card figures; whether the MTP layer is counted is not stated.","source":"card","at":"README.md; 3. Model Summary","quote":"Totoal ~1T, Activated ~63B"},"params.total":{"value":1000000000000,"source":"card","at":"README.md; 3. Model Summary","quote":"Totoal ~1T, Activated ~63B"},"shape.d_model":{"value":8192,"source":"report","at":"p.5 §2.1.1; config /hidden_size","quote":"We set the number of Transformer layers to 80 and the hidden dimension d to 8,192."},"shape.layers":{"value":80,"source":"report","at":"p.5 §2.1.1; config /num_hidden_layers","quote":"We set the number of Transformer layers to 80 and the hidden dimension d to 8,192."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":157184,"source":"report","at":"p.5 §2.1.1; config /vocab_size","quote":"The model uses a vocabulary size of 157,184"},"attn.bias":{"value":false,"source":"config","at":"/use_qkv_bias","note":"Read for MLA q_a/kv_a and Lightning QKV projections; output projections use /use_bias (false)."},"attn.gate":{"value":"sigmoid gate after group RMSNorm, before the output projection, on Lightning layers; none on MLA layers","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5LinearAttention.forward","quote":"o = o * torch.sigmoid_(g_proj)","note":"Group RMSNorm over 8 groups (/group_norm_size). MLA layers project straight to the output (no g_proj in BailingMoeV2_5MultiLatentAttention)."},"attn.global_layers":{"value":10,"source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5DecoderLayer.__init__","quote":"if (layer_idx + 1) % config.layer_group_size == 0","basis":"inferred","note":"MLA at layers 7, 15, …, 79: 80 / /layer_group_size 8."},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim","note":"MLA layers. /head_dim=128 is the Lightning Attention head size."},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"Lightning, MLA","source":"report","at":"p.5 §2.1","quote":"design a hybrid attention architecture combining Lightning Attention with MLA in a 7:1 ratio"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"7 Lightning : 1 MLA","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; DecoderLayer.__init__","quote":"if (layer_idx + 1) % config.layer_group_size == 0","note":"Ordered from that conditional with /layer_group_size=8; the implementation also makes any incomplete final group global.","basis":"inferred","sequence":["Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","MLA"]},"attn.linear_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.linear_heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.linear_kind":{"value":"Lightning Attention: 64 heads × 128, fixed per-head ALiBi-style decay slopes scaled down with depth, per-head QK RMSNorm, partial RoPE (64 of 128 dims), group-RMSNorm + sigmoid output gate","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5LinearAttention.build_slope_tensor","quote":"Build a tensor of slopes for Lightning Attention-2"},"attn.linear_layers":{"value":70,"source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5DecoderLayer.__init__","quote":"if (layer_idx + 1) % config.layer_group_size == 0","basis":"inferred","note":"80 layers minus 10 MLA layers (every 8th)."},"attn.position":{"value":"partial RoPE (MLA, decoupled 64 of 192 QK dims, interleaved), partial RoPE (Lightning, 64 of 128 dims)","source":"report","at":"p.5 §2.1.1","note":"Lightning rotary width = head_dim 128 × partial_rotary_factor 0.5.","quote":"The rotary dimension is set to 64, i.e., Partial RoPE is applied to a subset of head dimensions."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"report","at":"p.7 §2.1.3","quote":"We resolve this by removing QK Norm prior to the MLA conversion.","note":"MLA layers: Ling-2.0 QK Norm was fused into the projections before conversion. Lightning layers keep per-head QK RMSNorm (use_qk_norm, read only by BailingMoeV2_5LinearAttention)."},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":6000000,"source":"report","at":"p.5 §2.1.1; config /rope_theta","quote":"via RoPE (Su et al., 2024) positional encoding with θ = 6,000,000."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn = ACT2FN[/hidden_act] = silu; report: \"We employ SiLU as the activation function\"."},"ffn.dim":{"value":18432,"source":"report","at":"p.5 §2.1.1; config /intermediate_size","quote":"For the first 4 Transformer blocks, we use dense FFN layers with an intermediate size of 18,432 rather than MoE layers."},"moe.balancing":{"value":"aux-loss-free expert bias, update rate γ=0.001 in continued pretraining, 0.0001 in mid-training","source":"report","at":"p.9 §2.3.1; config /moe_router_enable_expert_bias, /topk_method noaux_tc","quote":"γ=0.001 during continue pre-training, which is then reduced to 0.0001 for mid-training."},"moe.dense_layers":{"value":4,"source":"report","at":"p.5 §2.1.1; config /first_k_dense_replace","quote":"For the first 4 Transformer blocks, we use dense FFN layers with an intermediate size of 18,432 rather than MoE layers."},"moe.expert_dim":{"value":2048,"source":"report","at":"p.5 §2.1.1; config /moe_intermediate_size","quote":"the intermediate hidden dimension of each routed expert is 2,048."},"moe.experts":{"value":256,"source":"report","at":"p.5 §2.1.1; config /num_experts, /n_routed_experts","quote":"Each MoE layer consists of 1 shared expert and 256 routed experts"},"moe.groups":{"value":"8 groups, top-4 groups per token","source":"report","at":"p.5 §2.1.1; config /n_group, /topk_group","quote":"We adopt a grouped routing strategy with ngroup = 8 groups and top-4 group selection"},"moe.norm_topk":{"value":true,"source":"report","at":"p.5 §2.1.1; config /norm_topk_prob","quote":"the routed output is scaled by a factor of 2.5 with normalized top-k probabilities."},"moe.routed_scale":{"value":2.5,"source":"report","at":"p.5 §2.1.1; config /routed_scaling_factor","quote":"the routed output is scaled by a factor of 2.5 with normalized top-k probabilities."},"moe.router":{"value":"sigmoid","source":"report","at":"p.5 §2.1.1; config /score_function, /scoring_func, /router_dtype fp32","quote":"where the router operates in FP32 precision with sigmoid scoring."},"moe.shared":{"value":1,"source":"report","at":"p.5 §2.1.1; config /num_shared_experts","quote":"Each MoE layer consists of 1 shared expert and 256 routed experts"},"moe.shared_dim":{"value":2048,"source":"config","at":"/moe_shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"report","at":"p.5 §2.1.1; config /num_experts_per_tok","quote":"Among the routed experts, 8 experts are activated for each token."},"mtp.depth":{"value":1,"source":"report","at":"p.42 App. B","quote":"We therefore incorporate two additional MTP layers during the post-training stage","basis":"inferred","note":"App. B contrasts the standard single MTP layer (MTP-1) used before post-training with the two layers added after it; pretraining used MTP loss weight 0.1 (§2.3.1). Base config /num_nextn_predict_layers is 1."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.block":{"value":"RMSNorm(embedding of token t+1) and RMSNorm(hidden) concatenated, eh_proj 2d→d, then one MLA + MoE block with final RMSNorm","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5MTPLayer","quote":"hidden_states = self.eh_proj(torch.cat([input_embeds, hidden_states], dim=-1))"},"mtp.ffn":{"value":"MoE","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5MTPLayer.__init__","quote":"self.mlp = BailingMoeV2_5SparseMoeBlock(config)"},"mtp.shares_embedding":{"value":true,"source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5ForCausalLM.forward","quote":"mtp_logits = self.lm_head(mtp_hidden_states).float()","note":"Input embedding is also the shared word_embeddings (BailingMoeV2_5Model.forward)."},"mtp.loss_weight":{"value":"0.1","source":"report","at":"p.9 §2.3.1","quote":"The MTP loss weight is set to 0.1."},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.80.eh_proj.weight\""},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_bailing_moe_v2_5","at":"modeling_bailing_moe_v2_5.py; BailingMoeV2_5DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"report","at":"p.5 §2.1.1","quote":"We employ SiLU as the activation function and RMSNorm"},"ctx.extension":{"value":"Mid-training: 4K → 32K (250B, 20% of sequences at 32K) → 32K (425B) → 256K (525B)","source":"report","at":"p.10 §2.3.2","quote":"The third phase covers 525B tokens and extends the context window to 256K"},"ctx.final":{"value":262144,"source":"report","at":"p.10 §2.3.2","quote":"The third phase covers 525B tokens and extends the context window to 256K"},"ctx.max":{"value":262144,"source":"report","at":"p.5 §2.1.1; config /max_position_embeddings","quote":"supports a maximum context length of 262,144 tokens"},"ctx.pretrain":{"value":4096,"source":"report","at":"p.10 §2.3.2","quote":"all parameters are unfrozen for full continued training on 8T tokens with a 4K context window."},"train.averaging":{"value":"checkpoint merging in place of LR decay (WSM)","source":"report","at":"p.9 §2.3.1","quote":"the final annealing effect is achieved through checkpoint merging."},"train.data":{"value":"Continued pretraining: ~46% reasoning (math, code), 50% general, 4% multilingual; multilingual web data dropped in mid-training. Adds 1.1T Fineweb2/Fineweb2-hq and ~70B synthetic web-code tokens; agentic trajectory corpus","source":"report","at":"p.10 §2.3.2","quote":"The data mixture allocates approximately 46% to reasoning-intensive domains (e.g., mathematics and code), 50% to general corpora (e.g., web text), and 4% to multilingual data.","note":"Fineweb2 and synthetic web-code figures: p.8 §2.2.2; multilingual mid-training mix: p.9."},"train.init":{"value":"Continued from Ling-2.0-1T-base (20T tokens); GQA converted to Lightning Attention (new params random) and to MLA via TransMLA after fusing QK Norm into the projections","source":"report","at":"p.9 §2.3.2","quote":"Starting from the Ling-2.0 base checkpoint, this stage smoothly transitions the architecture from GQA-based Softmax Attention to the hybrid linear attention with MLA"},"train.precision":{"value":"FP8, with FP32 optimizer master weights","source":"report","at":"p.29 §4.2","quote":"Since Ling-2.6 uses FP8 training from pretraining through supervised fine-tuning, FP32 optimizer master weights retain higher-precision information"},"train.schedule":{"value":"WSM: linear warmup, then constant LR to the end; no decay phase","source":"report","at":"p.9 §2.3.1","quote":"We continue to use the WSM scheduler from Ling-2.0: a linear warmup to a peak learning rate, followed by a constant phase until training concludes","note":"Mid-training phase 1 explicitly keeps the peak LR; other hyper-parameters are inherited from Ling-2.0 and not restated."},"train.stages":{"value":[{"name":"migration","tokens":400000000000,"seq_len":4096,"note":"Four steps: Lightning conversion, linear warmup (partial unfreeze), MLA conversion, MLA warmup; all at 4K (Figure 4).","quote":"It proceeds in four steps over approximately 400B tokens."},{"name":"continued","tokens":8000000000000,"seq_len":4096,"quote":"all parameters are unfrozen for full continued training on 8T tokens with a 4K context window."},{"name":"mid-1","tokens":250000000000,"seq_len":32768,"note":"20% of sequences at 32K; LR stays at the pretraining peak.","quote":"The first phase trains on 250B tokens, sampling 20% of sequences at 32K length"},{"name":"mid-2","tokens":425000000000,"seq_len":32768,"quote":"The second phase spans 425B tokens at 32K context length"},{"name":"mid-3","tokens":525000000000,"seq_len":262144,"quote":"The third phase covers 525B tokens and extends the context window to 256K"}],"source":"report","at":"p.9–10 §2.3.2"},"train.tokens":{"value":9600000000000,"source":"report","at":"p.9 §2.3","quote":"processes approximately 9.6T tokens across three stages.","note":"Continued-pretraining tokens on top of Ling-2.0-1T-base's 20T; approximate."},"scaling.hp_rule":{"value":"LR and batch size from Ling scaling laws re-fit on the hybrid linear-attention architecture for the total training budget; other HPs from Ling-2.0","source":"report","at":"p.9 §2.3.1","quote":"we first re-establish the Ling scaling laws on the hybrid linear attention architecture to determine the appropriate learning rate and batch size given the total training budget"},"scaling.ladder":{"value":"equal-FLOPs scaling-law runs over hybrid group size M ∈ {2, 4, 8, 16} (1 MLA + M−1 Lightning per group); model sizes not given","source":"report","at":"p.5 §2.1.2, Figure 3, Table 2","quote":"We define the layer group size M such that each group contains 1 Full Attention (MLA) layer and M − 1 Linear Attention layers, and compare configurations with M ∈ {2, 4, 8, 16}"},"eval.aa_index":{"value":17,"source":"aa","at":"Artificial Analysis model page /models/ling-2-6-1t, Wayback snapshot 2026-10-19","quote":"Ling-2.6-1T scores 17 on the Artificial Analysis Intelligence Index","note":"Score of Ling-2.6-1T, post-trained from this base. Variant: Ling-2.6-1T. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","train.optimizer","train.lr_peak","train.lr_min","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip"],"derived":{"params.counted_total":{"value":1012231160320,"formula":"embedding + output head + all experts + other weights, inclusionAI/Ling-2.6-1T-base@64beeb8 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":2575302656,"formula":"input embedding + output head, inclusionAI/Ling-2.6-1T-base@64beeb8 safetensors headers"},"params.counted_non_embedding":{"value":1009655857664,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, inclusionAI/Ling-2.6-1T-base@64beeb8 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":13272900096,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, inclusionAI/Ling-2.6-1T-base@64beeb8 safetensors headers"},"params.counted_active_non_embedding":{"value":61004956160.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, inclusionAI/Ling-2.6-1T-base@64beeb8 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":102.4,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":2.25,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":7.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.063,"formula":"params.active / params.total"},"scaling.tpp":{"value":9.6,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":157.3642635660899,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":11520.0,"formula":"11520 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1509949440.0,"formula":"11520 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":11520.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":182.85714285714286,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":23967451.42857143,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":11520.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":11520.0,"tokens_per_entry":1}],"formula":"Global KV: 10 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"ling-3-0-flash","name":"Ling-3.0-flash-base","org":"inclusionAI","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base/resolve/0a96b0a3c68f93e5565d6f2d56c98557bded74fb/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base/resolve/0a96b0a3c68f93e5565d6f2d56c98557bded74fb/config.json","kind":"config"},"config-30t":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base-30T/resolve/4c7a67683c9e7ea9fad199b58e7b54cd835b63c2/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base/resolve/0a96b0a3c68f93e5565d6f2d56c98557bded74fb/model.safetensors.index.json","kind":"config"},"modeling_bailing_moe_v3":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base/resolve/0a96b0a3c68f93e5565d6f2d56c98557bded74fb/modeling_bailing_moe_v3.py","kind":"code"},"chat-card":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash/resolve/fcc60a52b34c69c6271d0ac8a8cb0815020b6a43/README.md","kind":"model-card"},"tds":{"url":"https://huggingface.co/inclusionAI/AI-Transparency/resolve/b72c608a99347c0f4551161fd4c55ae11823460e/Ling-3.0-LLM_TDS-Summary.pdf","kind":"paper"},"hf-index-30t":{"url":"https://huggingface.co/inclusionAI/Ling-3.0-flash-base-30T/resolve/4c7a67683c9e7ea9fad199b58e7b54cd835b63c2/model.safetensors.index.json","kind":"config"},"hf-commits":{"url":"https://huggingface.co/api/models/inclusionAI/Ling-3.0-flash-base/commits/main","kind":"api"},"aa":{"url":"https://web.archive.org/web/20261019102435/https://artificialanalysis.ai/models/ling-3-0-flash","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-12","source":"hf-commits","at":"Hugging Face commit history; 'Initial model release' 4489af0","quote":"\"date\":\"2026-08-12T13:55:51.000Z\"","note":"Date of the first commit with weights; no dated publisher page exists for the base weights, and when the repository became public is not recorded."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower, and the TDS summary ticks only Text as a training-data modality."},"params.active":{"value":5100000000,"source":"card","at":"README.md; Model Overview table","quote":"Totoal 124B, Activated 5.1B (Non-emb)"},"params.active_non_embedding":{"value":5100000000,"source":"card","at":"README.md; Model Overview, Key features","quote":"activating just 5.1B (Non-emb) parameters per token"},"params.note":{"value":"Rounded publisher figures; active count excludes embeddings. Whether the MTP layer is counted is not stated; the checkpoint index (254.97 GB in BF16, ≈127.5B parameters) includes it.","source":"card","at":"README.md; Model Overview, Key features","quote":"activating just 5.1B (Non-emb) parameters per token"},"params.total":{"value":124000000000,"source":"card","at":"README.md; Model Overview table","quote":"Totoal 124B, Activated 5.1B (Non-emb)"},"shape.d_model":{"value":2560,"source":"config","at":"/hidden_size"},"shape.layers":{"value":42,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":157184,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/use_qkv_bias"},"attn.conv":{"value":4,"source":"config","at":"/short_conv_kernel_size"},"attn.gate":{"value":"head-wise sigmoid on the MLA output, before the output projection","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"gate = F.sigmoid(gate.float()).type_as(hidden_states)","note":"/gated_attention_proj_granularity_type = head_wise. KDA layers have their own sigmoid-gated output norm."},"attn.global_layers":{"value":7,"source":"card","at":"README.md; Model Overview table","quote":"35 KDA + 7 Gated MLA (5:1)"},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim","note":"MLA layers (= /qk_head_dim). /head_dim=128 is the KDA head size."},"attn.heads":{"value":32,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"KDA, MLA","source":"card","at":"README.md; Model Overview table","quote":"35 KDA + 7 Gated MLA (5:1)"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"5 KDA : 1 MLA","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; DecoderLayer.__init__","quote":"if (layer_idx + 1) % config.layer_group_size == 0","note":"/layer_group_size = 6: MLA at layers 5, 11, …, 41. Ordered from that conditional with /layer_group_size=6; the implementation also makes any incomplete final group global.","basis":"inferred","sequence":["KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","MLA"]},"attn.linear_head_dim":{"value":128,"source":"config","at":"/head_dim","note":"BailingMoeV3KimiDeltaAttention uses /head_dim for both key and value heads."},"attn.linear_heads":{"value":32,"source":"config","at":"/num_attention_heads","note":"BailingMoeV3KimiDeltaAttention sets num_heads = num_attention_heads."},"attn.linear_kind":{"value":"KDA: 32 heads × 128, short conv 4 + SiLU on q/k/v, L2-normed q/k, full-rank decay and output-gate projections, safe gate with lower bound −5, sigmoid-gated RMSNorm on output","source":"config","at":"config.json kda_lower_bound, kda_safe_gate, no_kda_lora","quote":"\"kda_lower_bound\": -5.0, \"kda_safe_gate\": true,","note":"no_kda_lora = true selects full-rank f_proj/g_proj in BailingMoeV3KimiDeltaAttention."},"attn.linear_layers":{"value":35,"source":"card","at":"README.md; Model Overview table","quote":"35 KDA + 7 Gated MLA (5:1)"},"attn.position":{"value":"partial RoPE (MLA, decoupled 64 of 192 QK dims, interleaved), NoPE (KDA)","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)","note":"The rotary embedding overrides /partial_rotary_factor=0.5 with 1.0 over the 64 qk_rope_head_dim dims; interleaved per /rope_interleave."},"attn.q_lora_rank":{"value":0,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.qk_head_dim, bias=False)","note":"/q_lora_rank is null: full-rank query projection, no Q latent."},"attn.qk_norm":{"value":false,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"key_states = torch.cat((k_pass, k_rot), dim=-1)","note":"MLA layers: RMSNorm only on the KV latent (/q_lora_rank is null, so queries are a direct projection); no per-head Q/K norm, and /use_qk_norm is not read. KDA layers L2-normalize q and k."},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":6000000,"source":"config","at":"/rope_theta","note":"The pretrained-only 30T checkpoint config has rope_theta 10000 and max_position_embeddings 8192."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn = ACT2FN[/hidden_act] = silu; used for dense, shared and routed FFNs. Config expert_swiglu_limit_list / share_expert_swiglu_limit_list are not read by the modeling code."},"ffn.bias":{"value":false,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":6144,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":2,"source":"config","at":"/first_k_dense_replace"},"moe.expert_dim":{"value":768,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/num_experts"},"moe.groups":{"value":"8 groups, top-4 groups per token","source":"config","at":"config.json n_group, topk_group","quote":"\"n_group\": 8,","note":"/topk_group = 4; groups ranked by the sum of their top-2 biased scores (BailingMoeV3Gate.group_limited_topk)."},"moe.norm_topk":{"value":true,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3Gate.forward; config /norm_topk_prob","quote":"topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor","note":"Multiplied into the renormalized top-k weights in BailingMoeV3Gate.forward."},"moe.router":{"value":"sigmoid","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3Gate.forward; config /score_function, /scoring_func","quote":"scores = torch.sigmoid(logits.float()).type_as(logits)","note":"Router logits computed in FP32 (/router_dtype fp32). BailingMoeV3Gate adds an expert_bias buffer (/moe_router_enable_expert_bias, /topk_method noaux_tc) to the scores for top-k selection only; how it was updated is not stated."},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.shared_dim":{"value":768,"source":"config","at":"/moe_shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":1,"source":"hf-index-30t","at":"Ling-3.0-flash-base-30T model.safetensors.index.json; weight_map","quote":"\"model.layers.42.eh_proj.weight\"","basis":"inferred","note":"The pretrained-only checkpoint (before mid-training) already carries one MTP layer (/num_nextn_predict_layers = 1); the depth used in pretraining is not stated in text."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers"},"mtp.block":{"value":"RMSNorm(embedding of token t+1) and RMSNorm(hidden) concatenated, eh_proj 2d→d, then one gated MLA + MoE block with final RMSNorm","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MTPLayer","quote":"hidden_states = self.eh_proj(torch.cat([input_embeds, hidden_states], dim=-1))"},"mtp.ffn":{"value":"MoE","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MTPLayer.__init__","quote":"self.mlp = BailingMoeV3SparseMoeBlock(config)"},"mtp.shares_embedding":{"value":true,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3ForCausalLM.forward","quote":"mtp_logits = self.lm_head(mtp_hidden_states).float()","note":"Input embedding is also the shared word_embeddings (BailingMoeV3Model.forward)."},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.42.eh_proj.weight\""},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3DecoderLayer.__init__","quote":"self.input_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.extension":{"value":"8K → 32K → 256K; RoPE base 10,000 in the pretrained-only checkpoint, 6,000,000 after mid-training","source":"chat-card","at":"Ling-3.0-flash README.md; Model Overview table","quote":"Context Training Schedule 8K -> 32K -> 256K","note":"RoPE bases from the -base-30T and -base-midtrain/-base configs; the stage boundaries relative to mid-training are not stated."},"ctx.final":{"value":262144,"source":"chat-card","at":"Ling-3.0-flash README.md; Model Overview table","quote":"Context Training Schedule 8K -> 32K -> 256K"},"ctx.max":{"value":262144,"source":"config","at":"/max_position_embeddings"},"ctx.pretrain":{"value":8192,"source":"config-30t","at":"/max_position_embeddings","note":"Config of the pretrained-only checkpoint; the Ling-3.0-flash card gives the schedule as '8K -> 32K -> 256K'."},"train.averaging":{"value":"WSM: weighted checkpoint merging of mid-training checkpoints replaces LR decay","source":"card","at":"README.md; Model Overview, Key features","quote":"We replace conventional learning-rate decay with weighted checkpoint merging."},"train.data":{"value":">10T text tokens; mainly English and Chinese: Common Crawl, academic papers, developer Q&A and code, licensed text, own crawl (Ant Spider), synthetic data from Ling-2.6 models","source":"tds","at":"Ling-3.0 LLM TDS summary, §1.3 and §2","quote":"More than 10 trillions tokens","note":"Training-data size, not tokens seen; the EU AI Act summary covers all training (pre- and post-) of Ling-3.0-flash, -tiny and -flash-fin."},"train.cutoff":{"value":"~2025-10","source":"tds","at":"Ling-3.0 LLM TDS summary, §1.3; latest date of data acquisition","quote":"Approximately October 2025.","note":"Covers all Ling-3.0 LLM training data, pre- and post-training."},"train.schedule":{"value":"Warmup-Stable and Merge (WSM): warmup, stable LR, no decay phase; checkpoint merging replaces decay","source":"card","at":"README.md; Model Overview, Key features","quote":"By eliminating the decay phase, our Base Model is better suited for continual pretraining and dynamic data expansion"},"train.stages":{"value":[{"name":"pretraining","seq_len":8192,"note":"Released as Ling-3.0-flash-base-30T; seq_len from that checkpoint's config (max_position_embeddings 8192).","quote":"Pretrained checkpoint have completed large-scale pretraining but have not undergone mid-training, WSM merging (or learning-rate decay), or post-training."},{"name":"mid-training","note":"Released as Ling-3.0-flash-base-midtrain; its config has max_position_embeddings 262144 and rope_theta 6,000,000.","quote":"Mid-trained checkpoint have completed mid-training but have not undergone WSM merging (or learning-rate decay) or post-training."},{"name":"WSM merge","note":"Checkpoint merge in place of LR decay; released as Ling-3.0-flash-base.","quote":"Merged checkpoints have undergone"}],"source":"card","at":"README.md; Introduction"},"eval.aa_index":{"value":20,"source":"aa","at":"Artificial Analysis model page /models/ling-3-0-flash, Wayback snapshot 2026-10-19","quote":"Ling 3.0 Flash scores 20 on the Artificial Analysis Intelligence Index","note":"Score of Ling-3.0-flash, post-trained from this base. Variant: Ling 3.0 Flash. Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.loss_weight","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":124414211552,"formula":"embedding + output head + all experts + other weights, inclusionAI/Ling-3.0-flash-base@0a96b0a safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":804782080,"formula":"input embedding + output head, inclusionAI/Ling-3.0-flash-base@0a96b0a safetensors headers"},"params.counted_non_embedding":{"value":123609429472,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, inclusionAI/Ling-3.0-flash-base@0a96b0a safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":3090163360,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, inclusionAI/Ling-3.0-flash-base@0a96b0a safetensors headers"},"params.counted_active_non_embedding":{"value":4700911072.0,"formula":"non-expert weights + routed experts × 8/512, excluding embedding and output head, inclusionAI/Ling-3.0-flash-base@0a96b0a safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":60.95238095238095,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"ffn.mult":{"value":2.4,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":5.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04112903225806452,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":8064.0,"formula":"8064 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1056964608.0,"formula":"8064 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":8064.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1581.1764705882354,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":207247962.3529412,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":8064.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":8064.0,"tokens_per_entry":1}],"formula":"Global KV: 7 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"ling31-flash","name":"Ling-3.1-flash","org":"inclusionAI","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/inclusionAI/Ling-3.1-flash/resolve/a4ccd6b45c002ddb236f9cb77d52cc427482bf19/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/inclusionAI/Ling-3.1-flash/resolve/a4ccd6b45c002ddb236f9cb77d52cc427482bf19/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/inclusionAI/Ling-3.1-flash/resolve/a4ccd6b45c002ddb236f9cb77d52cc427482bf19/model.safetensors.index.json","kind":"config"},"modeling_bailing_moe_v3":{"url":"https://huggingface.co/inclusionAI/Ling-3.1-flash/resolve/a4ccd6b45c002ddb236f9cb77d52cc427482bf19/modeling_bailing_moe_v3.py","kind":"code"},"tds":{"url":"https://huggingface.co/inclusionAI/AI-Transparency/resolve/b72c608a99347c0f4551161fd4c55ae11823460e/Ling-3.1_TDS-Summary.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019102618/https://artificialanalysis.ai/models/ling-3-1-flash","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-10-14","source":"tds","at":"Ling-3.1 TDS summary, §1.2 Date of placement of the model on the Union market","quote":"2026-10-14","note":"Hugging Face repository created 2026-10-13."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","source":"card","at":"README front matter","quote":"pipeline_tag: text-generation","basis":"inferred","note":"Card tags the checkpoint text-generation; config declares no vision or audio tower, and the TDS summary ticks only Text as a training-data modality."},"params.active":{"value":25000000000,"source":"card","at":"README.md; Introduction","quote":"With 560B total parameters, 25B active parameters per token"},"params.note":{"value":"Rounded publisher figures; whether the MTP layer is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Parameter Scale Total 560B, Activated 25B"},"params.total":{"value":560000000000,"source":"card","at":"README.md; Introduction","quote":"With 560B total parameters, 25B active parameters per token"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":72,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":157184,"source":"config","at":"/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/use_qkv_bias"},"attn.conv":{"value":4,"source":"config","at":"/short_conv_kernel_size"},"attn.gate":{"value":"head-wise sigmoid on the MLA output, before the output projection","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"gate = F.sigmoid(gate.float()).type_as(hidden_states)","note":"/gated_attention_proj_granularity_type = head_wise. KDA layers have their own sigmoid-gated output norm."},"attn.global_layers":{"value":9,"source":"card","at":"README.md; Model Overview","quote":"Transformer Layers 63 KDA + 9 Gated MLA (7:1)"},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim","note":"MLA layers (= /qk_head_dim). /head_dim=128 is the KDA head size."},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"KDA, MLA","source":"card","at":"README.md; Model Overview","quote":"Transformer Layers 63 KDA + 9 Gated MLA (7:1)"},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank"},"attn.layout":{"value":"7 KDA : 1 MLA","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; DecoderLayer.__init__","quote":"if (layer_idx + 1) % config.layer_group_size == 0","note":"/layer_group_size = 8: MLA at layers 7, 15, …, 71. Ordered from that conditional with /layer_group_size=8; the implementation also makes any incomplete final group global.","basis":"inferred","sequence":["KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA","KDA","KDA","KDA","KDA","KDA","KDA","KDA","MLA"]},"attn.linear_head_dim":{"value":128,"source":"config","at":"/head_dim","note":"BailingMoeV3KimiDeltaAttention uses /head_dim for both key and value heads."},"attn.linear_heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"BailingMoeV3KimiDeltaAttention sets num_heads = num_attention_heads."},"attn.linear_kind":{"value":"KDA: 64 heads × 128, short conv 4 + SiLU on q/k/v, L2-normed q/k, full-rank decay and output-gate projections, safe gate with lower bound −5, sigmoid-gated RMSNorm on output","source":"config","at":"config.json kda_lower_bound, kda_safe_gate, no_kda_lora","quote":"\"kda_lower_bound\": -5.0, \"kda_safe_gate\": true,","note":"no_kda_lora = true selects full-rank f_proj/g_proj in BailingMoeV3KimiDeltaAttention."},"attn.linear_layers":{"value":63,"source":"card","at":"README.md; Model Overview","quote":"Transformer Layers 63 KDA + 9 Gated MLA (7:1)"},"attn.position":{"value":"partial RoPE (MLA, decoupled 64 of 192 QK dims, interleaved), NoPE (KDA)","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)","note":"The rotary embedding overrides /partial_rotary_factor=0.5 with 1.0 over the 64 qk_rope_head_dim dims; interleaved per /rope_interleave."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MultiLatentAttention.forward","quote":"key_states = torch.cat((k_pass, k_rot), dim=-1)","note":"MLA layers: RMSNorm only on the Q and KV latents; no per-head Q/K norm, and /use_qk_norm is not read. KDA layers L2-normalize q and k."},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":6000000,"source":"config","at":"/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn = ACT2FN[/hidden_act] = silu; used for dense, shared and routed FFNs. Config expert_swiglu_limit_list (10 on layers 65–71) and share_expert_swiglu_limit_list (10 on layers 64–71) are not read by the modeling code."},"ffn.bias":{"value":false,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":10240,"source":"config","at":"/intermediate_size"},"moe.balancing":{"value":"aux-loss-free expert bias added to routing scores (noaux_tc)","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3Gate.forward; config /moe_router_enable_expert_bias, /topk_method","quote":"scores_for_routing = scores + self.expert_bias","note":"Bias update rate not disclosed."},"moe.dense_layers":{"value":4,"source":"config","at":"/first_k_dense_replace"},"moe.expert_dim":{"value":1280,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/num_experts"},"moe.groups":{"value":"8 groups, top-4 groups per token","source":"config","at":"config.json n_group, topk_group","quote":"\"n_group\": 8,","note":"/topk_group = 4; groups ranked by the sum of their top-2 biased scores (BailingMoeV3Gate.group_limited_topk)."},"moe.norm_topk":{"value":true,"source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3Gate.forward; config /norm_topk_prob","quote":"topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores"},"moe.routed_scale":{"value":2.5,"source":"config","at":"/routed_scaling_factor","note":"Multiplied into the renormalized top-k weights in BailingMoeV3Gate.forward."},"moe.router":{"value":"sigmoid","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3Gate.forward; config /score_function, /scoring_func","quote":"scores = torch.sigmoid(logits.float()).type_as(logits)","note":"Router logits computed in FP32 (/router_dtype fp32)."},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.shared_dim":{"value":1280,"source":"config","at":"/moe_shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.72.eh_proj.weight\"","note":"One MTP layer (MLA + MoE, /num_nextn_predict_layers = 1). Use in pretraining is not disclosed; the released config sets mtp_loss_scaling_factor to 0."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_bailing_moe_v3","at":"modeling_bailing_moe_v3.py; BailingMoeV3DecoderLayer.__init__","quote":"self.input_layernorm = BailingMoeV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Introduction","quote":"a 1M-token context window","note":"Config max_position_embeddings is 262144 with rope_scaling null."},"train.cutoff":{"value":"approximately July 2026","source":"tds","at":"Ling-3.1 TDS summary, §1.3","quote":"Approximately July 2026.","note":"Latest date of data acquisition/collection for model training, as stated in the EU AI Act training-content summary."},"train.data":{"value":">10T text tokens; multilingual (mainly English and Chinese) web incl. Common Crawl, academic papers, code and developer Q&A, licensed text, own crawl (Ant Spider), synthetic data from Ling-2.6 models","source":"tds","at":"Ling-3.1 TDS summary, §1.3 and §2","quote":"More than 10 trillions tokens","note":"Overall training-data size after pre-processing, not tokens seen; the summary covers all training, not only pretraining."},"eval.aa_index":{"value":41,"source":"aa","at":"Artificial Analysis model page /models/ling-3-1-flash, Wayback snapshot 2026-10-19","quote":"Ling 3.1 Flash scores 41 on the Artificial Analysis Intelligence Index","note":"Variant: Ling 3.1 Flash. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":563946483520,"formula":"embedding + output head + all experts + other weights, inclusionAI/Ling-3.1-flash@a4ccd6b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1287651328,"formula":"input embedding + output head, inclusionAI/Ling-3.1-flash@a4ccd6b safetensors headers"},"params.counted_non_embedding":{"value":562658832192,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, inclusionAI/Ling-3.1-flash@a4ccd6b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":8272593600,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, inclusionAI/Ling-3.1-flash@a4ccd6b safetensors headers"},"params.counted_active_non_embedding":{"value":23606882112.0,"formula":"non-expert weights + routed experts × 8/512, excluding embedding and output head, inclusionAI/Ling-3.1-flash@a4ccd6b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":56.888888888888886,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":2.5,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3125,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":7.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.044642857142857144,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":10368.0,"formula":"10368 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1358954496.0,"formula":"10368 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":10368.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":414.72,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":54358179.84,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":10368.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":10368.0,"tokens_per_entry":1}],"formula":"Global KV: 9 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"longcat-2","name":"LongCat-2.0","org":"Meituan","country":"China","kind":"reasoning","weights":"open","note":"Post-trained checkpoint (thinking and non-thinking modes); no base checkpoint released.","sources":{"card":{"url":"https://huggingface.co/meituan-longcat/LongCat-2.0/resolve/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/meituan-longcat/LongCat-2.0/resolve/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/config.json","kind":"config"},"weights-index":{"url":"https://huggingface.co/meituan-longcat/LongCat-2.0/resolve/834bf5ffe3047aa9f6cc7a64a9bc068b146b8274/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20260825022823id_/https://s3.meituan.net/static-prod01/com.sankuai.friday.longcat.next2/assets/index-B2a7Qfrx.js","kind":"blog"},"lsa":{"url":"https://arxiv.org/pdf/2608.01662v2","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019102808/https://artificialanalysis.ai/models/longcat-2-0","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-06-30","source":"blog","at":"blog bundle; hero date of 'Introducing LongCat-2.0'","quote":"date:`2026-06-30`","note":"Publisher-dated announcement; the weight files were uploaded to Hugging Face on 2026-07-05 (repository commit history)."},"meta.license":{"value":"mit","source":"card","at":"README.md; front matter","quote":"license: mit"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md front matter","quote":"pipeline_tag: text-generation","note":"No modality statement; the card's pipeline tag is text-generation and config.json (LongcatCausalLM) has no vision or audio config."},"params.active":{"value":48000000000,"source":"card","at":"README.md; Model Introduction","quote":"a large-scale MoE language model with 1.6 trillion total parameters and ~48 billion activated per token"},"params.note":{"value":"The 1.6T total includes 135B N-gram Embedding parameters","source":"card","at":"README.md; N-gram Embedding","quote":"135B N-gram Embedding parameters are included in the model"},"params.total":{"value":1600000000000,"source":"card","at":"README.md; Model Introduction","quote":"a large-scale MoE language model with 1.6 trillion total parameters and ~48 billion activated per token"},"shape.d_model":{"value":8192,"source":"config","at":"/hidden_size"},"shape.layers":{"value":38,"source":"config","at":"/num_layers","note":"Counts LongCat-Flash-style shortcut-connected blocks: each holds two attention + dense-FFN sublayers (self_attn.0/.1, mlps.0/.1 in the weight map) and one MoE, so the backbone has 76 MLA and 76 dense-FFN sublayers."},"shape.memory":{"value":"N-gram Embedding, n-gram size 5, 135B params; embedding space expanded ~100× over the token vocabulary","source":"blog","at":"blog bundle; N-gram Embedding","quote":"the n-gram size is configured to 5, and 135B N-gram Embedding parameters are included in the model","note":"Weight map has 16 N-gram tables (model.oe_embed_tokens0-15) with projections (oe_embed_proj0-15), and a separate set in the MTP module; config /oe_vocab_size_ratio 100.567, /oe_neighbor_num 5, /oe_split_num 4."},"shape.vocab":{"value":163840,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"weights-index","at":"model.safetensors.index.json; weight_map","quote":"\"lm_head.weight\"","basis":"inferred","note":"Weight map stores lm_head.weight separately from model.embed_tokens.weight."},"attn.global_layers":{"value":76,"source":"weights-index","at":"model.safetensors.index.json; weight_map","quote":"model.layers.37.self_attn.1.kv_a_proj_with_mqa.weight","basis":"inferred","note":"38 blocks (/num_layers) × 2 MLA sublayers, each with its own kv_a_proj_with_mqa and therefore its own latent KV cache; counts attention sublayers, unlike shape.layers."},"attn.head_dim":{"value":192,"source":"config","at":"/qk_nope_head_dim + /qk_rope_head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"MLA, LSA","source":"card","at":"README.md; LongCat Sparse Attention","quote":"we introduce LongCat Sparse Attention (LSA)","note":"Config /attention_method is MLA; LSA is the DSA-derived sparse selection inside those layers."},"attn.kv_lora_rank":{"value":512,"source":"config","at":"/kv_lora_rank","note":"Config also sets mla_scale_q_lora and mla_scale_kv_lora true; no modeling code is published to define them."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.position":{"value":"RoPE (MLA decoupled, 64 dims)","source":"config","at":"config.json","quote":"\"qk_rope_head_dim\": 64"},"attn.indexer_dim":{"value":128,"source":"config","at":"/index_head_dim","note":"Indexer wk weight is [128, 8192], one key head; present only in self_attn.0 of each block, so 38 indexer caches for 76 MLA sublayers."},"attn.q_lora_rank":{"value":1536,"source":"config","at":"/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"config","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":1000000,"source":"config","at":"/rope_theta"},"attn.sparse":{"value":"LSA: 2048-token budget per query = 16 sink tokens + 1024-token sliding window + indexer-selected tokens; one indexer pass shared by every 2 attention sublayers","source":"lsa","at":"p.8 §3.1","quote":"We set sink size Ksink = 16 and sliding window size Kswa = 1024","note":"Released config matches: /index_topk 2048, /index_init_tokens 16, /index_local_tokens 1024, /cli_factor 2; index_n_heads 32 query heads of index_head_dim 128, index_k_norm_type rms; dsa_mtp_cli true shares one index across the 3 MTP steps. Card: 'the target model shares an index every 2 layers'; indexer weights exist only in self_attn.0 of each block. Hierarchical Indexing is training-free and enabled only for selected ultra-long-context tasks (blog)."},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"ffn.dim":{"value":12288,"source":"config","at":"/ffn_hidden_size","note":"Two dense FFN sublayers per block (mlps.0/.1 in the weight map) alongside one MoE."},"moe.expert_dim":{"value":2048,"source":"config","at":"/expert_ffn_hidden_size"},"moe.experts":{"value":768,"source":"config","at":"/n_routed_experts","note":"Config also declares 128 identity zero-experts (/zero_expert_num, /zero_expert_type); the router classifier is [896, 8192], scoring 768 + 128 experts, with an e_score_correction_bias. No modeling code is published."},"moe.shared":{"value":0,"source":"weights-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.0.mlp.experts.0.gate_proj.weight\"","basis":"inferred","note":"Weight map has routed experts and the two dense branches (mlps.0/.1) but no shared-expert weights."},"moe.dense_layers":{"value":0,"source":"weights-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.0.mlp.router.classifier.weight\"","basis":"inferred","note":"Every block from layer 0 carries an MoE branch next to its dense branches."},"moe.routed_scale":{"value":9,"source":"config","at":"/routed_scaling_factor"},"moe.top_k":{"value":12,"source":"config","at":"/moe_topk"},"mtp.released":{"value":true,"source":"weights-index","at":"/weight_map | has mtp.","note":"One MTP layer is stored (model.mtp.layers.0: MLA with indexer, dense FFN, enorm/hnorm + eh_proj, own embedding and N-gram tables); config /mtp_num_layers 3 with /mtp_replicate_modules true reuses it for 3 draft steps; mtp_disable_over_tokenizer true is undocumented. Pretraining MTP depth is not disclosed."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps","note":"No modeling code is published; the norm type is not stated."},"ctx.extension":{"value":"YaRN (factor 120, beta_fast 32, beta_slow 1, mscale 1) over an 8,192-token original context","source":"config","at":"config.json; rope_scaling","quote":"\"original_max_position_embeddings\": 8192, \"rope_type\": \"deepseek_yarn\", \"factor\": 120","note":"Config rope_scaling keys factor, beta_fast, beta_slow, mscale, mscale_all_dim; the stages that used it are not disclosed."},"ctx.final":{"value":1000000,"source":"card","at":"README.md; Model Introduction","quote":"train LongCat-2.0 on hundreds of billions of tokens of 1M-context data","note":"Stated as 1M; the blog says CP scales past 512 'enabling native 1M-length training'. Released config /max_position_embeddings is 262,144."},"ctx.max":{"value":1000000,"source":"blog","at":"blog bundle; Inference","quote":"Serving a 1.6T-parameter model over a 1M-token context presents a significant challenge","note":"Stated as 1M; released config /max_position_embeddings is 262,144."},"train.hardware":{"value":"over 50K AI ASICs in superpods of up to 48 machines; millions of accelerator-days","source":"blog","at":"blog bundle; Training","quote":"LongCat-2.0 is pre-trained on over 50K AI ASICs","note":"6D parallelism (TP/CP/EP/DP/PP plus EMBP for the N-gram Embeddings), ZeRO-1, selective recomputation and allocator-level OOM-aware offloading; CP scaled past 512 for 1M-length training."},"train.accelerators":{"value":">50,000 AI ASICs","source":"blog","at":"blog bundle; Training","quote":"LongCat-2.0 is pre-trained on over 50K AI ASICs","note":"Chip not named."},"train.parallelism":{"value":"TP · PP · EP · CP · DP · EMBP","source":"blog","at":"blog bundle; Training, Training at Scale","quote":"Beyond standard TP/CP/EP/DP/PP, we introduce EMBP to parallelize and accelerate N-gram Embeddings.","note":"No degrees stated; EMBP parallelizes the N-gram Embeddings; the same section states ZeRO-1."},"train.optimizer":{"value":"Muon","source":"blog","at":"blog bundle; Training at Scale","quote":"We deploy the Muon optimizer at large scale on our accelerators"},"train.stages":{"value":[{"name":"pre-training","tokens":">35T","quote":"Pretraining spans millions of accelerator-days across more than 35 trillion tokens"},{"name":"1M long-context","seq_len":1000000,"note":"Hundreds of billions of tokens.","quote":"train LongCat-2.0 on hundreds of billions of tokens of 1M-context data"}],"source":"card","at":"README.md; Model Introduction","note":"Sparse attention is reached through a dense warmup computed forward-only alongside the KL loss (blog, Long Context Training)."},"train.tokens":{"value":">35T","source":"card","at":"README.md; Model Introduction","quote":"Pretraining spans millions of accelerator-days across more than 35 trillion tokens"},"eval.aa_index":{"value":19,"source":"aa","at":"Artificial Analysis model page /models/longcat-2-0, Wayback snapshot 2026-10-19","quote":"LongCat 2.0 scores 19 on the Artificial Analysis Intelligence Index","note":"Variant: LongCat 2.0. Index v4.3.2."}},"not_disclosed":["attn.qk_norm","ffn.activation","moe.router","moe.balancing","moe.norm_topk","mtp.depth","norm.type","norm.placement","ctx.pretrain","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":null,"formula":"checkpoint count (1503.6B backbone) does not reproduce the stated total and active counts"},"params.counted_embedding":{"value":null,"formula":"checkpoint count (1503.6B backbone) does not reproduce the stated total and active counts"},"params.counted_non_embedding":{"value":null,"formula":"checkpoint count (1503.6B backbone) does not reproduce the stated total and active counts"},"params.counted_active_non_embedding":{"value":null,"formula":"checkpoint count (1503.6B backbone) does not reproduce the stated total and active counts"},"shape.aspect":{"value":215.57894736842104,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"ffn.mult":{"value":1.5,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.03,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":107008.0,"formula":"107008 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":14025752576.0,"formula":"107008 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":107008.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2229.3333333333335,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":292203178.6666667,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":107008.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":107008.0,"tokens_per_entry":1}],"formula":"Global KV: 76 × (704 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mai-thinking-1","name":"MAI-Thinking-1","org":"Microsoft AI","country":"United States","kind":"reasoning","weights":"closed","note":"Facts describe MAI-Base-1, the pretrained base of MAI-Thinking-1; neither checkpoint is public (served via Microsoft Foundry).","sources":{"report":{"url":"https://web.archive.org/web/20261011065901id_/http://microsoft.ai/pdf/mai-thinking-1.pdf","kind":"paper"},"build-blog":{"url":"https://web.archive.org/web/20261013022631/https://blogs.microsoft.com/blog/2026/06/02/microsoft-build-2026-be-yourself-at-work/","kind":"blog"}},"facts":{"meta.released":{"value":"2026-06-02","source":"build-blog","at":"Microsoft Build 2026: Be yourself at work; byline","quote":"Jun 2, 2026 | Kyle Daigle","note":"Post announces MAI-Thinking-1 in private preview on Foundry."},"params.total":{"value":962000000000,"source":"report","at":"p.7 Table 1, MAI-Base-1 row","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80","note":"Rounded to 1T in the report text."},"params.active":{"value":34700000000,"source":"report","at":"p.7 Table 1, MAI-Base-1 row","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80","note":"Rounded to 35B in the report text."},"params.note":{"value":"Counts include the tied embedding (200,019 × 6656 ≈ 1.33B) once","source":"report","at":"p.7 Table 1, MAI-Base-1 row","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80","basis":"inferred","note":"From Table 1 shapes: 39 MoE layers × 512 experts × 3·3072·5120 + 78 attention layers (80×128 Q/O, 8×128 K/V) + 39 dense FFNs (3·6656²) + latent projections + router + embedding = 962.1B total, 34.66B active."},"shape.layers":{"value":78,"source":"report","at":"p.7 §2.2.1","quote":"The final architecture of MAI-Base-1 has 78 layers."},"shape.d_model":{"value":6656,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (Hidden)","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80"},"shape.vocab":{"value":200019,"source":"report","at":"p.6 §2.1","quote":"we chose the o200k_base tokenizer with a vocabulary size of 200,019"},"shape.tied_embeddings":{"value":true,"source":"report","at":"p.6 §2.1","quote":"the input and output embedding weights are tied"},"shape.tokenizer":{"value":"o200k_base (OpenAI)","source":"report","at":"p.6 §2.1","quote":"we chose the o200k_base tokenizer with a vocabulary size of 200,019"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.6 §2.1 Attention","quote":"pairing 5 local attention layers (Beltagy et al., 2020) with 1 global attention layer"},"attn.layout":{"value":"5 SWA : 1 global","source":"report","at":"p.6 §2.1 Attention","quote":"pairing 5 local attention layers (Beltagy et al., 2020) with 1 global attention layer","note":"The last layer of each group of 6 is global (p.7 §2.2.1). Ordered from the reported last-layer-of-each-group rule; no public checkpoint config.","basis":"inferred","sequence":["SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":13,"source":"report","at":"p.7 §2.2.1","quote":"For the 5:1 local to global attention ratio that we use, L must be a multiple of 6 so that the last attention layer applies global attention.","basis":"inferred","note":"78 layers / 6 = 13 global."},"attn.local_layers":{"value":65,"source":"report","at":"p.7 §2.2.1","quote":"For the 5:1 local to global attention ratio that we use, L must be a multiple of 6 so that the last attention layer applies global attention.","basis":"inferred","note":"78 layers × 5/6 = 65 local."},"attn.window":{"value":512,"source":"report","at":"p.6 §2.1 Attention","quote":"with sliding window size of 512 and a base frequency of 10,000"},"attn.heads":{"value":80,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (KV/Q)","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80"},"attn.kv_heads":{"value":8,"source":"report","at":"p.6 §2.1 Attention","quote":"We use group-query attention with 8 KV heads (Ainslie et al., 2023) and a per-head dimension of 128."},"attn.head_dim":{"value":128,"source":"report","at":"p.6 §2.1 Attention","quote":"We use group-query attention with 8 KV heads (Ainslie et al., 2023) and a per-head dimension of 128."},"attn.local_heads":{"value":80,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (KV/Q)","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80","note":"The report gives one head configuration for local and global layers."},"attn.local_kv_heads":{"value":8,"source":"report","at":"p.6 §2.1 Attention","quote":"We use group-query attention with 8 KV heads (Ainslie et al., 2023) and a per-head dimension of 128.","note":"Stated for all attention layers."},"attn.local_head_dim":{"value":128,"source":"report","at":"p.6 §2.1 Attention","quote":"We use group-query attention with 8 KV heads (Ainslie et al., 2023) and a per-head dimension of 128.","note":"Stated for all attention layers."},"attn.position":{"value":"RoPE (SWA), NoPE (global)","source":"report","at":"p.6 §2.1 Attention","quote":"The global attention layers do not use any position encoding"},"attn.local_rope_theta":{"value":10000,"source":"report","at":"p.6 §2.1 Attention","quote":"with sliding window size of 512 and a base frequency of 10,000"},"attn.qk_norm":{"value":true,"source":"report","at":"p.6 §2.1 Attention","quote":"We apply RMSNorms to the queries and the keys"},"attn.bias":{"value":false,"source":"report","at":"p.6 §2.1","quote":"We do not use any biases in the model"},"ffn.activation":{"value":"SwiGLU","source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"we use SwiGLU (Shazeer, 2020) for both dense and MoE feed-forward layers"},"ffn.dim":{"value":6656,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (FFN) and caption","quote":"The (Expert-)FFN sizes refer to the output size of the first linear layer, which is 2x of the input size for the second linear layer due to SwiGLU.","basis":"inferred","note":"Table 1 FFN 13312 is the fused gate+up output; intermediate width 13312 / 2 = 6656. Dense FFNs alternate with MoE layers (39 each)."},"ffn.bias":{"value":false,"source":"report","at":"p.6 §2.1","quote":"We do not use any biases in the model"},"moe.experts":{"value":512,"source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"each compressed representation is routed to 8 out of the 512 experts with softmax gating"},"moe.top_k":{"value":8,"source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"each compressed representation is routed to 8 out of the 512 experts with softmax gating"},"moe.router":{"value":"softmax","source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"each compressed representation is routed to 8 out of the 512 experts with softmax gating","note":"Router input is the uncompressed hidden state."},"moe.shared":{"value":0,"source":"report","at":"p.8 Table 2 caption","quote":"over the interleaved layout of MAI-Base-1 (8/512 MoE + dense FFN)","basis":"inferred","note":"The report writes shared-expert variants as 7+1 shared; MAI-Base-1 is 8/512 routed only, and §2.1 finds shared experts give little or no benefit in the interleaved layout."},"moe.expert_dim":{"value":5120,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (Expert FFN) and caption","quote":"The (Expert-)FFN sizes refer to the output size of the first linear layer, which is 2x of the input size for the second linear layer due to SwiGLU.","basis":"inferred","note":"Table 1 Expert FFN 10240 is the fused gate+up output; intermediate width 10240 / 2 = 5120, matching the 962B total."},"moe.latent_dim":{"value":3072,"source":"report","at":"p.7 Table 1, MAI-Base-1 row (Down Proj)","quote":"MAI-Base-1 34.7B 962B 78 6656 13312 3072 10240 8/512 8/80","note":"Shared down-projection before all-to-all dispatch, up-projection after combine (LatentMoE); changed from L78's 3328 so the dimension is divisible by 512 for DeepEP."},"moe.dense_layers":{"value":1,"source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"Our first feedforward layer is dense","note":"Thereafter dense FFN and MoE alternate layer by layer."},"moe.balancing":{"value":"global-batch GShard-style load-balancing loss (expert frequencies aggregated across DP ranks and micro-batches); dropless","source":"report","at":"p.6 §2.1 Feed-forward and MoE","quote":"The GShard-style loss (Lepikhin et al., 2020) that we use performs similarly to the loss-free variant"},"norm.type":{"value":"RMSNorm","source":"report","at":"p.6 §2.1","quote":"we employ RMSNorm (Zhang and Sennrich, 2019) at both the input and output immediately before the residual addition"},"norm.placement":{"value":"sandwich","source":"report","at":"p.6 §2.1","quote":"we employ RMSNorm (Zhang and Sennrich, 2019) at both the input and output immediately before the residual addition"},"norm.residual":{"value":"FP32 residual stream","source":"report","at":"p.22 §2.6.3","quote":"the entire residual stream all the way from the input embedding to the output"},"train.dropout":{"value":0.15,"source":"report","at":"p.21 §2.6.2","quote":"We apply dropout 0.15 at each layer's output before the residual add.","note":"Applied to each sublayer output before the residual add."},"ctx.pretrain":{"value":16384,"source":"report","at":"p.21 §2.6.1 Table 6","quote":"Pre-training 30 T 16,384 8,192"},"ctx.final":{"value":262144,"source":"report","at":"p.21 §2.6.1 Table 6","quote":"Mid-training 2 150 B 262,144 4,096"},"ctx.max":{"value":262144,"source":"report","at":"p.2 §1","quote":"After mid-training, MAI-Thinking-1 achieves a maximum context length of 256K."},"ctx.extension":{"value":"16K → 64K (3.4T tokens, mid-training 1) → 256K (150B tokens, mid-training 2); same mixture re-packed at the longer length; Ulysses context parallelism","source":"report","at":"p.20 §2.5.4; p.21 Table 6; p.85–87 App. C","quote":"the model is pre-trained at 16K, mid-trained at 64K (Stage 1), and then extended to 256K in a short dedicated phase (Stage 2).","note":"App. C.5 gives 140B tokens for the 256K extension; Table 6 gives 150B, which matches the 3.55T mid-training total in §2."},"train.tokens":{"value":33550000000000,"source":"report","at":"p.5 §2","quote":"we ran a main pre-training phase of 30 trillion tokens, followed by mid-training phases totaling 3.55 trillion tokens","basis":"inferred","note":"30T pre-training + 3.55T mid-training (Table 6: 3.4T + 150B)."},"train.stages":{"value":[{"name":"pre-training","tokens":30000000000000,"seq_len":16384,"lr":0.0002,"batch_tokens":134000000,"note":"Peak LR; ~12B-token warmup, cosine to 2e-5. 8,192 GB200.","quote":"Pre-training 30 T 16,384 8,192"},{"name":"mid-training 1","tokens":3400000000000,"seq_len":65536,"lr":2e-05,"batch_tokens":134000000,"note":"Cosine 2e-5 → 1e-5, no warmup; STEM/math 35%, code 55%. 8,192 GB200.","quote":"Mid-training 1 3.4 T 65,536 8,192"},{"name":"mid-training 2","tokens":150000000000,"seq_len":262144,"lr":1e-06,"batch_tokens":134000000,"note":"Constant LR, no warmup. 4,096 GB200.","quote":"Mid-training 2 150 B 262,144 4,096"}],"source":"report","at":"p.21–22 §2.6.1 Table 6, §2.6.2"},"train.optimizer":{"value":"AdamW","source":"report","at":"p.21 §2.6.2","quote":"MAI-Base-1 is optimized with AdamW (Loshchilov and Hutter, 2017) (β1 = 0.95, β2 = 0.925, ε = 10−8 )"},"train.betas":{"value":"(0.95, 0.925)","source":"report","at":"p.21 §2.6.2","quote":"MAI-Base-1 is optimized with AdamW (Loshchilov and Hutter, 2017) (β1 = 0.95, β2 = 0.925, ε = 10−8 )","note":"Adam ε 1e-8."},"train.weight_decay":{"value":0.1,"source":"report","at":"p.21 §2.6.2","quote":"a constant weight decay of 0.1, with reduced weight decay on attention weights (0.01) and embedding weights (0.005)","note":"0.01 on attention weights, 0.005 on embedding weights."},"train.grad_clip":{"value":1.0,"source":"report","at":"p.21 §2.6.2","quote":"The global gradient norm is clipped to 1.0."},"train.lr_peak":{"value":0.0002,"source":"report","at":"p.21 §2.6.2","quote":"then decays along a cosine schedule from a peak of 2 × 10−4 to a minimum of 2 × 10−5"},"train.lr_min":{"value":2e-05,"source":"report","at":"p.21 §2.6.2","quote":"then decays along a cosine schedule from a peak of 2 × 10−4 to a minimum of 2 × 10−5","note":"End of pre-training; mid-training decays further to 1e-5, then 1e-6 constant."},"train.warmup":{"value":"~12B tokens (linear)","source":"report","at":"p.21 §2.6.2","quote":"The learning rate follows a linear warmup of approximately 12 B tokens"},"train.schedule":{"value":"linear warmup ~12B tokens, cosine 2e-4 → 2e-5 (pre-training); cosine 2e-5 → 1e-5 (mid-training 1); constant 1e-6 (mid-training 2)","source":"report","at":"p.21–22 §2.6.2","quote":"Mid-training phase 1 uses a cosine decay learning rate schedule with a peak learning rate of 2 × 10−5 and a minimum of 1 × 10−5 , without warmup."},"train.batch":{"value":"134M tokens, all phases","source":"report","at":"p.22 §2.6.2","quote":"All phases use a global batch size of 134M tokens."},"train.batch_tokens":{"value":134000000,"source":"report","at":"p.22 §2.6.2","quote":"All phases use a global batch size of 134M tokens."},"train.precision":{"value":"FP8 with delayed scaling (1024-step amax history): E4M3 forward GEMMs, E5M2 data-gradient GEMMs; BF16 weight-gradient GEMMs with FP32 accumulation; BF16 default weights and activations","source":"report","at":"p.22 §2.6.3","quote":"We use FP8 E4M3 for GEMMs in the forward pass, FP8 E5M2 for the data-gradient, and BF16 for the compute of the weight-gradient with FP32 gradient accumulation","note":"FP32 for residual stream, pre-softmax activations, router, MoE combine, embedding/norm/router weights, LM-head GEMM, master weights and AdamW state; stochastic rounding on gradient downcasts."},"train.init":{"value":"normal std 0.02; residual output projections scaled by 1/sqrt(number of residual connections); attention output zero-init via zero output-RMSNorm gains","source":"report","at":"p.21–22 §2.6.2","quote":"we initialize the attention output to zero, achieved by setting the output RMSNorm gains to zero."},"train.data":{"value":"30.0T tokens: code 54.6% (16.4T, 2.22 epochs), STEM 15.8%, web text 14.9%, math 5.4% (5.28 epochs), PDFs 4.7%, books and journals 3.1%, multilingual 1.6%; no LM-generated synthetic data","source":"report","at":"p.20 Table 5","quote":"Code 7.4 16.4 54.6 2.22×","note":"Mid-training mix: STEM/math 35%, code 55%, other 10% (p.20 §2.5.4)."},"train.cutoff":{"value":"web HTML Sep 2025, web PDFs Dec 2025, GitHub code Jun 2025, books and journals Mar 2026","source":"report","at":"p.13 Table 4","quote":"Web HTML pages September 2025"},"train.hardware":{"value":"8,192 GB200 (pre-training, mid-training 1), 4,096 GB200 (mid-training 2); EP64 within NVLink domain, ZeRO-2","source":"report","at":"p.21 §2.6.1","quote":"We use expert parallelism 64 with ZeRO-2 for pre-training and mid-training 1, and we enable ZeRO-3 (FSDP) for mid-training phase 2."},"train.accelerators":{"value":"8,192 GB200","source":"report","at":"p.21 §2.6.1 Table 6","quote":"Phase Tokens Context length GB200 GPUs Pre-training 30 T 16,384 8,192","note":"Pre-training and mid-training 1; mid-training 2 used 4,096."},"train.parallelism":{"value":"EP64 · ZeRO-2 DP","source":"report","at":"p.21 §2.6.1","quote":"We use expert parallelism 64 with ZeRO-2 for pre-training and mid-training 1","note":"§2.8 states TP degree 1 with EP inside the NVL64 domain and DP across racks; mid-training 2 adds ZeRO-3 (FSDP) and context parallelism."},"scaling.ladder":{"value":"L12–L78 (365M/3.9B to 35.6B/1015B active/total): L layers, D = L·256/3, query heads L rounded up to a multiple of 16, 8 KV heads, 8/512 experts; architecture ablations at 100–200 TPP","source":"report","at":"p.7 §2.2.1, Table 1","quote":"most architecture ablations are conducted near the Chinchilla optimal region (Hoffmann et al., 2022) at 100-200 TPP","note":"Table 1 rungs L12, L18, L24, L30, L36, L42, L66, L78; MAI-Base-1 is L78 with latent 3072 instead of 3328 and Expert FFN 10240 instead of 9984."}},"not_disclosed":["mtp.depth","moe.norm_topk","norm.eps","params.active_non_embedding"],"derived":{"shape.aspect":{"value":85.33333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":10240,"formula":"heads × head_dim"},"attn.group_size":{"value":10.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":1.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.7692307692307693,"formula":"moe.expert_dim / d_model"},"moe.latent_ratio":{"value":2.1666666666666665,"formula":"d_model / moe.latent_dim"},"attn.swa_ratio":{"value":5.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.03607068607068607,"formula":"params.active / params.total"},"scaling.tpp":{"value":34.87525987525988,"formula":"train.tokens / params.total"},"scaling.adamw_tau":{"value":0.19970193740685543,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":53248.0,"formula":"53248 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":7115636736.0,"formula":"53248 B × 131072 (Global KV) + 266240 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":54288.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1564.4956772334292,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":205061577.40634003,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":53248.0,"windows":[[266240.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":53248.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":266240.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 13 × (2048 × 16/8) B/entry; SWA KV: 65 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","org":"Xiaomi","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint. Architecture from the RL release; recipe facts from the V2.6 technical report.","sources":{"card":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/dcd3fd83fb181622fbc606fbb1709901982a62c2/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/dcd3fd83fb181622fbc606fbb1709901982a62c2/config.json","kind":"config"},"index":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/dcd3fd83fb181622fbc606fbb1709901982a62c2/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20261010130906/https://mimo.xiaomi.com/mimo-v2-6/article","kind":"blog"},"modeling_mimo_v2":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/dcd3fd83fb181622fbc606fbb1709901982a62c2/modeling_mimo_v2.py","kind":"code"},"report":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/dcd3fd83fb181622fbc606fbb1709901982a62c2/MiMo_V2_6_technical_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019102942/https://artificialanalysis.ai/models/mimo-v2-6-flash","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-09-22","source":"blog","at":"Introducing MiMo-V2.6 series; header","quote":"September 22nd, 2026 Introducing MiMo-V2.6 series","note":"Same post: 'Today, we are releasing and open-sourcing the MiMo-V2.6 series.'"},"meta.license":{"value":"mit","source":"card","at":"README.md; front matter","quote":"license: mit"},"meta.modality":{"value":"text + image + video + audio","source":"card","at":"README.md; Model Summary","quote":"Modalities: Text, Image, Video, Audio"},"params.active":{"value":15000000000,"source":"report","at":"PDF page 6; Table 1","quote":"# Active Parameters 15B 42B"},"params.note":{"value":"Table 1 'Main Block' (text backbone); the card states 309B total. The 681M MiMo-ViT and the audio encoders (308M + 127M) are listed separately; MTP modules are not itemized.","source":"report","at":"PDF page 6; Table 1","quote":"# Total Parameters 310B 1.02T"},"params.total":{"value":310000000000,"source":"report","at":"PDF page 6; Table 1","quote":"# Total Parameters 310B 1.02T"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":152576,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":9,"source":"config","at":"/hybrid_layer_pattern | count 0","note":"0 = full attention, 1 = SWA (modeling_mimo_v2.py MiMoV2DecoderLayer). Global layers at 0, 5, 11, 17, 23, 29, 35, 41, 47."},"attn.local_layers":{"value":39,"source":"config","at":"/hybrid_layer_pattern | count 1"},"attn.head_dim":{"value":192,"source":"config","at":"/head_dim"},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":4,"source":"config","at":"/num_key_value_heads"},"attn.local_heads":{"value":64,"source":"config","at":"/swa_num_attention_heads","note":"MiMoV2Attention.__init__ reads swa_* head counts and dims for SWA layers and the unprefixed keys for global layers."},"attn.local_head_dim":{"value":192,"source":"config","at":"/swa_head_dim","note":"QK dim; V dim is swa_v_head_dim 128."},"attn.local_v_head_dim":{"value":128,"source":"config","at":"/swa_v_head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/swa_rope_theta","note":"MiMoV2RotaryEmbedding(is_swa=True) replaces rope_theta with swa_rope_theta."},"attn.qk_norm":{"value":false,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.forward","quote":"query_states = query_states.view(*input_shape, self.num_attention_heads, self.head_dim).transpose(1, 2)","note":"No norm between projection and RoPE in either layer type."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/swa_num_key_value_heads"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"PDF page 4; §2.1","quote":"interleave Local Sliding Window Attention (SWA) and Global Attention (GA)"},"attn.layout":{"value":"Global at layers 0, 5, 11, 17, 23, 29, 35, 41, 47; SWA elsewhere (0-based)","source":"config","at":"config.json; /hybrid_layer_pattern","quote":"\"hybrid_layer_pattern\": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0 ]","basis":"inferred","note":"N is not given in the report. Config hybrid_layer_pattern has runs of 4, 5, 5, 5, 5, 5, 5, 5 SWA layers between globals, with layers 0 and 47 global. /hybrid_layer_pattern: 1 = SWA, 0 = global (MiMoV2DecoderLayer); ordered backbone only.","sequence":["GQA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.window":{"value":128,"source":"report","at":"PDF page 4; §2.1","quote":"The sliding window size 𝑊 used in MiMo-V2.6 is 128.","note":"Config sliding_window 128 (read by SWA layers)."},"attn.position":{"value":"partial RoPE (64 of 192 QK dims; base 1e7 global, 1e4 SWA)","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.__init__","quote":"self.rope_dim = int(self.head_dim * getattr(config, \"partial_rotary_factor\", 1.0))","note":"partial_rotary_factor 0.334, rope_theta 1e7, swa_rope_theta 1e4 in config."},"attn.rope_fraction":{"value":0.334,"source":"config","at":"/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_theta"},"attn.sinks":{"value":"learnable per-head sink bias on SWA layers only","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.__init__","quote":"or (getattr(config, \"add_swa_attention_sink_bias\", False) and is_swa)","note":"Config: add_swa_attention_sink_bias true, add_full_attention_sink_bias false."},"attn.scale":{"value":"values × 0.707","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention._forward_attention","quote":"value_states = value_states * self.v_scale","note":"v_scale is config attention_value_scale (0.707), applied in all attention layers; not mentioned in the report. QK scaling is the standard head_dim**-0.5."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"ffn.activation":{"value":"SwiGLU","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))","note":"act_fn is config hidden_act silu; routed experts use the same MiMoV2MLP."},"ffn.dim":{"value":16384,"source":"config","at":"/intermediate_size","note":"Dense FFN in layer 0 only."},"moe.dense_layers":{"value":1,"source":"report","at":"PDF page 4; §2.1","quote":"The only exception is the very first Transformer block, which uses global attention with a dense Feed-Forward Network","note":"Config moe_layer_freq is 0 for layer 0 only."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/n_routed_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func","note":"topk_method noaux_tc: MiMoV2MoEGate adds e_score_correction_bias to the scores for top-k selection only; how it was trained is not stated."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.shared":{"value":0,"source":"report","at":"PDF page 4; §2.1","quote":"Both the SWA block and the GA block utilize a sparse MoE FFN without shared experts."},"mtp.ffn":{"value":"dense","source":"report","at":"PDF page 4; §2.1","quote":"MiMo-V2.6 also integrates an SWA and dense FFN based MTP"},"mtp.shares_embedding":{"value":true,"source":"report","at":"PDF page 5; Figure 2","quote":"LM Head (tied)","note":"Figure 2 labels the MTP block's embedding and LM head as tied to the main model's; the weight index has no MTP embedding or head weights."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.2.eh_proj.weight\"","note":"Native MTP: three SWA + dense-FFN modules, used in pretraining (§2.1) with the count during pretraining not stated. The 5-layer DFlash drafter (dflash/), which the card calls MTP, is a separate speculative decoder."},"norm.placement":{"value":"pre","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.forward","quote":"residual = hidden_states hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.__init__","quote":"self.input_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)"},"norm.eps":{"value":1e-06,"source":"config","at":"/layernorm_epsilon"},"ctx.extension":{"value":"32K to 256K partway through pretraining; mid-training at 256K, then 1M in its final stage","source":"report","at":"PDF page 7; §3.1, §3.2","quote":"We begin pre-training with a context length of 32K tokens and extend it to 256K partway through training."},"ctx.final":{"value":1048576,"source":"report","at":"PDF page 7; §3.2","quote":"then extend the context length to 1M in the final stage.","note":"Reached in mid-training, after the 48T-token pretraining."},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Model Summary","quote":"Context Length: 1M tokens","note":"Config max_position_embeddings 1048576."},"ctx.pretrain":{"value":32768,"source":"report","at":"PDF page 7; §3.1","quote":"We begin pre-training with a context length of 32K tokens and extend it to 256K partway through training."},"train.optimizer":{"value":"AdamW","source":"report","at":"PDF page 7; §3.1","quote":"We employ the AdamW optimizer in the pre-training of MiMo-V2.6.","note":"Mid-training switches hidden weight matrices to Muown (a Muon variant); embeddings, LM head and router stay on AdamW."},"train.data":{"value":"text (web, books, papers, code, STEM); vision (captioning, grounding, OCR, GUI, conversation, video, visual coding); audio (speech-text interleaving, ASR, audio captioning)","source":"report","at":"PDF page 7; §3.1","quote":"The text corpus draws from a broad range of sources, including public web content, books, academic papers, code, and STEM materials."},"train.stages":{"value":[{"name":"text","tokens":26000000000000,"note":"Text-only language backbone.","quote":"MiMo-V2.6-Flash is trained on 48T tokens, comprising 26T tokens in the text stage and 22T in the omni stage."},{"name":"omni","tokens":22000000000000,"note":"Joint training with the ViT and audio encoder on omni-modal data.","quote":"MiMo-V2.6-Flash is trained on 48T tokens, comprising 26T tokens in the text stage and 22T in the omni stage."}],"source":"report","at":"PDF page 7; §3.1"},"train.tokens":{"value":48000000000000,"source":"report","at":"PDF page 7; §3.1","quote":"MiMo-V2.6-Flash is trained on 48T tokens, comprising 26T tokens in the text stage and 22T in the omni stage.","note":"Excludes mid-training, whose token count is not stated."},"eval.aa_index":{"value":38,"source":"aa","at":"Artificial Analysis model page /models/mimo-v2-6-flash, Wayback snapshot 2026-10-19","quote":"MiMo-V2.6-Flash scores 38 on the Artificial Analysis Intelligence Index","note":"Variant: MiMo-V2.6-Flash. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","moe.balancing","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":308778780864,"formula":"embedding + output head + all experts + other weights, XiaomiMiMo/MiMo-V2.6-Flash-RL@dcd3fd8 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_embedding":{"value":1249902592,"formula":"input embedding + output head, XiaomiMiMo/MiMo-V2.6-Flash-RL@dcd3fd8 safetensors headers"},"params.counted_non_embedding":{"value":307528878272,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, XiaomiMiMo/MiMo-V2.6-Flash-RL@dcd3fd8 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_max_layer":{"value":6537879872,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, XiaomiMiMo/MiMo-V2.6-Flash-RL@dcd3fd8 safetensors headers"},"params.counted_active_non_embedding":{"value":14196033728.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, XiaomiMiMo/MiMo-V2.6-Flash-RL@dcd3fd8 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"shape.aspect":{"value":85.33333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":4.333333333333333,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04838709677419355,"formula":"params.active / params.total"},"scaling.tpp":{"value":154.83870967741936,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":3381.2261170756215,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":23040.0,"formula":"23040 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3045457920.0,"formula":"23040 B × 131072 (Global KV) + 199680 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":23235.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1549.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":203030528.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":23040.0,"windows":[[199680.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":23040.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":199680.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","org":"Xiaomi","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint. Architecture from the RL release.","sources":{"card":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/b61f21852680ee02a29e946e18cf8697f4844061/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/b61f21852680ee02a29e946e18cf8697f4844061/config.json","kind":"config"},"index":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/b61f21852680ee02a29e946e18cf8697f4844061/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20261010130906/https://mimo.xiaomi.com/mimo-v2-6/article","kind":"blog"},"modeling_mimo_v2":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/b61f21852680ee02a29e946e18cf8697f4844061/modeling_mimo_v2.py","kind":"code"},"report":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/b61f21852680ee02a29e946e18cf8697f4844061/MiMo_V2_6_technical_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019092917/https://artificialanalysis.ai/models/mimo-v2-6-pro","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-09-22","source":"blog","at":"Introducing MiMo-V2.6 series; header","quote":"September 22nd, 2026 Introducing MiMo-V2.6 series","note":"Same post: 'Today, we are releasing and open-sourcing the MiMo-V2.6 series.'"},"meta.license":{"value":"mit","source":"card","at":"README.md; front matter","quote":"license: mit"},"meta.modality":{"value":"text + image + video + audio","source":"card","at":"README.md; Model Summary","quote":"Modalities: Text, Image, Video, Audio"},"params.active":{"value":42000000000,"source":"report","at":"PDF page 6; Table 1","quote":"# Active Parameters 15B 42B"},"params.note":{"value":"Table 1 'Main Block' (text backbone). The 681M MiMo-ViT and the audio encoders (308M + 127M) are listed separately; MTP modules are not itemized.","source":"report","at":"PDF page 6; Table 1","quote":"# Total Parameters 310B 1.02T"},"params.total":{"value":1020000000000,"source":"report","at":"PDF page 6; Table 1","quote":"# Total Parameters 310B 1.02T"},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":70,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":152576,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":10,"source":"config","at":"/hybrid_layer_pattern | count 0","note":"0 = full attention, 1 = SWA (modeling_mimo_v2.py MiMoV2DecoderLayer). Global layers at 0, 7, 15, 23, 31, 39, 47, 55, 62, 69."},"attn.head_dim":{"value":192,"source":"config","at":"/head_dim"},"attn.heads":{"value":128,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"PDF page 4; §2.1","quote":"interleave Local Sliding Window Attention (SWA) and Global Attention (GA)"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_heads":{"value":128,"source":"config","at":"/swa_num_attention_heads","note":"MiMoV2Attention.__init__ reads swa_* head counts and dims for SWA layers and the unprefixed keys for global layers."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/swa_num_key_value_heads"},"attn.local_head_dim":{"value":192,"source":"config","at":"/swa_head_dim","note":"QK dim; V dim is swa_v_head_dim 128."},"attn.local_v_head_dim":{"value":128,"source":"config","at":"/swa_v_head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/swa_rope_theta","note":"MiMoV2RotaryEmbedding(is_swa=True) replaces rope_theta with swa_rope_theta."},"attn.qk_norm":{"value":false,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.forward","quote":"query_states = query_states.view(*input_shape, self.num_attention_heads, self.head_dim).transpose(1, 2)","note":"No norm between projection and RoPE in either layer type."},"attn.layout":{"value":"Global at layers 0, 7, 15, 23, 31, 39, 47, 55, 62, 69; SWA elsewhere (0-based)","source":"config","at":"config.json; /hybrid_layer_pattern","quote":"\"hybrid_layer_pattern\": [ 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0 ]","note":"Irregular: /hybrid_layer_pattern has runs of 6, 7, 7, 7, 7, 7, 7, 6, 6 SWA between globals at 0, 7, 15, 23, 31, 39, 47, 55, 62, 69. /hybrid_layer_pattern: 1 = SWA, 0 = global (MiMoV2DecoderLayer); ordered backbone only.","basis":"inferred","sequence":["GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.local_layers":{"value":60,"source":"config","at":"/hybrid_layer_pattern | count 1"},"attn.position":{"value":"partial RoPE (64 of 192 QK dims; base 1e7 global, 1e4 SWA)","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.__init__","quote":"self.rope_dim = int(self.head_dim * getattr(config, \"partial_rotary_factor\", 1.0))","note":"partial_rotary_factor 0.334, rope_theta 1e7, swa_rope_theta 1e4 in config."},"attn.rope_fraction":{"value":0.334,"source":"config","at":"/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"attn.scale":{"value":"values × 0.612 in every attention layer; QK scale 1/sqrt(192)","source":"config","at":"config.json; attention_value_scale","quote":"\"attention_value_scale\": 0.612","note":"modeling_mimo_v2.py MiMoV2Attention multiplies value_states by attention_value_scale before attention."},"attn.sinks":{"value":"per-head sink bias in SWA layers; none in global layers","source":"config","at":"config.json; add_*_attention_sink_bias","quote":"\"add_full_attention_sink_bias\": false, \"add_swa_attention_sink_bias\": true","note":"modeling_mimo_v2.py concatenates attention_sink_bias (one logit per head) to the attention logits before softmax."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.window":{"value":128,"source":"report","at":"PDF page 4; §2.1","quote":"The sliding window size 𝑊 used in MiMo-V2.6 is 128.","note":"Config sliding_window 128 (read by SWA layers)."},"ffn.activation":{"value":"SwiGLU","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))","note":"act_fn = ACT2FN[/hidden_act] = silu; used for the dense FFN and every routed expert."},"ffn.bias":{"value":false,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"MiMoV2MLP is used for the dense FFN and every routed expert."},"ffn.dim":{"value":16384,"source":"config","at":"/intermediate_size","note":"Dense FFN in layer 0 only."},"moe.dense_layers":{"value":1,"source":"report","at":"PDF page 4; §2.1","quote":"The only exception is the very first Transformer block, which uses global attention with a dense Feed-Forward Network","note":"Config moe_layer_freq is 0 for layer 0 only."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/n_routed_experts"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func","note":"topk_method noaux_tc: MiMoV2MoEGate adds e_score_correction_bias to the scores for top-k selection only; how it was trained is not stated."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.shared":{"value":0,"source":"report","at":"PDF page 4; §2.1","quote":"Both the SWA block and the GA block utilize a sparse MoE FFN without shared experts."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.ffn":{"value":"dense","source":"report","at":"PDF page 4; §2.1","quote":"MiMo-V2.6 also integrates an SWA and dense FFN based MTP"},"mtp.shares_embedding":{"value":true,"source":"report","at":"PDF page 5; Figure 2","quote":"LM Head (tied)","note":"Figure 2 labels the MTP block's embedding and LM head as tied to the main model's; the weight index has no MTP embedding or head weights."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.2.eh_proj.weight\"","note":"Three SWA + dense-FFN MTP modules (model.mtp.layers.0–2) in model_mtp.safetensors; report §2.1 uses MTP in pretraining without stating the count. The card's 5-layer DFlash speculative decoder is a separate drafter."},"norm.eps":{"value":1e-05,"source":"config","at":"/layernorm_epsilon"},"norm.placement":{"value":"pre","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.forward","quote":"residual = hidden_states hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.__init__","quote":"self.input_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)"},"ctx.extension":{"value":"32K to 256K partway through pretraining; mid-training at 256K, then 1M in its final stage","source":"report","at":"PDF page 7; §3.1, §3.2","quote":"We begin pre-training with a context length of 32K tokens and extend it to 256K partway through training."},"ctx.final":{"value":1048576,"source":"report","at":"PDF page 7; §3.2","quote":"then extend the context length to 1M in the final stage.","note":"Reached in mid-training, after the 30T-token pretraining."},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Model Summary","quote":"Context Length: 1M tokens","note":"Config max_position_embeddings 1048576."},"ctx.pretrain":{"value":32768,"source":"report","at":"PDF page 7; §3.1","quote":"We begin pre-training with a context length of 32K tokens and extend it to 256K partway through training."},"train.data":{"value":"Stage 1 text: web, books, academic papers, code, STEM. Stage 2 adds vision (captioning, grounding, OCR, GUI, video, visual coding) and audio (speech-text interleaving, ASR, audio captioning)","source":"report","at":"PDF page 7; §3.1","quote":"The text corpus draws from a broad range of sources, including public web content, books, academic papers, code, and STEM materials."},"train.optimizer":{"value":"AdamW","source":"report","at":"PDF page 7; §3.1","quote":"We employ the AdamW optimizer in the pre-training of MiMo-V2.6.","note":"Mid-training switches hidden weight matrices to Muown (a Muon variant); embeddings, LM head and router stay on AdamW."},"train.stages":{"value":[{"name":"text","tokens":27000000000000,"note":"Text-only language backbone.","quote":"MiMo-V2.6-Pro is trained on 30T tokens, with 27T and 3T tokens in the two stages"},{"name":"omni","tokens":3000000000000,"note":"Joint training with the ViT and audio encoder on omni-modal data.","quote":"MiMo-V2.6-Pro is trained on 30T tokens, with 27T and 3T tokens in the two stages"}],"source":"report","at":"PDF page 7; §3.1"},"train.tokens":{"value":30000000000000,"source":"report","at":"PDF page 7; §3.1","quote":"MiMo-V2.6-Pro is trained on 30T tokens, with 27T and 3T tokens in the two stages","note":"Excludes mid-training, whose token count is not stated."},"eval.aa_index":{"value":46,"source":"aa","at":"Artificial Analysis model page /models/mimo-v2-6-pro, Wayback snapshot 2026-10-19","quote":"MiMo-V2.6-Pro scores 46 on the Artificial Analysis Intelligence Index","note":"Variant: MiMo-V2.6-Pro. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","moe.balancing","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":1021248126336,"formula":"embedding + output head + all experts + other weights, XiaomiMiMo/MiMo-V2.6-Pro-RL@b61f218 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1874853888,"formula":"input embedding + output head, XiaomiMiMo/MiMo-V2.6-Pro-RL@b61f218 safetensors headers"},"params.counted_non_embedding":{"value":1019373272448,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, XiaomiMiMo/MiMo-V2.6-Pro-RL@b61f218 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":14765273600,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, XiaomiMiMo/MiMo-V2.6-Pro-RL@b61f218 safetensors headers"},"params.counted_active_non_embedding":{"value":40020065664.0,"formula":"non-expert weights + routed experts × 8/384, excluding embedding and output head, XiaomiMiMo/MiMo-V2.6-Pro-RL@b61f218 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":87.77142857142857,"formula":"d_model / layers"},"attn.qk_width":{"value":24576,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.6666666666666665,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":6.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":48.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.041176470588235294,"formula":"params.active / params.total"},"scaling.tpp":{"value":29.41176470588235,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":749.6239574385921,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":51200.0,"formula":"51200 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6750208000.0,"formula":"51200 B × 131072 (Global KV) + 307200 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":51500.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1226.1904761904761,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":160719238.0952381,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":51200.0,"windows":[[307200.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":51200.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":307200.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mimo-v2-flash-base","name":"MiMo-V2-Flash-Base","org":"Xiaomi","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash-Base/resolve/e4187012dd10281638071e417a56164dd7805fbd/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash-Base/resolve/e4187012dd10281638071e417a56164dd7805fbd/config.json","kind":"config"},"index":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash-Base/resolve/e4187012dd10281638071e417a56164dd7805fbd/model.safetensors.index.json","kind":"config"},"modeling_mimo_v2_flash":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash-Base/resolve/e4187012dd10281638071e417a56164dd7805fbd/modeling_mimo_v2_flash.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2601.02780v2","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20261019100652/https://mimo.xiaomi.com/mimo-v2-flash","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019103144/https://artificialanalysis.ai/models/mimo-v2-flash-reasoning","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-12-16","source":"blog","at":"Introducing MiMo-V2-Flash; header","quote":"December 16, 2025 Today, we are releasing and open-sourcing MiMo-V2-Flash."},"meta.license":{"value":"mit","source":"card","at":"README.md; front matter","quote":"license: mit"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; intro","quote":"MiMo-V2-Flash is a Mixture-of-Experts (MoE) language model","note":"No explicit modality statement; config.json (MiMoV2FlashForCausalLM) has no vision or audio config."},"params.total":{"value":309000000000,"source":"report","at":"p.10 §3.2","quote":"Overall, MiMo-V2-Flash has 309B total parameters, of which 15B are active."},"params.active":{"value":15000000000,"source":"report","at":"p.10 §3.2","quote":"Overall, MiMo-V2-Flash has 309B total parameters, of which 15B are active."},"params.note":{"value":"MTP inclusion not stated; each MTP block is 0.33B.","source":"report","at":"p.5 §2.1","quote":"The number of parameters for each MTP block is only 0.33B."},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":152576,"source":"config","at":"/vocab_size"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.5 §2.1","quote":"The model consists of 39 SWA layers and 9 GA layers. Both SWA and GA utilize Grouped-Query Attention (GQA)"},"attn.layout":{"value":"Global at layers 0, 5, 11, 17, 23, 29, 35, 41, 47; SWA elsewhere (0-based)","source":"config","at":"config.json; /hybrid_layer_pattern","quote":"\"hybrid_layer_pattern\": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0 ]","note":"Config hybrid_layer_pattern puts global layers at 0, 5, 11, 17, 23, 29, 35, 41, 47: 4 SWA layers before the first hybrid GA, 5 before each later one. /hybrid_layer_pattern: 1 = SWA, 0 = global (MiMoV2DecoderLayer); ordered backbone only.","basis":"inferred","sequence":["GQA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":9,"source":"report","at":"p.5 §2.1","quote":"The model consists of 39 SWA layers and 9 GA layers."},"attn.local_layers":{"value":39,"source":"report","at":"p.5 §2.1","quote":"The model consists of 39 SWA layers and 9 GA layers."},"attn.heads":{"value":64,"source":"report","at":"p.6 §2.1","quote":"SWA has 64 query heads and 8 key-value heads, while GA has 64 query heads and 4 key-value heads.","note":"Config num_attention_heads 64."},"attn.kv_heads":{"value":4,"source":"report","at":"p.6 §2.1","quote":"SWA has 64 query heads and 8 key-value heads, while GA has 64 query heads and 4 key-value heads.","note":"Config num_key_value_heads 4."},"attn.local_kv_heads":{"value":8,"source":"report","at":"p.6 §2.1","quote":"SWA has 64 query heads and 8 key-value heads, while GA has 64 query heads and 4 key-value heads.","note":"Config swa_num_key_value_heads 8."},"attn.local_heads":{"value":64,"source":"config","at":"/swa_num_attention_heads","note":"MiMoV2Attention.__init__ reads swa_* head counts and dims for SWA layers and the unprefixed keys for GA layers."},"attn.local_head_dim":{"value":192,"source":"config","at":"/swa_head_dim","note":"QK dim; V dim is swa_v_head_dim 128."},"attn.local_v_head_dim":{"value":128,"source":"config","at":"/swa_v_head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/swa_rope_theta","note":"MiMoV2FlashRotaryEmbedding(is_swa=True) replaces rope_theta with swa_rope_theta."},"attn.qk_norm":{"value":false,"source":"modeling_mimo_v2_flash","at":"modeling_mimo_v2_flash.py; MiMoV2Attention.forward","quote":"query_states = self.q_proj(hidden_states).view(qk_hidden_shape).transpose(1, 2)","note":"No norm between projection and RoPE in either layer type."},"attn.head_dim":{"value":192,"source":"report","at":"p.6 §2.1","quote":"(192 for queries and keys, and 128 for values)","note":"Config head_dim 192."},"attn.v_head_dim":{"value":128,"source":"report","at":"p.6 §2.1","quote":"(192 for queries and keys, and 128 for values)","note":"Config v_head_dim 128."},"attn.window":{"value":128,"source":"report","at":"p.5 §2.1","quote":"The sliding window size 𝑊 used in MiMo-V2-Flash is 128.","note":"Config sliding_window 128 (read by the SWA mask)."},"attn.position":{"value":"partial RoPE (first 64 of 192 QK dims)","source":"report","at":"p.6 §2.1","quote":"is partially applied to the first 64 dimensions query and key."},"attn.rope_fraction":{"value":0.334,"source":"config","at":"/partial_rotary_factor","note":"Config value; MiMoV2Attention rotates int(192 × 0.334) = 64 dims, i.e. 1/3."},"attn.rope_theta":{"value":5000000,"source":"config","at":"/rope_theta","note":"640,000 during the 32K stages; raised to 5,000,000 for the 256K extension (§3.2). SWA layers use 10,000."},"attn.sinks":{"value":"learnable per-head sink bias in the softmax denominator (gpt-oss style), SWA layers only","source":"report","at":"p.6 §2.2","quote":"our implementation follows the design used in gpt-oss","note":"Config: add_swa_attention_sink_bias true, add_full_attention_sink_bias false."},"attn.scale":{"value":"values × 0.707","source":"modeling_mimo_v2_flash","at":"modeling_mimo_v2_flash.py; MiMoV2Attention.forward","quote":"value_states = value_states * self.v_scale","note":"v_scale is config attention_value_scale (0.707), applied in all attention layers; not mentioned in the report. QK scaling is the standard head_dim**-0.5."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"ffn.activation":{"value":"SwiGLU","source":"modeling_mimo_v2_flash","at":"modeling_mimo_v2_flash.py; MiMoV2MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))","note":"act_fn is config hidden_act silu; routed experts use the same MiMoV2MLP."},"ffn.dim":{"value":16384,"source":"report","at":"p.10 §3.2","quote":"The intermediate hidden dimension of the FFN of dense layers is set to 16384.","note":"Dense FFN in layer 0 only; config intermediate_size 16384."},"moe.dense_layers":{"value":1,"source":"report","at":"p.10 §3.2","quote":"All layers except the first are equipped with sparse MoE.","note":"Config moe_layer_freq is 0 for layer 0 only."},"moe.experts":{"value":256,"source":"report","at":"p.10 §3.2","quote":"Each MoE layer contains 256 routed experts, with 8 experts activated per token, and an intermediate hidden dimension of 2048 for each expert.","note":"Config n_routed_experts 256."},"moe.top_k":{"value":8,"source":"report","at":"p.10 §3.2","quote":"Each MoE layer contains 256 routed experts, with 8 experts activated per token, and an intermediate hidden dimension of 2048 for each expert.","note":"Config num_experts_per_tok 8."},"moe.expert_dim":{"value":2048,"source":"report","at":"p.10 §3.2","quote":"Each MoE layer contains 256 routed experts, with 8 experts activated per token, and an intermediate hidden dimension of 2048 for each expert.","note":"Config moe_intermediate_size 2048."},"moe.shared":{"value":0,"source":"report","at":"p.5 §2.1","quote":"with 8 activated per token, and contains no shared experts."},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.balancing":{"value":"aux-loss-free expert bias (update 0.001 in stages 1–2, 1e-5 in stage 3) + sequence aux loss 1e-5","source":"report","at":"p.10 §3.2","quote":"the MoE sequence auxiliary loss coefficient is set to 1.0 × 10−5 for all stages.","note":"Config topk_method noaux_tc: the expert bias is added to scores for selection only."},"mtp.depth":{"value":1,"source":"report","at":"p.10 §3.2","quote":"The model uses a single MTP layer during pre-training."},"mtp.modules":{"value":1,"source":"report","at":"p.9 §2.3.2","quote":"During pre-training, only a single MTP head is attached to the model to avoid extra training overhead.","note":"Released checkpoint holds three modules: the head replicated and trained jointly in post-training."},"mtp.block":{"value":"SWA (64 Q / 8 KV heads, sink bias) + dense SwiGLU FFN; RMSNorm'd hidden state and next-token embedding concatenated and linearly projected","source":"report","at":"p.5 Figure 2 caption; §2.1","quote":"The MTP blocks employ SWA and a dense FFN.","note":"Input combination from Figure 2 (two RMSNorms, Linear) and index weights enorm, hnorm, eh_proj; SWA heads from Table 1."},"mtp.ffn":{"value":"dense","source":"report","at":"p.9 §2.3.2","quote":"We use a small dense FFN rather than MoE to limit parameter count","note":"Width not stated."},"mtp.shares_embedding":{"value":true,"source":"report","at":"p.5 Figure 2","quote":"LM Head (tied)","note":"Figure 2 labels the MTP block's embedding and LM head as tied to the main model's; the weight index has no MTP embedding or head weights."},"mtp.loss_weight":{"value":"0.3 in stage 1; 0.1 in stages 2 and 3","source":"report","at":"p.10 §3.2","quote":"The MTP loss weight is set to 0.3 for Stage 1 and 0.1 for Stage 2 and 3."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.2.eh_proj.weight\"","note":"Three SWA + dense-FFN MTP blocks; the single pretraining head is replicated and trained jointly in post-training (§2.3.2)."},"norm.type":{"value":"RMSNorm","source":"modeling_mimo_v2_flash","at":"modeling_mimo_v2_flash.py; MiMoV2DecoderLayer.__init__","quote":"self.input_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)"},"norm.placement":{"value":"pre","source":"modeling_mimo_v2_flash","at":"modeling_mimo_v2_flash.py; MiMoV2DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.eps":{"value":1e-05,"source":"config","at":"/layernorm_epsilon"},"ctx.pretrain":{"value":32768,"source":"report","at":"p.10 §3.2","quote":"we set the pre-training sequence length to 32,768"},"ctx.final":{"value":262144,"source":"report","at":"p.10 §3.2","quote":"the sequence length is extended to 262,144"},"ctx.max":{"value":262144,"source":"card","at":"README.md; §1 Introduction","quote":"The context window supports up to 256k length.","note":"Config max_position_embeddings 262144."},"ctx.extension":{"value":"Stage 3 (26T–27T tokens): 32K → 256K, GA RoPE base 640K → 5M","source":"report","at":"p.10 §3.2","quote":"In Stage 3, the sequence length is extended to 262,144, and the RoPE base frequency of GA is adjusted to 5,000,000."},"train.tokens":{"value":27000000000000,"source":"report","at":"p.1 Abstract","quote":"The model is pre-trained on 27 trillion tokens with Multi-Token Prediction (MTP)","note":"Includes mid-training (stage 2) and context extension (stage 3)."},"train.optimizer":{"value":"AdamW","source":"report","at":"p.10 §3.2","quote":"We employ the AdamW optimizer with 𝛽1 = 0.9, 𝛽2 = 0.95, and a weight decay of 0.1."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"p.10 §3.2","quote":"We employ the AdamW optimizer with 𝛽1 = 0.9, 𝛽2 = 0.95, and a weight decay of 0.1."},"train.weight_decay":{"value":0.1,"source":"report","at":"p.10 §3.2","quote":"We employ the AdamW optimizer with 𝛽1 = 0.9, 𝛽2 = 0.95, and a weight decay of 0.1."},"train.grad_clip":{"value":1.0,"source":"report","at":"p.10 §3.2","quote":"Gradient clipping is applied with a maximum norm of 1.0."},"train.lr_peak":{"value":0.00032,"source":"report","at":"p.10 §3.2","quote":"the learning rate starts with a linear warm-up from 0 to 3.2 × 10−4 over the first 50B tokens"},"train.lr_min":{"value":1e-05,"source":"report","at":"p.10 §3.2","quote":"The learning rate in Stage 3 decays from 3.0 × 10−5 to 1.0 × 10−5 following a cosine schedule"},"train.warmup":{"value":"50B tokens","source":"report","at":"p.10 §3.2","quote":"the learning rate starts with a linear warm-up from 0 to 3.2 × 10−4 over the first 50B tokens"},"train.schedule":{"value":"Stage 1: 50B warmup, constant 3.2e-4 for 12T, cosine to 1e-4 over 10T; stage 2: cosine 1e-4 → 3e-5 over 4T; stage 3: cosine 3e-5 → 1e-5","source":"report","at":"p.10 §3.2","quote":"followed by a constant phase at 3.2 × 10−4 for 12T tokens, and concludes with a cosine decay to 1.0 × 10−4 over 10T tokens."},"train.batch":{"value":"2,048 × 32,768 (67.1M tokens) after a linear warmup over the first 500B tokens; 256 × 262,144 (67.1M tokens) in stage 3","source":"report","at":"p.10 §3.2","quote":"The batch size warms up linearly to 2,048 over the initial 500B tokens and remains constant for the remainder of both stages.","note":"The report does not give the unit; read as sequences because both stages then give the same tokens per step."},"train.batch_tokens":{"value":67108864,"source":"report","at":"p.10 §3.2","quote":"The batch size warms up linearly to 2,048 over the initial 500B tokens","basis":"inferred","note":"Unit not stated; read as sequences: 2,048 × 32,768. Stage 3's 256 × 262,144 gives the same total."},"train.precision":{"value":"FP8 mixed precision (DeepSeek-V3-style); BF16 attention output projections, embedding and output head; FP32 router","source":"report","at":"p.6 §2.1","quote":"we retain BF16 precision for the attention output projections, as well as for the embedding and output head parameters, while maintaining FP32 precision for the MoE router parameters."},"train.init":{"value":"std 0.006 for all parameters","source":"report","at":"p.10 §3.2","quote":"All learnable parameters are randomly initialized with a standard deviation of 0.006."},"train.data":{"value":"web, books, papers, code, math, STEM; upweighted long-form web documents and repository-level code, PRs, issues and commits; ~5% synthetic reasoning data from stage 2","source":"report","at":"pp.9–10 §3, §3.1","quote":"we emphasize long-form web documents and carefully curated code corpora"},"train.stages":{"value":[{"name":"pre-training","tokens":22000000000000,"seq_len":32768,"lr":0.00032,"batch_sequences":2048,"note":"General corpus; warmup, constant, then cosine to 1e-4.","quote":"Stage 1 (Pre-training, 0 – 22T)."},{"name":"mid-training","tokens":4000000000000,"note":"Code upsampled, ~5% synthetic reasoning data; LR cosine 1e-4 → 3e-5.","quote":"Stage 2 (Mid-training, 22 – 26T)."},{"name":"context-extension","tokens":1000000000000,"seq_len":262144,"batch_sequences":256,"note":"Stage 2 mix with long-range-dependency data upsampled; LR cosine 3e-5 → 1e-5.","quote":"Stage 3 (Context Extension, 26 – 27T)."}],"source":"report","at":"p.10 §3.1, §3.2"},"scaling.ladder":{"value":"attention ablations on a 32B dense model: 250B tokens at 8K, then 40B at 32K","source":"report","at":"p.6–7 §2.2.1","quote":"we conduct exploratory and empirical studies on a 32B dense model","note":"Token budget from §2.2.1 Baselines and Benchmarks."},"eval.aa_index":{"value":21,"source":"aa","at":"Artificial Analysis model page /models/mimo-v2-flash-reasoning, Wayback snapshot 2026-10-19","quote":"MiMo-V2-Flash (Reasoning) scores 21 on the Artificial Analysis Intelligence Index","note":"Score of MiMo-V2-Flash, post-trained from this base. Variant: MiMo-V2-Flash (Reasoning). AA entry for the December 2025 release. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding"],"derived":{"params.counted_total":{"value":308778780864,"formula":"embedding + output head + all experts + other weights, XiaomiMiMo/MiMo-V2-Flash-Base@e418701 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":1249902592,"formula":"input embedding + output head, XiaomiMiMo/MiMo-V2-Flash-Base@e418701 safetensors headers"},"params.counted_non_embedding":{"value":307528878272,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, XiaomiMiMo/MiMo-V2-Flash-Base@e418701 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":6537879872,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, XiaomiMiMo/MiMo-V2-Flash-Base@e418701 safetensors headers"},"params.counted_active_non_embedding":{"value":14196033728.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, XiaomiMiMo/MiMo-V2-Flash-Base@e418701 safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":85.33333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":4.333333333333333,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04854368932038835,"formula":"params.active / params.total"},"scaling.tpp":{"value":87.37864077669903,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1901.939690855037,"formula":"train.tokens / params.counted_active_non_embedding"},"scaling.adamw_tau":{"value":0.07767229629629628,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":23040.0,"formula":"23040 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3045457920.0,"formula":"23040 B × 131072 (Global KV) + 199680 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":23235.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1549.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":203030528.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":23040.0,"windows":[[199680.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":23040.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":199680.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 9 × (1280 × 16/8) B/entry; SWA KV: 39 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mimo25-pro-base","name":"MiMo-V2.5-Pro-Base","org":"Xiaomi","country":"China","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-Base/resolve/1e94ccfda10a2e4dc62877139cd8b7d7b523a4be/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-Base/resolve/1e94ccfda10a2e4dc62877139cd8b7d7b523a4be/config.json","kind":"config"},"index":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-Base/resolve/1e94ccfda10a2e4dc62877139cd8b7d7b523a4be/model.safetensors.index.json","kind":"config"},"modeling_mimo_v2":{"url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-Base/resolve/1e94ccfda10a2e4dc62877139cd8b7d7b523a4be/modeling_mimo_v2.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20261011084411/https://mimo.xiaomi.com/mimo-v2-5-pro/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019103314/https://artificialanalysis.ai/models/mimo-v2-5-pro","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-27","source":"blog","at":"MiMo-V2.5-Pro blog; header and Open Source section","quote":"April 27th, 2026 Xiaomi MiMo- V2.5 -Pro","note":"The blog's model table lists the Base checkpoint for download."},"meta.license":{"value":"mit","source":"card","at":"README.md; front matter","quote":"license: mit"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; MiMo-V2.5-Pro","quote":"MiMo-V2.5-Pro is an open-source Mixture-of-Experts (MoE) language model","note":"No explicit modality statement; config.json (MiMoV2ForCausalLM) has no vision or audio config, unlike the omnimodal MiMo-V2.5/V2.6 releases."},"params.active":{"value":42000000000,"source":"card","at":"README.md; intro","quote":"with 1.02T total parameters and 42B active parameters"},"params.note":{"value":"Publisher's rounded figures; whether the three MTP modules are included is not stated.","source":"card","at":"README.md; intro","quote":"with 1.02T total parameters and 42B active parameters"},"params.total":{"value":1020000000000,"source":"card","at":"README.md; intro","quote":"with 1.02T total parameters and 42B active parameters"},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.layers":{"value":70,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":152576,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":10,"source":"config","at":"/hybrid_layer_pattern | count 0","note":"0 = full attention, 1 = SWA (modeling_mimo_v2.py MiMoV2DecoderLayer). Global layers at 0, 7, 15, 23, 31, 39, 47, 55, 62, 69."},"attn.head_dim":{"value":192,"source":"config","at":"/head_dim"},"attn.heads":{"value":128,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, SWA","source":"card","at":"README.md; §1 Introduction","quote":"Interleaves Sliding Window Attention (SWA) and Global Attention (GA)"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_heads":{"value":128,"source":"config","at":"/swa_num_attention_heads","note":"MiMoV2Attention.__init__ reads swa_* head counts and dims for SWA layers and the unprefixed keys for global layers."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/swa_num_key_value_heads"},"attn.local_head_dim":{"value":192,"source":"config","at":"/swa_head_dim","note":"QK dim; V dim is swa_v_head_dim 128."},"attn.local_v_head_dim":{"value":128,"source":"config","at":"/swa_v_head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/swa_rope_theta","note":"MiMoV2RotaryEmbedding(is_swa=True) replaces rope_theta with swa_rope_theta."},"attn.qk_norm":{"value":false,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.forward","quote":"query_states = query_states.view(*input_shape, self.num_attention_heads, self.head_dim).transpose(1, 2)","note":"No norm between projection and RoPE in either layer type."},"attn.layout":{"value":"Global at layers 0, 7, 15, 23, 31, 39, 47, 55, 62, 69; SWA elsewhere (0-based)","source":"config","at":"config.json; /hybrid_layer_pattern","quote":"\"hybrid_layer_pattern\": [ 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0 ]","basis":"inferred","note":"Card states 6:1; /hybrid_layer_pattern has 60 SWA and 10 global layers, in runs of 6, 7, 7, 7, 7, 7, 7, 6, 6 SWA between globals at 0, 7, 15, 23, 31, 39, 47, 55, 62, 69. /hybrid_layer_pattern: 1 = SWA, 0 = global (MiMoV2DecoderLayer); ordered backbone only.","sequence":["GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","SWA","SWA","SWA","GQA"]},"attn.local_layers":{"value":60,"source":"config","at":"/hybrid_layer_pattern | count 1"},"attn.position":{"value":"partial RoPE (64 of 192 QK dims; base 1e7 global, 1e4 SWA)","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2Attention.__init__","quote":"self.rope_dim = int(self.head_dim * getattr(config, \"partial_rotary_factor\", 1.0))","note":"partial_rotary_factor 0.334, rope_theta 1e7, swa_rope_theta 1e4 in config."},"attn.rope_fraction":{"value":0.334,"source":"config","at":"/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_theta"},"attn.v_head_dim":{"value":128,"source":"config","at":"/v_head_dim"},"attn.scale":{"value":"values × 0.612 in every attention layer; QK scale 1/sqrt(192)","source":"config","at":"config.json; attention_value_scale","quote":"\"attention_value_scale\": 0.612","note":"modeling_mimo_v2.py MiMoV2Attention multiplies value_states by attention_value_scale before attention."},"attn.sinks":{"value":"learnable per-head sink bias in SWA layers; none in global layers","source":"config","at":"config.json; add_*_attention_sink_bias","quote":"\"add_full_attention_sink_bias\": false, \"add_swa_attention_sink_bias\": true","note":"modeling_mimo_v2.py concatenates attention_sink_bias (one logit per head) to the attention logits before softmax. The card calls it a learnable attention sink bias."},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.window":{"value":128,"source":"config","at":"/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MLP.forward","quote":"return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))","note":"act_fn = ACT2FN[/hidden_act] = silu; used for the dense FFN and every routed expert."},"ffn.bias":{"value":false,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"MiMoV2MLP is used for the dense FFN and every routed expert."},"ffn.dim":{"value":16384,"source":"config","at":"/intermediate_size","note":"Dense FFN in layer 0 only."},"moe.dense_layers":{"value":1,"source":"config","at":"/moe_layer_freq | count 0"},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":384,"source":"config","at":"/n_routed_experts"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func","note":"topk_method noaux_tc: MiMoV2MoEGate adds e_score_correction_bias to the scores for top-k selection only; how it was trained is not stated."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.shared":{"value":0,"source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2MoE.__init__","quote":"self.experts = nn.ModuleList( [MiMoV2MLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(config.n_routed_experts)] ) self.gate = MiMoV2MoEGate(config)","note":"MoE block holds routed experts and gate only; config n_shared_experts is null."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"mtp.ffn":{"value":"dense","source":"blog","at":"MiMo-V2.5-Pro blog; Architecture & training","quote":"A lightweight MTP module with dense FFNs is natively integrated for training and inference"},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.mtp.layers.2.eh_proj.weight\"","note":"Three MTP modules (model.mtp.layers.0–2, SWA + dense FFN) in model_mtp.safetensors. The blog says MTP is used in training; the number used in pretraining is not stated."},"norm.eps":{"value":1e-05,"source":"config","at":"/layernorm_epsilon"},"norm.placement":{"value":"pre","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.forward","quote":"residual = hidden_states hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_mimo_v2","at":"modeling_mimo_v2.py; MiMoV2DecoderLayer.__init__","quote":"self.input_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)"},"ctx.max":{"value":262144,"source":"card","at":"README.md; §2 Model Downloads","quote":"MiMo-V2.5-Pro-Base 1.02T 42B 256K","note":"Card value for the base checkpoint; config max_position_embeddings is 1,048,576 (the post-trained MiMo-V2.5-Pro's 1M)."},"ctx.pretrain":{"value":32768,"source":"card","at":"README.md; §1 Introduction","quote":"Trained on 27T tokens using FP8 mixed precision and native 32k seq length."},"ctx.extension":{"value":"extended from 32K up to 1M (stages not stated)","source":"blog","at":"MiMo-V2.5-Pro blog; Architecture & training","quote":"Pre-training runs on 27T tokens using FP8 mixed precision at a native 32K sequence length, with context extended up to 1M tokens.","note":"Covers the series; the Base checkpoint is listed at 256K, and the length it was trained to is not stated."},"train.precision":{"value":"FP8 mixed precision","source":"card","at":"README.md; §1 Introduction","quote":"Trained on 27T tokens using FP8 mixed precision and native 32k seq length."},"train.tokens":{"value":27000000000000,"source":"card","at":"README.md; §1 Introduction","quote":"Trained on 27T tokens using FP8 mixed precision and native 32k seq length."},"eval.aa_index":{"value":26,"source":"aa","at":"Artificial Analysis model page /models/mimo-v2-5-pro, Wayback snapshot 2026-10-19","quote":"MiMo-V2.5-Pro (Reasoning) scores 26 on the Artificial Analysis Intelligence Index","note":"Score of MiMo-V2.5-Pro, post-trained from this base. Variant: MiMo-V2.5-Pro (Reasoning). Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","mtp.depth","moe.balancing","ctx.final","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip"],"derived":{"params.counted_total":{"value":1021248126336,"formula":"embedding + output head + all experts + other weights, XiaomiMiMo/MiMo-V2.5-Pro-Base@1e94ccf safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1874853888,"formula":"input embedding + output head, XiaomiMiMo/MiMo-V2.5-Pro-Base@1e94ccf safetensors headers"},"params.counted_non_embedding":{"value":1019373272448,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, XiaomiMiMo/MiMo-V2.5-Pro-Base@1e94ccf safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":14765273600,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, XiaomiMiMo/MiMo-V2.5-Pro-Base@1e94ccf safetensors headers"},"params.counted_active_non_embedding":{"value":40020065664.0,"formula":"non-expert weights + routed experts × 8/384, excluding embedding and output head, XiaomiMiMo/MiMo-V2.5-Pro-Base@1e94ccf safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":87.77142857142857,"formula":"d_model / layers"},"attn.qk_width":{"value":24576,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.6666666666666665,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":6.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":48.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.041176470588235294,"formula":"params.active / params.total"},"scaling.tpp":{"value":26.470588235294116,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":674.661561694733,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":51200.0,"formula":"51200 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6750208000.0,"formula":"51200 B × 131072 (Global KV) + 307200 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":51500.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1226.1904761904761,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":160719238.0952381,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":51200.0,"windows":[[307200.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":51200.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":307200.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 10 × (2560 × 16/8) B/entry; SWA KV: 60 × (2560 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"minimax-m1","name":"MiniMax-M1","org":"MiniMax","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe MiniMax-M1-80k (MiniMax-M1-40k is an intermediate checkpoint of the same run).","sources":{"card":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/resolve/8d1494b1a260e22040d5b9b2eb332eb44500b34d/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/resolve/8d1494b1a260e22040d5b9b2eb332eb44500b34d/config.json","kind":"config"},"modeling_minimax_m1":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M1-80k/resolve/8d1494b1a260e22040d5b9b2eb332eb44500b34d/modeling_minimax_m1.py","kind":"code"},"m1-report":{"url":"https://arxiv.org/pdf/2506.13585v1","kind":"paper"},"text01-report":{"url":"https://arxiv.org/pdf/2501.08313v1","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019111044/https://artificialanalysis.ai/models/minimax-m1-80k","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-06-16","source":"m1-report","at":"p.1 arXiv stamp","quote":"arXiv:2506.13585v1 [cs.CL] 16 Jun 2025","note":"arXiv v1 date; MiniMax's M1 news page (minimax.io/news/minimaxm1) is dated the same day."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","basis":"inferred","source":"config","at":"config.json; architectures","quote":"\"architectures\": [ \"MiniMaxM1ForCausalLM\" ]","note":"No modality statement; causal-LM class with no vision or audio config, built on MiniMax-Text-01."},"params.total":{"value":456000000000,"source":"card","at":"README.md; 1. Model Overview","quote":"which contains a total of 456 billion parameters with 45.9 billion parameters activated"},"params.active":{"value":45900000000,"source":"card","at":"README.md; 1. Model Overview","quote":"which contains a total of 456 billion parameters with 45.9 billion parameters activated"},"params.note":{"value":"Counts are those of MiniMax-Text-01, on which M1 is built; embedding scope not stated.","source":"card","at":"README.md; 1. Model Overview","quote":"The model is developed based on our previous"},"shape.layers":{"value":80,"source":"config","at":"/num_hidden_layers"},"shape.d_model":{"value":6144,"source":"config","at":"/hidden_size"},"shape.vocab":{"value":200064,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.tokenizer":{"value":"byte-level BPE, 200K vocab (MiniMax-Text-01 tokenizer)","source":"text01-report","at":"p.22 §4.1.2","quote":"The resulting vocabulary size is set to 200K"},"attn.kind":{"value":"Lightning, GQA","source":"m1-report","at":"p.2 §1","quote":"a transformer block with softmax attention follows every seven transnormer blocks","note":"Transnormer blocks use Lightning attention (linear); the softmax blocks use GQA."},"attn.layout":{"value":"7 Lightning : 1 GQA","source":"config","at":"config.json; /attn_type_list","quote":"\"attn_type_list\": [ 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1 ]","note":"Repeats 10 times; layer 79 is the last softmax layer. /attn_type_list: 0 = Lightning, 1 = softmax/GQA (MiniMaxM1Model.__init__).","basis":"inferred","sequence":["Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","Lightning","GQA"]},"attn.global_layers":{"value":10,"source":"config","at":"/attn_type_list | count 1","note":"1 = softmax attention (MiniMaxM1FlashAttention2), 0 = Lightning attention (MiniMaxM1Model.__init__)."},"attn.linear_layers":{"value":70,"source":"config","at":"/attn_type_list | count 0"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.position":{"value":"partial RoPE (softmax layers); Lightning layers use a fixed per-head decay, no RoPE","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1Attention.__init__","quote":"self.rotary_dim = getattr(config, 'rotary_dim', self.head_dim)","note":"/rotary_dim 64 of /head_dim 128."},"attn.rope_fraction":{"value":0.5,"source":"text01-report","at":"p.4 §2","quote":"is applied to half of the attention","note":"Softmax layers only."},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_theta","note":"Base 10,000 in pretraining (p.4 §2), 5M at the 128K stage and 10M from the 512K stage (Table 6)."},"attn.qk_norm":{"value":false,"source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1Attention.forward","quote":"attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)"},"attn.bias":{"value":false,"source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1Attention.__init__","quote":"self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)","note":"Lightning qkv_proj, output_gate and out_proj are also bias-free."},"attn.gate":{"value":"sigmoid gate on the Lightning-layer output, after the output RMSNorm, before out_proj; none in softmax layers","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1LightningAttention.inference","quote":"output = F.sigmoid(self.output_gate(x)) * output"},"attn.linear_kind":{"value":"Lightning attention: SiLU on Q, K, V; per-head exponential decay (ALiBi-style slopes, scaled down linearly with depth); RMSNorm on output; 128×128 state per head","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1LightningAttention.inference","quote":"qkv = self.act(self.qkv_proj(x))","note":"act is /hidden_act silu; slopes from MiniMaxM1Model._build_slope_tensor times (1 - idx/(L-1) + 1e-5) in MiniMaxM1Model.forward."},"attn.linear_heads":{"value":64,"source":"config","at":"/num_attention_heads","note":"MiniMaxM1LightningAttention reads num_attention_heads and head_dim, as the softmax layers do."},"attn.linear_head_dim":{"value":128,"source":"config","at":"/head_dim"},"ffn.activation":{"value":"SwiGLU","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1BlockSparseTop2MLP.forward","quote":"current_hidden_states = self.act_fn(self.w1(hidden_states)) * self.w3(hidden_states)","note":"act_fn is SiLU (/hidden_act)."},"ffn.bias":{"value":false,"source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1BlockSparseTop2MLP.__init__","quote":"self.w1 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False)"},"moe.experts":{"value":32,"source":"config","at":"/num_local_experts"},"moe.top_k":{"value":2,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":0,"source":"config","at":"/shared_intermediate_size","note":"shared_intermediate_size 0 disables the shared MLP in MiniMaxM1DecoderLayer."},"moe.expert_dim":{"value":9216,"source":"config","at":"/intermediate_size"},"moe.dense_layers":{"value":0,"source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1DecoderLayer.__init__","quote":"self.block_sparse_moe = MiniMaxM1SparseMoeBlock(config)","note":"Every decoder layer is MoE."},"moe.router":{"value":"softmax","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1SparseMoeBlock.forward","quote":"routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)"},"moe.norm_topk":{"value":true,"source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1SparseMoeBlock.forward","quote":"routing_weights /= routing_weights.sum(dim=-1, keepdim=True)"},"moe.balancing":{"value":"GShard auxiliary loss (coefficient 0.01 for Text-01, lowered in M1 continued pretraining) + global router synchronizing token counts across EP groups","source":"text01-report","at":"p.24 §4.2","quote":"loss coefficient is set to 0.01.","note":"Global router: p.5 §2.1; M1 decrease: M1 report p.3 §2.1, value not given."},"norm.type":{"value":"RMSNorm","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1DecoderLayer.__init__","quote":"self.input_layernorm = MiniMaxM1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.placement":{"value":"post","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1DecoderLayer.forward","quote":"if self.postnorm: residual = hidden_states","note":"/postnorm true: the normed input also replaces the residual, so each block computes α·Norm(x) + F(Norm(x)) (DeepNorm)."},"norm.residual":{"value":"DeepNorm: residual × α = (2N)^0.25 = 3.5566 (N = 80), branch × β = 1","source":"modeling_minimax_m1","at":"modeling_minimax_m1.py; MiniMaxM1DecoderLayer.forward","quote":"hidden_states = residual * self.layernorm_attention_alpha","note":"/layernorm_full_attention_alpha, /layernorm_linear_attention_alpha, /layernorm_mlp_alpha 3.5565588200778455; /layernorm_full_attention_beta, /layernorm_linear_attention_beta, /layernorm_mlp_beta 1.0."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps","note":"The Lightning output RMSNorm uses the class default 1e-6."},"ctx.pretrain":{"value":8192,"source":"text01-report","at":"p.24 §4.2","quote":"The training sequence length is 8192"},"ctx.final":{"value":1000000,"source":"m1-report","at":"p.3 §2.1","quote":"ultimately extending the training context to 1M tokens"},"ctx.max":{"value":1000000,"source":"card","at":"README.md; 1. Model Overview","quote":"the M1 model natively supports a context length of 1 million tokens"},"ctx.extension":{"value":"Text-01: 8K → 128K (RoPE base 5M, 300B tokens) → 512K (10M, 32B) → 1M (10M, 26B); M1 continued pretraining: four stages from 32K to 1M","source":"m1-report","at":"p.3 §2.1","quote":"adapting a smoother extension of context length across four stages","note":"Text-01 stages: report p.25 Table 6."},"train.optimizer":{"value":"AdamW","source":"text01-report","at":"p.24 §4.2","quote":"We employ the AdamW optimizer"},"train.betas":{"value":"(0.9, 0.95)","source":"text01-report","at":"p.24 §4.2","quote":"β1 = 0.9, β2 = 0.95, and the weight decay is set to 0.1"},"train.weight_decay":{"value":0.1,"source":"text01-report","at":"p.24 §4.2","quote":"β1 = 0.9, β2 = 0.95, and the weight decay is set to 0.1"},"train.lr_peak":{"value":0.0002,"source":"text01-report","at":"p.24 §4.2","quote":"value of 2 × 10−4 , followed by training with a","note":"Text-01 initial pretraining; M1 continued pretraining used a constant 8e-5."},"train.lr_min":{"value":3e-05,"source":"text01-report","at":"p.24 §4.2","quote":"rate to 3 × 10−5","note":"End of the Text-01 initial-pretraining fast-decay phase (the stage of train.lr_peak); M1 continued pretraining then ran 8e-5 decaying to 8e-6."},"train.warmup":{"value":"500 iterations","source":"text01-report","at":"p.24 §4.2","quote":"linear warm-up over 500 iterations to a peak"},"train.schedule":{"value":"Text-01: 500-iter warmup, constant 2e-4 for 7.2T tokens, 1.3e-4 for the remaining 3.2T, exponential decay to 3e-5 over 1T; M1: constant 8e-5 for 2.5T, decay to 8e-6 over 5T","source":"text01-report","at":"p.24 §4.2","quote":"constant learning rate for 7.2T tokens","note":"The report does not say whether the 3.2T at 1.3e-4 is part of the 7.2T. M1 part: M1 report p.3 §2.1."},"train.batch":{"value":"16M → 32M (at 69B tokens) → 64M (790B) → 128M tokens (4.7T), then constant","source":"text01-report","at":"p.24 §4.2","quote":"the batch size is progressively scaled from an initial size of 16M to 32M at 69B tokens, to 64M at 790B tokens, and finally to 128M at 4.7T tokens","note":"Doubled when the loss reaches a fitted critical-batch-size threshold (Figure 13)."},"train.batch_tokens":{"value":128000000,"source":"text01-report","at":"p.24 §4.2","quote":"and finally to 128M at 4.7T tokens, where it remains until the end of training","note":"Text-01 initial pretraining after the 16M → 128M ramp; M1 continued-pretraining batch not stated."},"train.init":{"value":"Xavier; DeepNorm α = (2N)^0.25, β = (8N)^-0.25","source":"text01-report","at":"p.24 §4.2","quote":"We initialize all model parameters using the Xavier initialization method"},"train.stability":{"value":"LR cut from 2e-4 to 1.3e-4 after anomalous gradient norms; M1 context extension spread over four stages after gradient explosions with aggressive extension","source":"text01-report","at":"p.24 §4.2","quote":"lr to 1.3 × 10−4 for the remaining 3.2T","note":"Context-extension instability: M1 report p.3 §2.1."},"train.hardware":{"value":"1500–2500 H800 GPUs (cluster size varied)","source":"text01-report","at":"p.14 §3","quote":"the number of H800 GPUs ranges from 1500 to 2500"},"train.accelerators":{"value":"1,500–2,500 H800","source":"text01-report","at":"p.14 §3","quote":"the number of H800 GPUs ranges from 1500 to 2500","note":"MiniMax-Text-01 pretraining cluster (base of M1), size varied over the run."},"train.data":{"value":"Text-01: academic literature, books, web, code, scored by a 5B-active/60B MoE; high-quality data up to 4 epochs. M1: STEM, code, book and reasoning data raised to 70%, no synthetic data","source":"m1-report","at":"p.3 §2.1","quote":"and reasoning-related data to 70%","note":"Text-01 corpus: report p.21–24 §4.1."},"train.stages":{"value":[{"name":"Text-01 main","seq_len":8192,"lr":0.0002,"batch_tokens":128000000,"note":"Constant 2e-4 for 7.2T tokens, then 1.3e-4 for 3.2T; batch ramped 16M → 128M by 4.7T tokens.","quote":"constant learning rate for 7.2T tokens"},{"name":"Text-01 fast decay","tokens":1000000000000,"seq_len":8192,"batch_tokens":128000000,"note":"Exponential decay to 3e-5.","quote":"During the fast decay phase, we train 1T"},{"name":"Text-01 long-context 128K","tokens":300000000000,"seq_len":131072,"note":"RoPE base 5M; 30% short, 70% medium data.","quote":"128K 5M 300B"},{"name":"Text-01 long-context 512K","tokens":32000000000,"seq_len":524288,"note":"RoPE base 10M; 35% short, 35% medium, 30% long.","quote":"512K 10M 32B"},{"name":"Text-01 long-context 1M","tokens":26000000000,"seq_len":1000000,"note":"RoPE base 10M; 30% short, 30% medium, 40% long.","quote":"1M 10M 26B"},{"name":"M1 continued pretraining","tokens":7500000000000,"lr":8e-05,"note":"Constant 8e-5 for 2.5T, then decay to 8e-6 over 5T; context 32K → 1M in four stages; reasoning-heavy mix (70% STEM, code, book, reasoning).","quote":"we continue training the MiniMax-Text-01 model with additional 7.5T tokens","source":"m1-report"}],"source":"text01-report","at":"Text-01 report p.24–25 §4.2, Table 6; M1 report p.3 §2.1"},"scaling.ladder":{"value":"Text-01: softmax / lightning / hybrid scaling study at 70M–7B dense (≤300B tokens, 8K); size law fit on 44M–1.2B active MoE (16/32/64 experts, 500B tokens); MoE ablations at 28B-A5B (1T tokens) and 60B-A9.3B, 48 layers (500B tokens)","source":"text01-report","at":"Text-01 report p.8 §2.2.2.1, p.12–14 §2.3–2.4","quote":"We train models with activation parameters ranging from 44 million to 1.2 billion across 500 billion tokens, utilizing 16, 32, and 64 experts."},"eval.aa_index":{"value":12,"source":"aa","at":"Artificial Analysis model page /models/minimax-m1-80k, Wayback snapshot 2026-10-19","quote":"MiniMax M1 80k scores 12 on the Artificial Analysis Intelligence Index","note":"Variant: MiniMax M1 80k. Index v4.3.2."}},"not_disclosed":["mtp.depth","train.tokens","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":456089655296,"formula":"embedding + output head + all experts + other weights, MiniMaxAI/MiniMax-M1-80k@8d1494b safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":2458386432,"formula":"input embedding + output head, MiniMaxAI/MiniMax-M1-80k@8d1494b safetensors headers"},"params.counted_non_embedding":{"value":453631268864,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, MiniMaxAI/MiniMax-M1-80k@8d1494b safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":5687693312,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, MiniMaxAI/MiniMax-M1-80k@8d1494b safetensors headers"},"params.counted_active_non_embedding":{"value":45944920064.0,"formula":"non-expert weights + routed experts × 2/32, excluding embedding and output head, MiniMaxAI/MiniMax-M1-80k@8d1494b safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":76.8,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":1.5,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":7.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":16.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.1006578947368421,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":40960.0,"formula":"40960 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":5368709120.0,"formula":"40960 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":40960.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":892.3747276688454,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":116965340.3050109,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":40960.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":40960.0,"tokens_per_entry":1}],"formula":"Global KV: 10 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"minimax-m2-7","name":"MiniMax-M2.7","org":"MiniMax","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"config":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/d494266a4affc0d2995ba1fa35c8481cbd84294b/config.json","kind":"config"},"configuration_minimax_m2":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/d494266a4affc0d2995ba1fa35c8481cbd84294b/configuration_minimax_m2.py","kind":"code"},"modeling_minimax_m2":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/d494266a4affc0d2995ba1fa35c8481cbd84294b/modeling_minimax_m2.py","kind":"code"},"index":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/d494266a4affc0d2995ba1fa35c8481cbd84294b/model.safetensors.index.json","kind":"config"},"license":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/d494266a4affc0d2995ba1fa35c8481cbd84294b/LICENSE","kind":"model-card"},"report":{"url":"https://arxiv.org/pdf/2605.26494v2","kind":"paper"},"hf-api":{"url":"https://huggingface.co/api/models/MiniMaxAI/MiniMax-M2.7","kind":"api"},"aa":{"url":"https://web.archive.org/web/20261019103459/https://artificialanalysis.ai/models/minimax-m2-7","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-09","source":"hf-api","at":"/createdAt","quote":"2026-04-09T","note":"Hugging Face repository creation date (no dated publisher page for the weight release found); the 2026-03-18 blog announced the model without weights."},"meta.license":{"value":"other (NON-COMMERCIAL LICENSE)","source":"license","at":"LICENSE; title","quote":"NON-COMMERCIAL LICENSE Non-commercial use permitted based on MIT-style terms"},"meta.modality":{"value":"text","basis":"inferred","source":"config","at":"config.json; architectures","quote":"\"architectures\": [ \"MiniMaxM2ForCausalLM\" ]","note":"No modality statement; causal-LM class with no vision or audio config."},"params.active":{"value":9800000000,"source":"report","at":"p.3 §2.1","quote":"It contains 229.9B total parameters, with 9.8B activated per token."},"params.note":{"value":"Report figures for M2, the series flagship; the report attributes the M2 → M2.5 → M2.7 progression to post-training. MTP scope not stated.","source":"report","at":"p.2 §1","quote":"Pre-training on 29.2T tokens establishes the base; the bulk of"},"params.total":{"value":229900000000,"source":"report","at":"p.3 §2.1","quote":"It contains 229.9B total parameters, with 9.8B activated per token."},"shape.d_model":{"value":3072,"source":"config","at":"/hidden_size"},"shape.layers":{"value":62,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":200064,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":62,"source":"config","at":"/attn_type_list | count 1","note":"Every layer is full softmax attention (/sliding_window unset)."},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA","source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2Attention.__init__","quote":"self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False)"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.position":{"value":"partial RoPE (64 of 128 dims)","source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; apply_rotary_pos_emb","quote":"q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]"},"attn.qk_norm":{"value":false,"source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2Attention.__init__","quote":"self.q_norm = MiniMaxM2RMSNorm(self.head_dim * config.num_attention_heads, eps=config.rms_norm_eps)","note":"QK-norm exists (/use_qk_norm true) but is not per head: one RMSNorm spans all heads' concatenated Q (48×128) and another all K (8×128), before the per-head reshape (/qk_norm_type per_layer)."},"attn.rope_fraction":{"value":0.5,"source":"configuration_minimax_m2","at":"configuration_minimax_m2.py; MiniMaxM2Config.__init__","quote":"self.partial_rotary_factor = self.rotary_dim / self.head_dim","basis":"inferred","note":"/rotary_dim 64 ÷ /head_dim 128."},"attn.rope_theta":{"value":5000000,"source":"config","at":"/rope_theta"},"attn.bias":{"value":false,"source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2Attention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)"},"ffn.activation":{"value":"SwiGLU","source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2MLP.forward","quote":"current_hidden_states = self.act_fn(self.w1(hidden_states)) * self.w3(hidden_states)","note":"act_fn is SiLU (config hidden_act)."},"ffn.bias":{"value":false,"source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2MLP.__init__","quote":"self.w1 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False)"},"moe.balancing":{"value":"learnable per-expert routing bias trained jointly with the model + greatly reduced auxiliary loss (coefficient not stated)","source":"report","at":"p.3–4 §2.2.1","quote":"These biases are optimized jointly with model parameters and implicitly regulate expert","note":"Reduced auxiliary loss: same sentence, p.4. The bias enters top-k selection only (MiniMaxM2SparseMoeBlock e_score_correction_bias; config /use_routing_bias true)."},"moe.dense_layers":{"value":0,"source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2DecoderLayer.__init__","quote":"self.block_sparse_moe = MiniMaxM2SparseMoeBlock(config)","note":"Every decoder layer is MoE."},"moe.expert_dim":{"value":1536,"source":"config","at":"/intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/num_local_experts"},"moe.norm_topk":{"value":true,"source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2SparseMoeBlock.route_tokens_to_experts","quote":"top_k_weights /= top_k_weights.sum(dim=-1, keepdim=True)"},"moe.router":{"value":"sigmoid","source":"config","at":"/scoring_func"},"moe.shared":{"value":0,"source":"config","at":"config.json; shared_intermediate_size","quote":"\"shared_intermediate_size\": 0","note":"The modeling code has no shared-expert path."},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_minimax_m2","at":"modeling_minimax_m2.py; MiniMaxM2DecoderLayer.__init__","quote":"self.input_layernorm = MiniMaxM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.pretrain":{"value":8192,"source":"report","at":"p.6 §3","quote":"context window from 8K tokens through 32K and ultimately to 192K tokens.","note":"Report states 8K."},"ctx.final":{"value":196608,"source":"report","at":"p.6 §3","quote":"context window from 8K tokens through 32K and ultimately to 192K tokens.","note":"Report states 192K."},"ctx.extension":{"value":"8K → 32K → 192K in stages within the 9.3T-token decay phase","source":"report","at":"p.6 §3","quote":"The decay phase uses a total data budget of 9.3T tokens"},"ctx.max":{"value":204800,"source":"config","at":"/max_position_embeddings","note":"The M2.7 card states no context length."},"mtp.depth":{"value":3,"source":"report","at":"p.5 §2.3","quote":"we expand from one to three MTP modules ( K = 3) during the decay phase of continued pre-training.","note":"One MTP module (K=1) during the constant phase; three in the decay phase, matching config /num_mtp_modules 3."},"mtp.modules":{"value":3,"source":"report","at":"p.5 §2.3","quote":"we expand from one to three MTP modules ( K = 3) during the decay phase of continued pre-training.","note":"Added modules are copy-initialized, then trained jointly with the main model; config /num_mtp_modules 3."},"mtp.block":{"value":"DeepSeek-V3 design: RMSNorm on previous hidden state and next-token embedding, concat, linear projection, 1 transformer block","source":"report","at":"p.7 Figure 2","quote":"Figure 2 | Multi-Token Prediction (MTP) module architecture used in M2.","note":"One transformer layer per module per config /mtp_transformer_layers; whether its FFN is MoE is not stated."},"mtp.shares_embedding":{"value":true,"source":"report","at":"p.7 Figure 2","quote":"Figure 2 | Multi-Token Prediction (MTP) module architecture used in M2.","note":"Figure 2 marks the embedding layer and output head as shared with the main model."},"mtp.loss_weight":{"value":"0.3, annealed to 0.1 in the decay phase","source":"report","at":"p.5 §2.3","quote":"with an initial MTP loss weight of 0.3, which is annealed to 0.1 during the decay phase."},"mtp.released":{"value":false,"source":"index","at":"/weight_map | has mtp","note":"config.json declares num_mtp_modules 3; the checkpoint contains no MTP tensors."},"train.tokens":{"value":29200000000000,"source":"report","at":"p.3 §2.1","quote":"is pre-trained on 29.2T tokens with a maximum context length of 192K.","note":"19.9T constant phase + 9.3T decay phase; stated for M2."},"train.schedule":{"value":"constant phase (19.9T tokens), then decay phase (9.3T tokens, with context extension)","source":"report","at":"p.6 §3","quote":"During the constant phase of pre-training, we train on a total of 19.9T tokens."},"train.stages":{"value":[{"name":"constant","tokens":19900000000000,"seq_len":8192,"quote":"During the constant phase of pre-training, we train on a total of 19.9T tokens."},{"name":"decay","tokens":9300000000000,"note":"short-text decay data plus long-context data; context 8K → 32K → 192K; MTP expanded to 3 modules","quote":"The decay phase uses a total data budget of 9.3T tokens"}],"source":"report","at":"p.6 §3"},"train.data":{"value":"web, academic, books, code, structured QA; code, math and STEM upsampled","source":"report","at":"p.6 §3","quote":"with code, mathematics, and STEM content significantly upsampled relative to their natural distribution."},"scaling.ladder":{"value":"MoE fine-grained-expert and MTP ablations at 2B active / 17.8B total, 500B tokens","source":"report","at":"p.4 §2.2.1, Table 1","quote":"Total Params – 17.8B 17.8B 17.8B Training Tokens – 500B 500B 500B"},"eval.aa_index":{"value":23,"source":"aa","at":"Artificial Analysis model page /models/minimax-m2-7, Wayback snapshot 2026-10-19","quote":"MiniMax-M2.7 scores 23 on the Artificial Analysis Intelligence Index","note":"Variant: MiniMax-M2.7. Index v4.3.2."}},"not_disclosed":["mtp.ffn","train.optimizer","train.lr_peak","train.lr_min","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":228689764864,"formula":"embedding + output head + all experts + other weights, MiniMaxAI/MiniMax-M2.7@d494266 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":1229193216,"formula":"input embedding + output head, MiniMaxAI/MiniMax-M2.7@d494266 safetensors headers"},"params.counted_non_embedding":{"value":227460571648,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, MiniMaxAI/MiniMax-M2.7@d494266 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":3668718848,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, MiniMaxAI/MiniMax-M2.7@d494266 safetensors headers"},"params.counted_active_non_embedding":{"value":9801359872.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, MiniMaxAI/MiniMax-M2.7@d494266 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":49.54838709677419,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.042627229230100046,"formula":"params.active / params.total"},"scaling.tpp":{"value":127.01174423662462,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":2979.178438638601,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":253952.0,"formula":"253952 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":33285996544.0,"formula":"253952 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":253952.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":25913.4693877551,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":3396530259.5918365,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":253952.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":253952.0,"tokens_per_entry":1}],"formula":"Global KV: 62 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"minimax-m3","name":"MiniMax-M3","org":"MiniMax","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M3/resolve/f0e1c1e04d40177e4673a22097036854f536e9c0/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M3/resolve/f0e1c1e04d40177e4673a22097036854f536e9c0/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/minimax_m3_vl/modeling_minimax_m3_vl.py","kind":"code"},"index":{"url":"https://huggingface.co/MiniMaxAI/MiniMax-M3/resolve/f0e1c1e04d40177e4673a22097036854f536e9c0/model.safetensors.index.json","kind":"config"},"msa-report":{"url":"https://arxiv.org/pdf/2606.13392v1","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019092918/https://artificialanalysis.ai/models/minimax-m3","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-06-11","source":"msa-report","at":"p.1 abstract; arXiv stamp","quote":"arXiv:2606.13392v1 [cs.AI] 11 Jun 2026","note":"Technical report linked from the model card; its abstract says M3 has been publicly released on Hugging Face. The Hugging Face repository was created 2026-06-02, but its commit history starts 2026-06-12."},"meta.license":{"value":"other (minimax-community)","source":"card","at":"README.md; YAML front matter","quote":"license_name: minimax-community"},"meta.modality":{"value":"text + image + video","source":"card","at":"README.md; Highlights","quote":"enabling deeper semantic fusion across text, image, and video"},"params.active":{"value":23000000000,"source":"card","at":"README.md; intro","quote":"It has ~428B parameters and ~23B activated parameters."},"params.note":{"value":"Approximate (~) publisher figures for a native multimodal model; whether the vision encoder is included is not stated.","source":"card","at":"README.md; intro","quote":"MiniMax-M3 is a native multimodal model with 1M context. It has ~428B parameters and ~23B activated parameters."},"params.total":{"value":428000000000,"source":"card","at":"README.md; intro","quote":"It has ~428B parameters and ~23B activated parameters."},"shape.d_model":{"value":6144,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":60,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":200064,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLAttention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)"},"attn.global_layers":{"value":60,"source":"config","at":"/text_config/num_hidden_layers","note":"3 dense GQA + 57 MSA layers; MSA is block-sparse global attention."},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"GQA, MSA","source":"card","at":"README.md; MiniMax Sparse Attention (MSA)","quote":"M3 introduces MiniMax Sparse Attention (MSA) to improve long context efficiency."},"attn.kv_heads":{"value":4,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 GQA + 57 MSA","source":"config","at":"config.json; /text_config/sparse_attention_config/sparse_attention_freq","quote":"\"sparse_attention_freq\": [ 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]","note":"sparse_attention_freq has 3 zeros (layers 0–2) and 57 ones. sparse_attention_freq selects MSA at 1 and dense GQA at 0; backbone only.","basis":"inferred","sequence":["GQA","GQA","GQA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA","MSA"]},"attn.position":{"value":"partial RoPE (64 of 128 dims)","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLAttention docstring","quote":"M3 attention: per-head Gemma QK-norm + partial RoPE","note":"/text_config/rotary_dim 64 of /text_config/head_dim 128."},"attn.qk_norm":{"value":true,"source":"config","at":"/text_config/use_qk_norm","note":"Per head (/text_config/qk_norm_type per_head; MiniMaxM3VLAttention q_norm/k_norm over head_dim)."},"attn.rope_fraction":{"value":0.5,"source":"config","at":"/text_config/partial_rotary_factor"},"attn.rope_theta":{"value":5000000,"source":"config","at":"/text_config/rope_theta"},"attn.sparse":{"value":"MSA: indexer picks 16 key blocks of 128 tokens per query and KV group, the query's own block always among them; 4 index heads × 128 dims","source":"config","at":"config.json text_config.sparse_attention_config","quote":"\"sparse_topk_blocks\": 16, \"sparse_block_size\": 128,","note":"Max-pooled block scores (sparse_score_type max); local-block boost and per-KV-group selection in MiniMaxM3VLIndexer.forward (sparse_local_block 1 forces block position_id // 128); index-value path disabled on all sparse layers (sparse_disable_index_value); use_sparse_attention true."},"attn.indexer_dim":{"value":128,"source":"config","at":"/text_config/sparse_attention_config/sparse_index_dim","note":"One index key head shared across KV groups (MiniMaxM3VLIndexer.k_proj outputs index_head_dim); 4 index query heads (sparse_num_index_heads); cached only in the 57 MSA layers."},"ffn.activation":{"value":"SwiGLU (swigluoai: gate·σ(1.702·gate)·(up + 1))","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLDenseMLP.forward","quote":"glu = gate * torch.sigmoid(gate * self.swiglu_alpha)","note":"Config hidden_act = swigluoai, swiglu_alpha = 1.702; output (up + 1.0) * glu; inputs clamped (see ffn.clamp)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLDenseMLP.__init__","quote":"self.gate_up_proj = nn.Linear(config.hidden_size, 2 * inter, bias=False)"},"ffn.clamp":{"value":"gate clamped at max 7, up clamped to [−7, 7] before the GLU; all dense, shared and routed FFNs","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLDenseMLP.forward, MiniMaxM3VLExperts._apply_gate","quote":"up = up.clamp(min=-self.swiglu_limit, max=self.swiglu_limit)","note":"/text_config/swiglu_limit = 7.0."},"ffn.dim":{"value":12288,"source":"config","at":"/text_config/dense_intermediate_size"},"moe.balancing":{"value":"score-correction bias added to sigmoid scores for top-k selection only","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLTopKRouter.forward","quote":"scores_for_choice = routing_weights + self.e_score_correction_bias","note":"Inference routing code (config /text_config/use_routing_bias true); the training-time bias update and any auxiliary loss are not disclosed."},"moe.dense_layers":{"value":3,"source":"config","at":"/text_config/moe_layer_freq | count 0"},"moe.expert_dim":{"value":3072,"source":"config","at":"/text_config/intermediate_size"},"moe.experts":{"value":128,"source":"config","at":"/text_config/num_local_experts"},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLTopKRouter.forward","quote":"top_k_weights /= top_k_weights.sum(dim=-1, keepdim=True)"},"moe.routed_scale":{"value":2.0,"source":"config","at":"/text_config/routed_scaling_factor","note":"Multiplies the routed-expert output before the shared expert is added (MiniMaxM3VLSparseMoeBlock.forward)."},"moe.router":{"value":"sigmoid","source":"config","at":"/text_config/scoring_func"},"moe.shared":{"value":1,"source":"config","at":"/text_config/n_shared_experts"},"moe.shared_dim":{"value":3072,"source":"config","at":"/text_config/shared_intermediate_size"},"moe.top_k":{"value":4,"source":"config","at":"/text_config/num_experts_per_tok"},"mtp.released":{"value":false,"source":"index","at":"/weight_map | has mtp.","note":"Config declares num_mtp_modules=7 and num_nextn_predict_layers=1; the checkpoint has no mtp.* tensors."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_minimax_m3_vl.py; MiniMaxM3VLRMSNorm","quote":"Gemma-style RMSNorm: normalizes in fp32 and scales by `weight + 1`.","note":"Config /text_config/use_gemma_norm true."},"ctx.max":{"value":1048576,"source":"config","at":"/text_config/max_position_embeddings","note":"Card: \"native multimodal model with 1M context\"."},"train.objective":{"value":"native multimodal: mixed text, image and video training from the first step","source":"card","at":"README.md; Highlights","quote":"M3 undergoes mixed-modality training from the very first step, enabling deeper semantic fusion across text, image, and video."},"scaling.ladder":{"value":"MSA ablations: 109B-A6B MoE (41 layers, d_model 3072) at 3T tokens; 10.53B-A1.47B pilot (16 layers, d_model 2048)","source":"msa-report","at":"MSA report p.8 §5.1; App. B.1","quote":"All models use the same 41-layer MoE backbone, with approximately 109B total parameters and 6B activated parameters per token."},"eval.aa_index":{"value":29,"source":"aa","at":"Artificial Analysis model page /models/minimax-m3, Wayback snapshot 2026-10-19","quote":"MiniMax-M3 scores 29 on the Artificial Analysis Intelligence Index","note":"Variant: MiniMax-M3. Index v4.3.2."}},"not_disclosed":["mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":426363328896,"formula":"embedding + output head + all experts + other weights, MiniMaxAI/MiniMax-M3@f0e1c1e safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"params.counted_embedding":{"value":2458386432,"formula":"input embedding + output head, MiniMaxAI/MiniMax-M3@f0e1c1e safetensors headers"},"params.counted_non_embedding":{"value":423904942464,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, MiniMaxAI/MiniMax-M3@f0e1c1e safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"params.counted_max_layer":{"value":7416066688,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, MiniMaxAI/MiniMax-M3@f0e1c1e safetensors headers"},"params.counted_active_non_embedding":{"value":23692843392.0,"formula":"non-expert weights + routed experts × 4/128, excluding embedding and output head, MiniMaxAI/MiniMax-M3@f0e1c1e safetensors headers; reproduces stated total (other modalities) and active (non-embedding)"},"shape.aspect":{"value":102.4,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.053738317757009345,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":138240.0,"formula":"138240 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":18119393280.0,"formula":"138240 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":138240.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":6010.434782608696,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":787799707.826087,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":138240.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":138240.0,"tokens_per_entry":1}],"formula":"Global KV: 60 × (1152 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mistral-large-3","name":"Mistral-Large-3-675B-Base-2512","org":"Mistral AI","country":"France","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/mistralai/Mistral-Large-3-675B-Base-2512/resolve/3123f82420d99a7fc9313b2d69f9a0ab6cb6ab4d/README.md","kind":"model-card"},"params":{"url":"https://huggingface.co/mistralai/Mistral-Large-3-675B-Base-2512/resolve/3123f82420d99a7fc9313b2d69f9a0ab6cb6ab4d/params.json","kind":"config"},"index":{"url":"https://huggingface.co/mistralai/Mistral-Large-3-675B-Base-2512/resolve/3123f82420d99a7fc9313b2d69f9a0ab6cb6ab4d/consolidated.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20261011202338/https://mistral.ai/news/mistral-3/","kind":"blog"},"data_summary":{"url":"https://web.archive.org/web/20261019100726id_/https://legal.mistral.ai/documents/Mistral%20Large%203%20-%20Public%20Summary%20of%20Training%20Content.docx.pdf","kind":"paper"},"vllm_mistral_large_3":{"url":"https://raw.githubusercontent.com/vllm-project/vllm/d8c6210eeaa7f3b474e50cf74926f77a8dc79adf/vllm/model_executor/models/mistral_large_3.py","kind":"code"},"vllm_mistral_config":{"url":"https://raw.githubusercontent.com/vllm-project/vllm/d8c6210eeaa7f3b474e50cf74926f77a8dc79adf/vllm/transformers_utils/configs/mistral.py","kind":"code"},"vllm_deepseek_v2":{"url":"https://raw.githubusercontent.com/vllm-project/vllm/d8c6210eeaa7f3b474e50cf74926f77a8dc79adf/vllm/model_executor/models/deepseek_v2.py","kind":"code"},"vllm_mla":{"url":"https://raw.githubusercontent.com/vllm-project/vllm/d8c6210eeaa7f3b474e50cf74926f77a8dc79adf/vllm/model_executor/layers/mla.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019103721/https://artificialanalysis.ai/models/mistral-large-3","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-12-02","source":"blog","at":"Introducing Mistral 3; header","quote":"Introducing Mistral 3 December 2, 2025","note":"Same post: \"We release both the base and instruction fine-tuned versions of Mistral Large 3\"."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Key Features","quote":"Enables the model to analyze images and provide insights based on visual content, in addition to text."},"params.active":{"value":41000000000,"source":"card","at":"README.md; intro","quote":"with 41B active parameters and 675B total parameters"},"params.note":{"value":"Headline counts include the 2.5B vision encoder; the language model alone is 673B total, 39B active.","source":"card","at":"README.md; Key Features","quote":"A Granular MoE Language Model with 673B params and 39B active"},"params.total":{"value":675000000000,"source":"card","at":"README.md; intro","quote":"with 41B active parameters and 675B total parameters"},"shape.d_model":{"value":7168,"source":"params","at":"/dim"},"shape.layers":{"value":61,"source":"params","at":"/n_layers"},"shape.tied_embeddings":{"value":false,"source":"params","at":"/tied_embeddings"},"shape.vocab":{"value":131072,"source":"params","at":"/vocab_size"},"attn.global_layers":{"value":61,"source":"params","at":"/n_layers","note":"Every block has the same MLA attention tensors; params.json defines no sliding window."},"attn.head_dim":{"value":192,"source":"params","at":"/qk_nope_head_dim + /qk_rope_head_dim","note":"params.json /head_dim is also 192."},"attn.position":{"value":"partial RoPE (MLA decoupled, 64 of 192 QK dims)","source":"vllm_mla","at":"vllm layers/mla.py; MultiHeadLatentAttentionWrapper.forward_native","quote":"q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(","note":"Rotary applied to the qk_rope_head_dim (64) slice only; the 128 nope dims are unrotated."},"attn.qk_norm":{"value":false,"source":"vllm_mla","at":"vllm layers/mla.py; MultiHeadLatentAttentionWrapper.forward_native","quote":"kv_c_normed = self.kv_a_layernorm(kv_c)","note":"RMSNorm only on the Q and KV latents (q_a_norm, kv_a_norm); per-head Q/K are not normalized."},"attn.scale":{"value":"queries × (1 + 0.1·ln(1 + ⌊pos/8192⌋)) (Llama-4 style)","source":"vllm_deepseek_v2","at":"vllm models/deepseek_v2.py; _get_llama_4_scaling","quote":"scaling = 1 + scaling_beta * torch.log( 1 + torch.floor(positions / original_max_position_embeddings)","note":"beta and length from params.json /llama_4_scaling (0.1, 8,192); added for Mistral Large 3 in the same PR."},"attn.bias":{"value":false,"source":"index","at":"/weight_map | has bias","note":"No bias tensor anywhere in the checkpoint."},"attn.heads":{"value":128,"source":"params","at":"/n_heads"},"attn.kind":{"value":"MLA","source":"vllm_mistral_large_3","at":"vllm models/mistral_large_3.py; MistralLarge3ForCausalLM","quote":"class MistralLarge3ForCausalLM(DeepseekV3ForCausalLM):","note":"Checkpoint attention.wkv_a_with_mqa is remapped to DeepSeek-V3 kv_a_proj_with_mqa (512 latent + 64 RoPE key)."},"attn.kv_lora_rank":{"value":512,"source":"params","at":"/kv_lora_rank"},"attn.q_lora_rank":{"value":1536,"source":"params","at":"/q_lora_rank"},"attn.rope_dim":{"value":64,"source":"params","at":"/qk_rope_head_dim"},"attn.rope_theta":{"value":10000,"source":"params","at":"/rope_theta"},"attn.v_head_dim":{"value":128,"source":"params","at":"/v_head_dim"},"ffn.activation":{"value":"SwiGLU","source":"vllm_deepseek_v2","at":"vllm models/deepseek_v2.py; DeepseekV2MLP.__init__","quote":"Only silu is supported for now.","note":"SiluAndMul over gate (w1) and up (w3) in dense and expert FFNs; params.json sets no activation and the Mistral adapter defaults it to silu."},"ffn.dim":{"value":16384,"source":"params","at":"/hidden_dim","note":"Used by the 3 leading dense blocks; checkpoint layers.0.feed_forward.w1 is [16384, 7168]."},"moe.dense_layers":{"value":3,"source":"params","at":"/moe/first_k_dense_replace","note":"/moe/route_every_n 1 (moe_layer_freq): every later block is MoE."},"moe.router":{"value":"softmax","source":"vllm_mistral_config","at":"vllm transformers_utils/configs/mistral.py; _remap_moe_args","quote":"config[\"scoring_func\"] = \"softmax\"","note":"Same function sets topk_method None, so no score-correction bias."},"moe.norm_topk":{"value":true,"source":"vllm_mistral_config","at":"vllm transformers_utils/configs/mistral.py; _remap_moe_args","quote":"config[\"norm_topk_prob\"] = True"},"moe.expert_dim":{"value":4096,"source":"params","at":"/moe/expert_hidden_dim"},"moe.experts":{"value":128,"source":"params","at":"/moe/num_experts"},"moe.routed_scale":{"value":1.0,"source":"params","at":"/moe/routed_scale"},"moe.shared":{"value":1,"source":"params","at":"/moe/num_shared_experts"},"moe.shared_dim":{"value":4096,"source":"params","at":"/moe/expert_hidden_dim","basis":"inferred","note":"One shared expert of expert width; checkpoint layers.3.shared_experts.w1 is [4096, 7168]."},"moe.top_k":{"value":4,"source":"params","at":"/moe/num_experts_per_tok","note":"/moe/num_expert_groups and /moe/num_expert_groups_per_tok are 1: no group-limited routing."},"norm.eps":{"value":1e-06,"source":"params","at":"/norm_eps"},"norm.type":{"value":"RMSNorm","source":"vllm_deepseek_v2","at":"vllm models/deepseek_v2.py; DeepseekV2DecoderLayer.__init__","quote":"self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"vllm_deepseek_v2","at":"vllm models/deepseek_v2.py; DeepseekV2DecoderLayer.forward","quote":"hidden_states, residual = self.input_layernorm(hidden_states, residual)","note":"Checkpoint attention_norm and ffn_norm map to input_layernorm and post_attention_layernorm, applied before attention and FFN."},"ctx.extension":{"value":"YaRN factor 36 over original 8,192 (release params.json)","source":"params","at":"params.json; yarn","quote":"\"yarn\": { \"original_max_position_embeddings\": 8192, \"factor\": 36,","note":"Training-time context stages are not disclosed. Same dict: beta 32, alpha 1, apply_scale false; params.json also sets llama_4_scaling (beta 0.1, original 8,192)."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Key Features","quote":"Supports a 256k context window.","note":"params.json /max_seq_len is 262144."},"train.hardware":{"value":"3000 H200 GPUs","source":"card","at":"README.md; intro","quote":"trained from scratch with 3000 H200s"},"train.accelerators":{"value":"3,000 H200","source":"card","at":"README.md; intro","quote":"trained from scratch with 3000 H200s"},"train.cutoff":{"value":"July 2025","source":"data_summary","at":"§1.3 Latest date of data acquisition/collection","quote":"The latest date of data collection was July 2025."},"train.data":{"value":"multilingual text and images: Common Crawl and other public datasets, licensed and third-party datasets, synthetic data, user interaction data (subject to opt-out)","source":"data_summary","at":"§2.1 Publicly available datasets","quote":"The datasets used to train the model include Common Crawl.","note":"§1.3 ticks \"More than 10 trillions tokens\" for the text training data; §2.4-2.5 cover user and synthetic data."},"eval.aa_index":{"value":9,"source":"aa","at":"Artificial Analysis model page /models/mistral-large-3, Wayback snapshot 2026-10-19","quote":"Mistral Large 3 scores 9 on the Artificial Analysis Intelligence Index","note":"Score of Mistral-Large-3-675B-Instruct-2512, post-trained from this base. Variant: Mistral Large 3. Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":24576,"formula":"heads × head_dim"},"ffn.mult":{"value":2.2857142857142856,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.5714285714285714,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.06074074074074074,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":70272.0,"formula":"70272 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9210691584.0,"formula":"70272 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":70272.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1713.9512195121952,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":224651014.24390244,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":70272.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":70272.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mistral-medium-3-5","name":"Mistral-Medium-3.5-128B","org":"Mistral AI","country":"France","kind":"reasoning","weights":"open","note":"No base checkpoint is public; the card calls the release \"our first flagship merged model\", handling instruction-following, reasoning, and coding in a single set of weights.","sources":{"card":{"url":"https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/resolve/22b2b868a15677cfa6061277ed2f653d1349a9ab/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/resolve/22b2b868a15677cfa6061277ed2f653d1349a9ab/config.json","kind":"config"},"modeling_ministral3":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/ministral3/modeling_ministral3.py","kind":"code"},"governance":{"url":"https://web.archive.org/web/20261007202143/https://legal.mistral.ai/ai-governance/models/mistral-medium-3/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019103907/https://artificialanalysis.ai/models/mistral-medium-3-5","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-04-29","source":"governance","at":"AI Governance; Mistral Medium 3.0 family; Model Lifecycle table","quote":"Mistral Medium 3.5 model version general purpose ai model April 29, 2026"},"meta.license":{"value":"other (Modified MIT License)","source":"card","at":"README.md; License","quote":"This model is licensed under a [Modified MIT License](https://huggingface.co/mistralai/Mistral-Medium-3.5-128B/blob/main/LICENSE).","note":"Card front matter: license: other."},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Key Features","quote":"Multimodal input: Accepts both text and image input, with text output."},"params.total":{"value":128000000000,"source":"card","at":"README.md; Key Features","quote":"Dense 128B parameters"},"params.active":{"value":128000000000,"source":"card","at":"README.md; Key Features","quote":"Dense 128B parameters","basis":"inferred","note":"Dense model: every parameter is active."},"shape.d_model":{"value":12288,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":88,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":131072,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3Attention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)"},"attn.global_layers":{"value":88,"source":"config","at":"/text_config/num_hidden_layers","note":"/text_config/sliding_window is null, so every block uses full causal attention."},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":96,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"GQA","source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3Attention.__init__","quote":"self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False)","basis":"inferred","note":"96 query heads share 8 KV heads."},"attn.kv_heads":{"value":8,"source":"config","at":"/text_config/num_key_value_heads"},"attn.position":{"value":"RoPE","source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3Attention.forward","quote":"query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)"},"attn.qk_norm":{"value":false,"source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3Attention.forward","quote":"query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)","note":"Q and K go from the projections straight to RoPE; no per-head norm."},"attn.rope_theta":{"value":1000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"ffn.activation":{"value":"SwiGLU","source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"Config /text_config/hidden_act is silu."},"ffn.bias":{"value":false,"source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.dim":{"value":28672,"source":"config","at":"/text_config/intermediate_size"},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_ministral3","at":"modeling_ministral3.py; Ministral3DecoderLayer.__init__","quote":"self.input_layernorm = Ministral3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.extension":{"value":"YaRN factor 64 over original 4,096 (release config)","source":"config","at":"config.json; text_config.rope_parameters","quote":"\"factor\": 64.0, \"llama_4_scaling_beta\": 0, \"mscale\": 1.0, \"mscale_all_dim\": 0.0, \"original_max_position_embeddings\": 4096,","note":"Training-time context stages are not disclosed; mscale_all_dim was corrected from 1.0 in commit c4be198. Same dict: rope_type/type yarn, beta_fast 4, beta_slow 1; llama_4_scaling_beta 0 disables the query length scaling."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Key Features","quote":"256k context length"},"eval.aa_index":{"value":14,"source":"aa","at":"Artificial Analysis model page /models/mistral-medium-3-5, Wayback snapshot 2026-10-19","quote":"Mistral Medium 3.5 scores 14 on the Artificial Analysis Intelligence Index","note":"Variant: Mistral Medium 3.5. Index v4.3.2."}},"not_disclosed":["mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":125025988608,"formula":"embedding + output head + all experts + other weights, mistralai/Mistral-Medium-3.5-128B@22b2b86 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"params.counted_embedding":{"value":3221225472,"formula":"input embedding + output head, mistralai/Mistral-Medium-3.5-128B@22b2b86 safetensors headers"},"params.counted_non_embedding":{"value":121804763136,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, mistralai/Mistral-Medium-3.5-128B@22b2b86 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"params.counted_max_layer":{"value":1384144896,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, mistralai/Mistral-Medium-3.5-128B@22b2b86 safetensors headers"},"params.counted_active_non_embedding":{"value":121804763136.0,"formula":"all weights except embedding and output head, mistralai/Mistral-Medium-3.5-128B@22b2b86 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"shape.aspect":{"value":139.63636363636363,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"attn.group_size":{"value":12.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.3333333333333335,"formula":"ffn.dim / d_model"},"kv.growth_per_token":{"value":360448.0,"formula":"360448 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":47244640256.0,"formula":"360448 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":360448.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2816.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":369098752.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":360448.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":360448.0,"tokens_per_entry":1}],"formula":"Global KV: 88 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"mistral-small-4","name":"Mistral-Small-4-119B-2603","org":"Mistral AI","country":"France","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/resolve/a11f36bebf709121056b1dbcc943d1c6afbe494d/config.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20261006190028/https://mistral.ai/news/mistral-small-4/","kind":"blog"},"modeling_mistral4":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/mistral4/modeling_mistral4.py","kind":"code"},"data_summary":{"url":"https://web.archive.org/web/20261011054034id_/https://legal.mistral.ai/documents/Mistral%20Small%204%20-%20Public%20Summary%20of%20Training%20Content.docx.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019104103/https://artificialanalysis.ai/models/mistral-small-4","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-03-16","source":"blog","at":"Introducing Mistral Small 4; header","quote":"Introducing Mistral Small 4 March 16, 2026","note":"Same post: \"Mistral Small 4 is released under the Apache 2.0 license\"."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image","source":"card","at":"README.md; Key Features","quote":"Multimodal input: Accepts both text and image input, with text output."},"params.active":{"value":6500000000,"source":"card","at":"README.md; Key Features","quote":"119B parameters, with 6.5B activated per token.","note":"The blog says 6B active (8B including embedding and output layers); the card title says A6B."},"params.active_non_embedding":{"value":6000000000,"source":"blog","at":"Key architectural details","quote":"119B total parameters, with 6B active parameters per token (8B including embedding and output layers)."},"params.note":{"value":"Card's 6.5B active has no stated scope; the blog gives 6B without and 8B with embedding and output layers.","source":"blog","at":"Key architectural details","quote":"119B total parameters, with 6B active parameters per token (8B including embedding and output layers)."},"params.total":{"value":119000000000,"source":"card","at":"README.md; Key Features","quote":"119B parameters, with 6.5B activated per token."},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":36,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":131072,"source":"config","at":"/text_config/vocab_size"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.global_layers":{"value":36,"source":"config","at":"/text_config/num_hidden_layers","note":"Every block uses Mistral4Attention (MLA); /text_config/sliding_window is null."},"attn.head_dim":{"value":128,"source":"config","at":"/text_config/qk_nope_head_dim + /text_config/qk_rope_head_dim","note":"configuration_mistral4.py recomputes head_dim and qk_head_dim (both 128 in config) as this sum."},"attn.v_head_dim":{"value":128,"source":"config","at":"/text_config/v_head_dim"},"attn.heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"MLA","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4Attention docstring","quote":"Multi-headed Latent Attention (MLA) from Deepseek V2, with support for rope interleave."},"attn.kv_lora_rank":{"value":256,"source":"config","at":"/text_config/kv_lora_rank"},"attn.position":{"value":"partial RoPE (MLA decoupled, interleaved, 64 of 128 QK dims)","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4Attention.forward","quote":"q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)","note":"Config /text_config/rope_interleave is true."},"attn.q_lora_rank":{"value":1024,"source":"config","at":"/text_config/q_lora_rank"},"attn.qk_norm":{"value":false,"source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4Attention.forward","quote":"q_states = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states)))","note":"RMSNorm is applied only to the Q and KV latents; per-head Q/K are not normalized."},"attn.rope_dim":{"value":64,"source":"config","at":"/text_config/qk_rope_head_dim"},"attn.rope_theta":{"value":10000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.scale":{"value":"queries × (1 + 0.1·ln(1 + ⌊pos/8192⌋)) (Llama-4 style)","source":"modeling_mistral4","at":"modeling_mistral4.py; get_llama_4_attn_scale","quote":"scaling = 1 + beta * torch.log(1 + torch.floor(positions_ids / max_position_embeddings))","note":"beta = /text_config/rope_parameters/llama_4_scaling_beta (0.1), length = original_max_position_embeddings (8192)."},"ffn.activation":{"value":"SwiGLU","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4Experts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"Config /text_config/hidden_act is silu."},"ffn.bias":{"value":false,"source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.dense_layers":{"value":0,"source":"config","at":"/text_config/first_k_dense_replace","note":"All 36 blocks are MoE; /text_config/intermediate_size (12288) is unused."},"moe.expert_dim":{"value":2048,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":128,"source":"config","at":"/text_config/n_routed_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_topk_prob"},"moe.routed_scale":{"value":1.0,"source":"config","at":"/text_config/routed_scaling_factor"},"moe.router":{"value":"softmax","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4TopkRouter.forward","quote":"scores = router_logits.softmax(-1)"},"moe.shared":{"value":1,"source":"config","at":"/text_config/n_shared_experts"},"moe.shared_dim":{"value":2048,"source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4MoE.__init__","quote":"config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts","basis":"inferred","note":"moe_intermediate_size 2048 × n_shared_experts 1."},"moe.top_k":{"value":4,"source":"config","at":"/text_config/num_experts_per_tok"},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.placement":{"value":"pre","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm","source":"modeling_mistral4","at":"modeling_mistral4.py; Mistral4DecoderLayer.__init__","quote":"self.input_layernorm = Mistral4RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"ctx.extension":{"value":"YaRN factor 128 over original 8,192 (release config)","source":"config","at":"config.json; text_config.rope_parameters","quote":"\"factor\": 128.0, \"llama_4_scaling_beta\": 0.1, \"mscale\": 1.0, \"mscale_all_dim\": 1.0, \"original_max_position_embeddings\": 8192,","note":"Training-time context stages are not disclosed. Same dict: rope_type/type yarn, beta_fast 32, beta_slow 1."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Key Features","quote":"256k context length","note":"Config /text_config/max_position_embeddings is 1,048,576."},"train.cutoff":{"value":"July 2025","source":"data_summary","at":"§1.3 Latest date of data acquisition/collection","quote":"The latest date of data collection was July 2025."},"train.data":{"value":"multilingual text and images: Common Crawl and other public datasets, licensed and third-party datasets, synthetic data, user interaction data (subject to opt-out)","source":"data_summary","at":"§2.1 Publicly available datasets","quote":"The datasets used to train the model include Common Crawl.","note":"§1.3 ticks \"More than 10 trillions tokens\" for the text training data."},"eval.aa_index":{"value":11,"source":"aa","at":"Artificial Analysis model page /models/mistral-small-4, Wayback snapshot 2026-10-19","quote":"Mistral Small 4 (Reasoning) scores 11 on the Artificial Analysis Intelligence Index","note":"Variant: Mistral Small 4 (Reasoning). Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":118972826624,"formula":"embedding + output head + all experts + other weights, mistralai/Mistral-Small-4-119B-2603@a11f36b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1073741824,"formula":"input embedding + output head, mistralai/Mistral-Small-4-119B-2603@a11f36b safetensors headers"},"params.counted_non_embedding":{"value":117899084800,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, mistralai/Mistral-Small-4-119B-2603@a11f36b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":3274974464,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, mistralai/Mistral-Small-4-119B-2603@a11f36b safetensors headers"},"params.counted_active_non_embedding":{"value":5558846464.0,"formula":"non-expert weights + routed experts × 4/128, excluding embedding and output head, mistralai/Mistral-Small-4-119B-2603@a11f36b safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":113.77777777777777,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.0546218487394958,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":23040.0,"formula":"23040 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3019898880.0,"formula":"23040 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":23040.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":3544.6153846153848,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":464599827.6923077,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":23040.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":23040.0,"tokens_per_entry":1}],"formula":"Global KV: 36 × (320 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"motif-3","name":"Motif-3-Base","org":"Motif Technologies","country":"South Korea","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/Motif-Technologies/Motif-3-Base/resolve/ac7c9a74d113691e789b3df34d4ef685eeaef9a1/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Motif-Technologies/Motif-3-Base/resolve/ac7c9a74d113691e789b3df34d4ef685eeaef9a1/config.json","kind":"config"},"index":{"url":"https://huggingface.co/Motif-Technologies/Motif-3-Base/resolve/ac7c9a74d113691e789b3df34d4ef685eeaef9a1/model.safetensors.index.json","kind":"config"},"modeling_motif":{"url":"https://huggingface.co/Motif-Technologies/Motif-3-Base/resolve/ac7c9a74d113691e789b3df34d4ef685eeaef9a1/modeling_motif.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2608.09119v1","kind":"paper"},"train_model":{"url":"https://raw.githubusercontent.com/MotifTechnologies/motif3-training-example/f2cad4115bde073bc61dead4dce8db9711db0ab6/llm_training/motif/model/model.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019104255/https://artificialanalysis.ai/models/motif-3","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-10","source":"report","at":"p.1 arXiv stamp","quote":"arXiv:2608.09119v1 [cs.AI] 10 Aug 2026","note":"Card is undated; the Hugging Face repository was created 2026-08-09."},"meta.license":{"value":"mit","source":"card","at":"README.md; YAML front matter","quote":"license: mit"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md front matter","quote":"pipeline_tag: text-generation","note":"No modality statement; text-generation pipeline tag and a causal LM config with no vision or audio config."},"params.total":{"value":314000000000,"source":"report","at":"p.1 Abstract","quote":"We introduce Motif 3, a decoder-only Mixture-of-Experts language model with 314 billion total parameters and 13.2 billion parameters activated for each token."},"params.active":{"value":13200000000,"source":"report","at":"p.1 Abstract","quote":"We introduce Motif 3, a decoder-only Mixture-of-Experts language model with 314 billion total parameters and 13.2 billion parameters activated for each token."},"shape.layers":{"value":53,"source":"config","at":"/num_hidden_layers"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.vocab":{"value":220160,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.tokenizer":{"value":"SuperBPE trained from scratch (stage 2 adds multi-word superword tokens), 220,160 vocab","source":"report","at":"p.15 §4.1 Tokenizer","quote":"We trained the Motif 3 tokenizer from scratch using the SuperBPE reference implementation"},"attn.kind":{"value":"GDLA (MLA latent KV + grouped differential attention), SWA","source":"report","at":"p.1 Abstract","quote":"Motif 3 is built around Grouped Differential Latent Attention (GDLA), which integrates grouped differential attention with the compressed key-value representation of Multi-head Latent Attention."},"attn.layout":{"value":"3 SWA : 1 global (global first: layers 0, 4, …, 52)","source":"report","at":"p.19 §4.3 Model configuration","quote":"Under zero-based indexing, layers 0, 4, 8, . . . use full attention, while the three intervening layers use sliding-window attention.","note":"Both layer types use GDLA; config use_sliding_window true, sliding_window_pattern interleave. The source gives zero-based order explicitly; modeling_motif.py uses layer_idx % sliding_window_period, with config period 4.","basis":"inferred","sequence":["GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA","SWA","SWA","SWA","GDLA"]},"attn.global_layers":{"value":14,"source":"modeling_motif","at":"modeling_motif.py; MotifGDLAttention.__init__","quote":"elif pattern == \"interleave\" and layer_idx % period != 0:","basis":"inferred","note":"Config has no per-layer list; with /sliding_window_period 4 and /num_hidden_layers 53, layers 0, 4, …, 52 are full attention."},"attn.local_layers":{"value":39,"source":"modeling_motif","at":"modeling_motif.py; MotifGDLAttention.__init__","quote":"elif pattern == \"interleave\" and layer_idx % period != 0:","basis":"inferred","note":"53 layers (/num_hidden_layers) minus the 14 with layer_idx % 4 == 0 (/sliding_window_period 4)."},"attn.window":{"value":128,"source":"report","at":"p.19 §4.3 Model configuration","quote":"the remaining layers use sliding-window attention with a window of 128 tokens.","note":"config sliding_window 128; HF code passes sliding_window + 1 to flash attention."},"attn.heads":{"value":80,"source":"report","at":"p.19 §4.3 Model configuration","quote":"GDLA uses 80 query heads and 16 KV heads","note":"num_attention_heads 80, num_key_value_heads 16. All 80 heads (64 signal + 16 noise, g = 4) compute attention; each group of 4 signal + 1 noise heads shares one of 16 KV heads up-projected from the latent. Each noise output is subtracted from its group's signal outputs, so W_O sees 64 heads × 128."},"attn.head_dim":{"value":192,"source":"report","at":"p.19 §4.3 Model configuration","quote":"with a query-key head dimension of 192, a value-head dimension of 128, and a 64-dimensional rotary component.","note":"config head_dim 192 = 128 NoPE + 64 RoPE."},"attn.v_head_dim":{"value":128,"source":"report","at":"p.19 §4.3 Model configuration","quote":"with a query-key head dimension of 192, a value-head dimension of 128, and a 64-dimensional rotary component."},"attn.kv_lora_rank":{"value":512,"source":"report","at":"p.19 §4.3 Model configuration","quote":"The query and KV low-rank dimensions are 1,024 and 512, respectively.","note":"config kv_lora_rank 512; v_head_dim 128."},"attn.q_lora_rank":{"value":1024,"source":"report","at":"p.19 §4.3 Model configuration","quote":"The query and KV low-rank dimensions are 1,024 and 512, respectively."},"attn.rope_dim":{"value":64,"source":"report","at":"p.19 §4.3 Model configuration","quote":"with a query-key head dimension of 192, a value-head dimension of 128, and a 64-dimensional rotary component.","note":"config qk_rope_head_dim 64."},"attn.position":{"value":"partial RoPE (decoupled 64-dim RoPE key shared across heads + 128 NoPE dims); YaRN on global layers only","source":"report","at":"p.19 §4.3 Batch size and context-length schedule","quote":"The full-attention layers use a DeepSeek-YaRN extension from the original 4K context length to 256K, with an extension factor of 64, while the sliding-window layers retain local RoPE without long-range extension."},"attn.rope_theta":{"value":10000,"source":"config","at":"/rope_scaling/rope_theta","note":"Top-level rope_theta is also 10000."},"attn.local_heads":{"value":80,"source":"config","at":"/num_attention_heads","note":"MotifGDLAttention reads the same head counts and dims for SWA and full layers; only the window and rotary embedding differ."},"attn.local_kv_heads":{"value":16,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":192,"source":"config","at":"/head_dim","note":"QK 192, V 128 (/v_head_dim), as in global layers."},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/swa_rope_theta","note":"SWA layers build their own rotary embedding with rope_scaling None (no YaRN)."},"attn.qk_norm":{"value":false,"source":"modeling_motif","at":"modeling_motif.py; MotifGDLAttention.forward","quote":"kv_proj = self.wkv_b(self.kv_norm(kv_latent))","note":"RMSNorm is applied only to the Q and KV latents before up-projection, not per head."},"attn.gate":{"value":"element-wise sigmoid gate from the normalized query latent, on the differential output before W_O","source":"report","at":"p.5 §2.2.3","quote":"Before the final output projection, the differential representation is modulated by an element-wise sigmoid gate computed from the normalized query latent","note":"config elementwise_attn_output_gate true; diff_v2 true selects the input-dependent λ variant (the only one in HF code)."},"attn.scale":{"value":"global layers: 1/sqrt(192) × (0.1·ln 64 + 1)² (YaRN mscale); SWA layers: 1/sqrt(192)","source":"modeling_motif","at":"modeling_motif.py; MotifGDLAttention.__init__","quote":"self.scaling = self.scaling * mscale_val * mscale_val","note":"/rope_factor 64, /mscale 1. Differential output: attn1 - sigmoid(lambda_proj(x)) * attn2, λ per token and signal head (report Eq. 12–13)."},"attn.bias":{"value":false,"source":"modeling_motif","at":"modeling_motif.py; MotifGDLAttention.__init__","quote":"self.wq_a = nn.Linear(self.hidden_size, self.q_lora_rank, bias=False)","note":"wq_b, wkv_a, wkv_b, lambda_proj and wo are also bias=False."},"ffn.activation":{"value":"Expert-Specific PolyNorm (gated: PolyNorm(gate) × up; per-expert sigmoid-bounded coefficients)","source":"report","at":"p.6 §2.4","quote":"Motif 3 instead replaces the SiLU gate activation with Expert-Specific PolyNorm, whose polynomial coefficients and bias are learned independently for each expert.","note":"config hidden_act poly_norm. PolyNorm(z) = Σ_{n=1..3} a_n·z^n/RMS(z^n) + b; dense layers and the shared expert use a single PolyNorm. Code multiplies the activation output by /polynorm_output_scale 0.5."},"ffn.clamp":{"value":"none (soft FFN-output magnitude penalty instead)","source":"report","at":"p.16 §4.2 FFN magnitude regularization","quote":"We avoid this forward-pass intervention because activation outliers are not necessarily meaningless numerical artifacts.","note":"Routed-expert PolyNorm bias is clipped to [-0.5, 0.5] (Eq. 24, /polynorm_bias_clamp); the HF code's ±1e6 /hidden_clamp is a numerical guard."},"ffn.dim":{"value":12288,"source":"report","at":"p.18 §4.3 Model configuration","quote":"The first two layers use dense feed-forward networks with an intermediate dimension of 12,288"},"ffn.bias":{"value":false,"source":"modeling_motif","at":"modeling_motif.py; MotifMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Up and down projections also bias=False; the PolyNorm activation itself has a learned scalar bias."},"moe.experts":{"value":384,"source":"report","at":"p.18 §4.3 Model configuration","quote":"Each MoE layer contains 384 routed experts and one shared expert.","note":"config num_experts 384."},"moe.top_k":{"value":8,"source":"report","at":"p.18 §4.3 Model configuration","quote":"The router uses normalized sigmoid scores and selects eight routed experts per token.","note":"config experts_top_k 8."},"moe.shared":{"value":1,"source":"report","at":"p.18 §4.3 Model configuration","quote":"Each MoE layer contains 384 routed experts and one shared expert."},"moe.expert_dim":{"value":1280,"source":"report","at":"p.18 §4.3 Model configuration","quote":"Each routed expert has an intermediate dimension of 1,280."},"moe.shared_dim":{"value":1280,"source":"config","at":"/moe_intermediate_size","note":"1 shared expert (/num_shared_experts) of the routed-expert width (MoE.__init__)."},"moe.dense_layers":{"value":2,"source":"report","at":"p.18 §4.3 Model configuration","quote":"The first two layers use dense feed-forward networks with an intermediate dimension of 12,288, while the remaining 51 layers use sparse MoE feed-forward networks.","note":"config n_dense_first_layers 2, interleave_moe_layer_step 1."},"moe.router":{"value":"normalized sigmoid","source":"report","at":"p.18 §4.3 Model configuration","quote":"The router uses normalized sigmoid scores and selects eight routed experts per token.","note":"Routing computed in FP32; config score_func sigmoid, route_norm true."},"moe.norm_topk":{"value":true,"source":"config","at":"/route_norm"},"moe.routed_scale":{"value":2.0,"source":"config","at":"/route_scale"},"moe.balancing":{"value":"aux-loss-free bias (coefficient 1e-3) + sequence-wise aux loss 1e-4 (2.4e-4 layer 2, 2.0e-4 layers 3–5) + cosine-decayed Gaussian router-logit noise","source":"report","at":"p.19 §4.3 Hyperparameters","quote":"The sequence-wise auxiliary load-balancing loss is summed over tokens and uses a default weight of 1 × 10−4 . We increase this weight to 2.4 × 10−4 for layer 2 and to 2.0 × 10−4 for layers 3–5.","note":"Bias coefficient 1 × 10−3 in the same paragraph; router noise in §4.2. Collapsed routers were replaced with an adjacent layer's router and biases."},"mtp.depth":{"value":1,"source":"report","at":"p.7 Table 1","quote":"MTP head 1 layer for self-speculative decoding","note":"One DeepSeek-V3-style MTP module (§2.5); the index holds only mtp_layers.0."},"mtp.modules":{"value":1,"source":"index","at":"model.safetensors.index.json; weight_map keys","quote":"model.mtp_layers.0.input_proj.weight","note":"Only mtp_layers.0 in the index; training code builds one module (multi_token_preds=1)."},"mtp.block":{"value":"concat(h, RMSNorm(next-token embedding)) → linear 2d→d → GDLA block (SWA, no mHC) with dense PolyNorm FFN → RMSNorm → shared output head","source":"train_model","at":"llm_training/motif/model/model.py; compute_mtp_loss, Motif.__init__","quote":"h_out = model.mtp_layer[str(token_idx)](torch.cat([h_in, next_h], dim=-1))","note":"Code comment: 'Force dense FFN, SWA, and no MHC for MTP blocks'."},"mtp.ffn":{"value":"dense (12288)","source":"train_model","at":"llm_training/motif/model/model.py; Motif.__init__","quote":"interleave_moe_layer_step=0: always use dense FeedForward (no MoE)","note":"Dense FFN width is intermediate_dim 12288; index has mlp.gate_proj/up_proj/down_proj, no experts."},"mtp.shares_embedding":{"value":true,"source":"train_model","at":"llm_training/motif/model/model.py; compute_mtp_loss","quote":"mtp_loss = loss_fn(model.output(h), shifted_labels)","note":"The next-token embedding is the main tok_embeddings output (raw_embeds in Motif.forward)."},"mtp.loss_weight":{"value":"0.2","source":"report","at":"p.19 §4.3 Hyperparameters","quote":"The MTP auxiliary objective uses a loss weight of 0.2.","note":"Training code multiplies by mtp_depth_decay 0.95 and divides the summed loss by 1 + weight."},"mtp.released":{"value":true,"source":"index","at":"/weight_map | has mtp_layers.","note":"The card says the MTP head is not shipped, but shard model-00104-of-00155 contains mtp_layers.0 (GDLA + dense PolyNorm FFN, 19 tensors); the HF modeling code does not load it."},"norm.type":{"value":"RMSNorm","source":"report","at":"p.2 §2.1","quote":"We apply RMSNorm [74] before each sublayer"},"norm.placement":{"value":"pre","source":"report","at":"p.2 §2.1","quote":"We apply RMSNorm [74] before each sublayer and use modified manifold-constrained hyper-connections (mHC) [68] in place of conventional residual additions."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.residual":{"value":"modified mHC: 4 streams, Sinkhorn-Knopp doubly stochastic residual mixing, post-mapping multiplier annealed 2 → 1 during pretraining","source":"report","at":"p.5–6 §2.3","quote":"We use an expansion rate of n = 4, corresponding to four parallel residual streams.","note":"config mhc_enabled true, mhc_expansion_rate 4. Mappings computed in FP32 from an RMSNorm of the flattened streams; 20 Sinkhorn iterations (/mhc_sinkhorn_iters); embedding copied to all 4 streams and streams averaged before the final norm (MotifModel.forward)."},"ctx.pretrain":{"value":4096,"source":"report","at":"p.19 §4.3 Batch size and context-length schedule","quote":"Pretraining begins with sequences of up to 4K tokens."},"ctx.final":{"value":262144,"source":"card","at":"README.md; 2. Model Summary","quote":"Context Length 262,144 (256K)","note":"Report §4.3: a dedicated long-context stage with sequences of up to 256K tokens."},"ctx.max":{"value":262144,"source":"card","at":"README.md; Key Features","quote":"Native 256K context (262,144 tokens), trained with window-aware context parallelism.","note":"config max_position_embeddings 262144."},"ctx.extension":{"value":"4K → 32K during the LR decay phase → 256K stage; DeepSeek-YaRN factor 64 on global layers only","source":"report","at":"p.19 §4.3 Batch size and context-length schedule","quote":"During the learning-rate decay phase, we transition to a maximum sequence length of 32K tokens, followed by a dedicated long-context stage with sequences of up to 256K tokens.","note":"Config rope_scaling: rope_type yarn, factor 64, original_max_position_embeddings 4096 (original_seq_len), beta_fast 32, beta_slow 1, theta 10000."},"train.tokens":{"value":12500000000000,"source":"report","at":"p.14 §4.1 Data","quote":"Motif 3 is pretrained on approximately 12.5 trillion tokens, measured with the Motif tokenizer"},"train.optimizer":{"value":"Muon + AdamW (AdamW for embeddings, output head and vector parameters)","source":"report","at":"p.10 §3.3","quote":"Embeddings, the output projection, and vector-shaped parameters are instead optimized with AdamW.","note":"Each routed expert is orthogonalized as its own matrix; Newton–Schulz in BF16."},"train.lr_peak":{"value":0.0003,"source":"report","at":"p.19 §4.3 Hyperparameters","quote":"Training starts with a learning rate of 3 × 10−4 and follows a warmup-stable-decay (WSD) learning-rate schedule."},"train.lr_min":{"value":3e-05,"source":"report","at":"p.19 §4.3 Hyperparameters","quote":"The subsequent 256K long-context stage continues from this checkpoint and further reduces the learning rate to 3 × 10−5 ."},"train.schedule":{"value":"WSD: stable at 3e-4, decay to 7.5e-5 during the 4K → 32K transition, then 256K stage decays to 3e-5","source":"report","at":"p.19 §4.3 Hyperparameters","quote":"The stable phase performs the majority of pretraining at the maximum learning rate, after which the decay phase reduces the learning rate to 7.5 × 10−5 while transitioning from 4K to 32K sequences."},"train.batch":{"value":"up to 75M tokens","source":"report","at":"p.19 §4.3 Batch size and context-length schedule","quote":"The global training batch contains up to 75 million tokens."},"train.batch_tokens":{"value":75000000,"source":"report","at":"p.19 §4.3 Batch size and context-length schedule","quote":"The global training batch contains up to 75 million tokens.","note":"Stated as an upper bound; no ramp described."},"train.precision":{"value":"MXFP8 for MoE expert grouped GEMMs and EP dispatch; BF16 elsewhere; FP32 master weights, gradients, router logits and Muon states; BF16 gradient communication with FP32 reduction","source":"report","at":"p.9 §3.2","quote":"More specifically, we apply MXFP8 [59] to the MoE experts"},"train.stability":{"value":"QK-Clip (τ=100, 200 in the 256K stage; every 10 steps; key-side share 1/(1+√G)); FFN-output magnitude penalty (weight 2e-4, threshold 128, 1,024 on the final layer); mHC post-multiplier annealing; FP32 routing; router replacement on collapse","source":"report","at":"p.11 §3.3 QK-Clip; p.19 §4.3","quote":"We perform monitoring and clipping every 10 steps, using τ = 100 during main pre-training and τ = 200 during the 256K-token long-context stage.","note":"Penalty values: 'The FFN magnitude regularizer uses a loss weight of 2.0 × 10−4 and a threshold of 128' (p.19)."},"train.masking":{"value":"document-masked packing","source":"report","at":"p.13 §3.5 Attention-Workload-Aware Re-Scheduling","quote":"Document-masked packing concatenates multiple documents into a fixed-length sequence while preventing cross-document attention.","note":"Stated in the long-context training section; not stated separately for the 4K stage."},"train.data":{"value":"~70% NVIDIA Nemotron pretraining collection (some components filtered), most of the rest in-house web, STEM and synthetic; dedicated Korean, multilingual, legal and financial data; dynamic mixture 19 → 57 datasets; reasoning data <5%","source":"report","at":"p.14 §4.1 Data","quote":"The retained Nemotron data account for approximately 70% of the full Motif 3 pretraining corpus.","note":"Long-context stage uses ~5% of the corpus reconstructed as long-context data (§4.3)."},"train.cutoff":{"value":"March 2026","source":"report","at":"p.14 §4.1 Data","quote":"The latest data included in the corpus were collected through March 2026, which we use as the model’s knowledge cutoff."},"train.hardware":{"value":"B200 GPUs (count not stated); EP 8, DP-shard 8, no PP","source":"report","at":"p.7 §3.1","quote":"With expert and parameter sharding, optimizer-state CPU offloading, and the memory capacity of B200 GPUs, our 314B-parameter model fits within the available device memory"},"train.accelerators":{"value":"B200","source":"report","at":"p.7 §3.1","quote":"With expert and parameter sharding, optimizer-state CPU offloading, and the memory capacity of B200 GPUs","note":"Count not stated."},"train.parallelism":{"value":"EP8 · HSDP (DP-shard 8 · DP-replicate)","source":"report","at":"p.7 §3.1","quote":"Outside the expert-parallel dimension, we use a sharded data-parallel (DP-shard) degree of 8 and assign the remaining ranks to the replicated data-parallel (DP-replicate) dimension.","note":"Same paragraph: EP = 8 within one eight-GPU node, FSDP sharding, no pipeline parallelism."},"train.stages":{"value":[{"name":"main","seq_len":4096,"lr":0.0003,"note":"WSD stable phase.","quote":"Pretraining begins with sequences of up to 4K tokens."},{"name":"decay","seq_len":32768,"note":"LR decays to 7.5e-5 while moving from 4K to 32K sequences.","quote":"During the learning-rate decay phase, we transition to a maximum sequence length of 32K tokens"},{"name":"long-context","seq_len":262144,"note":"LR decays to 3e-5; ~5% of the corpus rebuilt as long-context data; EP 8 + CP 8.","quote":"followed by a dedicated long-context stage with sequences of up to 256K tokens."}],"source":"report","at":"p.19 §4.3"},"eval.aa_index":{"value":34,"source":"aa","at":"Artificial Analysis model page /models/motif-3, Wayback snapshot 2026-10-19","quote":"Motif 3 scores 34 on the Artificial Analysis Intelligence Index","note":"Score of Motif-3, post-trained from this base. Variant: Motif 3. Index v4.3.2."}},"not_disclosed":["train.warmup","train.weight_decay","train.betas","train.grad_clip","params.active_non_embedding"],"derived":{"params.counted_total":{"value":314590623874,"formula":"embedding + output head + all experts + other weights, Motif-Technologies/Motif-3-Base@ac7c9a7 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1803550720,"formula":"input embedding + output head, Motif-Technologies/Motif-3-Base@ac7c9a7 safetensors headers"},"params.counted_non_embedding":{"value":312787073154,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Motif-Technologies/Motif-3-Base@ac7c9a7 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":6124514746,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Motif-Technologies/Motif-3-Base@ac7c9a7 safetensors headers"},"params.counted_active_non_embedding":{"value":11174595810.0,"formula":"non-expert weights + routed experts × 8/384, excluding embedding and output head, Motif-Technologies/Motif-3-Base@ac7c9a7 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":77.28301886792453,"formula":"d_model / layers"},"attn.qk_width":{"value":15360,"formula":"heads × head_dim"},"ffn.mult":{"value":3.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3125,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":2.7857142857142856,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":48.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.04203821656050955,"formula":"params.active / params.total"},"scaling.tpp":{"value":39.80891719745223,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1118.6086917626062,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":16128.0,"formula":"16128 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":2119680000.0,"formula":"16128 B × 131072 (Global KV) + 44928 B × min(131072, 128) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":16171.875,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1225.1420454545455,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":160581818.1818182,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":16128.0,"windows":[[44928.0,128]],"groups":[{"name":"Global KV","bytes_per_entry":16128.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":44928.0,"tokens_per_entry":1,"window":128}],"formula":"Global KV: 14 × (576 × 16/8) B/entry; SWA KV: 39 × (576 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"muse-spark-open","name":"Muse-Spark-Open-816B-A42B","org":"Meta","country":"United States","kind":"reasoning","weights":"open","note":"Only the post-trained multimodal checkpoint is public. Its config.json is gated, so values noted as MuseSparkTextConfig defaults come from the transformers config class declared for this checkpoint.","sources":{"card":{"url":"https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B/resolve/0fb02478b02565b2ce7fc57e6051c91172774ac9/README.md","kind":"model-card"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/muse_spark/modeling_muse_spark.py","kind":"code"},"configuration_muse_spark":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/muse_spark/configuration_muse_spark.py","kind":"code"},"torchtitan_flavors":{"url":"https://raw.githubusercontent.com/pytorch/torchtitan/48bbce4434dfac2c38fedc9c7ef9c1998b04cf50/torchtitan/models/muse_spark/flavors.py","kind":"code"},"aa":{"url":"https://web.archive.org/web/20261019092913/https://artificialanalysis.ai/models/muse-spark-open","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-10-15","source":"card","at":"README.md; Muse Spark Open Model Overview; Date of Release","quote":"Date of Release | October 15th, 2026"},"meta.license":{"value":"other (Muse Model License)","source":"card","at":"README.md; YAML front matter","quote":"license_name: muse-model-license"},"meta.modality":{"value":"text + image + video + audio","source":"card","at":"README.md; model summary table, Modalities","quote":"Input: Text \\+ Image \\+ Video \\+ Audio / Output: Text"},"params.total":{"value":816000000000,"source":"card","at":"README.md; Muse Spark Open Model Card","quote":"Muse Spark Open is an 816B total/42B active parameter open weights model with a context window of 1M tokens."},"params.active":{"value":42000000000,"source":"card","at":"README.md; Muse Spark Open Model Card","quote":"Muse Spark Open is an 816B total/42B active parameter open weights model with a context window of 1M tokens."},"params.note":{"value":"Whether 816B includes the 1.8B-param Perception Encoder is not stated.","source":"card","at":"README.md; Muse Spark Open Model Overview; Type","quote":"Mixture-of-Experts Transformer, with Perception Encoder"},"params.embedding":{"value":3310354432,"source":"card","at":"README.md; Model Overview; Language Model","quote":"untied input/output embeddings, vocab × dim \\= 202,048 × 8192 ≈ \\~1.66B","basis":"inferred","note":"2 × 202,048 × 8192 for the untied input embedding and output layer."},"shape.d_model":{"value":8192,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Hidden Dimension: 8192"},"shape.layers":{"value":62,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Layers: 62"},"shape.vocab":{"value":202048,"source":"card","at":"README.md; Model Overview; Vocabulary Size","quote":"Vocabulary Size | 202,048"},"shape.tied_embeddings":{"value":false,"source":"card","at":"README.md; Model Overview; Language Model","quote":"untied input/output embeddings"},"attn.kind":{"value":"GQA, SWA","source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig.__post_init__","quote":"Full attention on NoPE layers (every 4th, counted backward from the last), sliding otherwise — the reference config's sliding_window_pattern [w, w, w, 0].","note":"Card: grouped-query attention."},"attn.layout":{"value":"1 SWA, then 1 global, then (3 SWA : 1 global) × 15","source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig.__post_init__; lines 153–155","quote":"\"full_attention\" if (self.num_hidden_layers - 1 - i) % 4 == 0 else \"sliding_attention\"","basis":"inferred","note":"At 62 layers, these defaults put global attention at zero-based indices 1, 5, ..., 61 (one-based layers 2, 6, ..., 62). The checkpoint config is gated; this sequence is derived from MuseSparkTextConfig defaults, not an extracted checkpoint array.","sequence":["SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":16,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig.__post_init__","quote":"Full attention on NoPE layers (every 4th, counted backward from the last), sliding otherwise — the reference config's sliding_window_pattern [w, w, w, 0].","basis":"inferred","note":"Layers i in 0..61 (62 layers, card) with (61 - i) % 4 == 0."},"attn.local_layers":{"value":46,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig.__post_init__","quote":"Full attention on NoPE layers (every 4th, counted backward from the last), sliding otherwise — the reference config's sliding_window_pattern [w, w, w, 0].","basis":"inferred","note":"62 layers (card) minus 16 global layers."},"attn.heads":{"value":128,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Attention Heads: 128 for Q and 16 for KV (grouped-query attention)"},"attn.kv_heads":{"value":16,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Attention Heads: 128 for Q and 16 for KV (grouped-query attention)","note":"MuseSparkTextAttention uses the same KV head count on SWA and global layers."},"attn.local_heads":{"value":128,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Attention Heads: 128 for Q and 16 for KV (grouped-query attention)","note":"One attention class for both layer types; only the window and RoPE differ (MuseSparkTextAttention, torchtitan _muse_spark_config)."},"attn.local_kv_heads":{"value":16,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Attention Heads: 128 for Q and 16 for KV (grouped-query attention)","note":"Same attention class on SWA and global layers."},"attn.local_head_dim":{"value":64,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Head Dimension: 64","note":"Same attention class on SWA and global layers."},"attn.local_rope_theta":{"value":500000,"source":"torchtitan_flavors","at":"torchtitan/models/muse_spark/flavors.py; module constants","quote":"_ROPE_THETA = 500_000.0","note":"Also the rope_parameters default of MuseSparkTextConfig; global layers are NoPE."},"attn.head_dim":{"value":64,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Head Dimension: 64"},"attn.window":{"value":2048,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"sliding_window: int | None = 2048","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B)."},"attn.position":{"value":"RoPE (SWA, θ 500000), NoPE (global)","source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig docstring, layer_rope_theta","quote":"Defaults to the global theta everywhere except every 4th layer counted backward from the last, which is NoPE.","note":"θ 500000 is the default rope_parameters in MuseSparkTextConfig.__post_init__, a class declared for this checkpoint."},"attn.qk_norm":{"value":true,"source":"card","at":"README.md; Model Overview; Language Model","quote":"QK-norm enabled","note":"Weightless RMSNorm on Q and K (MuseSparkTextAttention); see attn.scale."},"attn.scale":{"value":"1/sqrt(head_dim) × 5.4730064863875 (Q multiplied by qk_scale_factor after QK-norm)","source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"qk_scale_factor: float = 5.4730064863875","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B). Applied in MuseSparkTextAttention.forward."},"attn.bias":{"value":false,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"attention_bias: bool = False","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B)."},"attn.gate":{"value":"elementwise sigmoid gate computed from the block input, applied after a weightless per-head output RMSNorm, before o_proj","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTextAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(self.gate_proj(hidden_states))","note":"Card: output gate and output norm on the attention block."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is hidden_act, silu by MuseSparkTextConfig default (class declared for this checkpoint)."},"ffn.clamp":{"value":"none","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"The torchtitan 816B-A42B flavor also uses plain SwiGLU; only torchtitan's Dist-MoE throughput recipes for continued training from the released checkpoint swap in ClampedSwiGLU (limit 7, α 1.702, up + 1), a different function, so it does not describe pretraining."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkExperts.__init__","quote":"self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))","basis":"inferred","note":"Experts hold only gate_up_proj and down_proj weights; no bias parameters."},"moe.experts":{"value":256,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Experts: 256"},"moe.top_k":{"value":8,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Experts Activated per Token: 8 (top-k routing)"},"moe.expert_dim":{"value":4096,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"moe_intermediate_size: int = 4096","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B). Experts run at moe_hidden_size 4096 (see moe.latent_dim)."},"moe.shared":{"value":0,"source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkMoE.__init__","quote":"self.experts = MuseSparkExperts(config) self.post_expert_proj = nn.Linear(config.moe_hidden_size, config.hidden_size, bias=False)","basis":"inferred","note":"MuseSparkMoE has a router, pre/post projections and routed experts only; no shared-expert branch."},"moe.dense_layers":{"value":0,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig.__post_init__","quote":"self.mlp_layer_types = [\"sparse\"] * self.num_hidden_layers","note":"MuseSparkTextConfig default; the torchtitan 816B-A42B flavor also builds a MoE FFN in all 62 layers."},"moe.router":{"value":"sigmoid","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTopkRouter.__init__","quote":"self.scoring_function = ACT2FN[\"sigmoid\"]","note":"Logits computed in FP32 and scaled by output_multiplier; an e_score_correction_bias buffer is added for top-k selection only."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTopkRouter.forward","quote":"top_k_weights = top_k_weights / (top_k_weights.sum(dim=-1, keepdim=True) + self.norm_topk_prob_eps)"},"moe.routed_scale":{"value":1.0,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"routed_scaling_factor: float = 1.0","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B)."},"moe.latent_dim":{"value":4096,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"moe_hidden_size: int = 4096","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B). pre_expert_proj 8192 → 4096 (then RMSNorm) before the experts, post_expert_proj back to 8192."},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTextDecoderLayer.__init__","quote":"self.input_layernorm = MuseSparkTextCenteredRMSNorm(config.hidden_size, eps=config.rms_norm_eps)","note":"Pre-attention and pre-FFN norms; post-attention norm is weightless, post-FFN norm scales the input by (1+w) before normalizing, final norm is a plain RMSNorm."},"norm.placement":{"value":"sandwich","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTextDecoderLayer.forward","quote":"hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.post_attention_residual_gate(residual, hidden_states)","note":"The post-norm output joins the residual through a learned gate (see norm.residual)."},"norm.eps":{"value":1e-05,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"rms_norm_eps: float = 1e-5","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B). Post-attention, post-FFN and post-expert norms use post_norm_eps 1e-8."},"norm.residual":{"value":"learned per-channel residual gate: alpha·residual + beta·h, beta = sigmoid(g/0.3), alpha = sqrt(sigmoid(-g/0.3)·(1+beta))","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkResidualGate.forward","quote":"alpha = torch.sqrt(torch.clamp(torch.sigmoid(-w) * (1.0 + beta), min=self.eps))","note":"Temperature 0.3 is the residual_gate_temperature default of MuseSparkTextConfig (class declared for this checkpoint)."},"norm.embedding":{"value":"weightless RMSNorm after embedding lookup","source":"hf-transformers","at":"modeling_muse_spark.py; MuseSparkTextNormedEmbedding.forward","quote":"return self.embed_norm(super().forward(input_ids))"},"norm.output_scale":{"value":"logits × 1/sqrt(d_model/256) = 0.1767766952966369, before the softcap","source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig docstring, output_multiplier","quote":"Equal to `1/sqrt(hidden_size / 256)` for the released checkpoint.","note":"Router logits are scaled by the same output_multiplier."},"norm.logit_softcap":{"value":20,"source":"configuration_muse_spark","at":"configuration_muse_spark.py; MuseSparkTextConfig","quote":"final_logit_softcapping: float = 20.0","note":"MuseSparkTextConfig default; the class is declared for this checkpoint (auto_docstring checkpoint=meta-models/Muse-Spark-Open-816B-A42B). Logits are first multiplied by output_multiplier (see norm.output_scale)."},"ctx.max":{"value":1000000,"source":"card","at":"README.md; Muse Spark Open Model Card","quote":"with a context window of 1M tokens"},"train.data":{"value":"text, images, audio and video; more than 100 languages","source":"card","at":"README.md; Training data","quote":"Training data includes text, images, audio, and video.","note":"Card Languages row: trained on data from more than 100 languages."},"eval.aa_index":{"value":39,"source":"aa","at":"Artificial Analysis model page /models/muse-spark-open, Wayback snapshot 2026-10-19","quote":"Muse Spark Open (Xhigh) scores 39 on the Artificial Analysis Intelligence Index","note":"Variant: Muse Spark Open (Xhigh). Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"shape.aspect":{"value":132.1290322580645,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.5,"formula":"moe.expert_dim / d_model"},"moe.latent_ratio":{"value":2.0,"formula":"d_model / moe.latent_dim"},"attn.swa_ratio":{"value":2.875,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.051470588235294115,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":65536.0,"formula":"65536 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":8975810560.0,"formula":"65536 B × 131072 (Global KV) + 188416 B × min(131072, 2048) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":68480.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1630.4761904761904,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":213709775.23809522,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":65536.0,"windows":[[188416.0,2048]],"groups":[{"name":"Global KV","bytes_per_entry":65536.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":188416.0,"tokens_per_entry":1,"window":2048}],"formula":"Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 46 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"nemotron-3-ultra","name":"Nemotron-3-Ultra-550B-A55B-Base","org":"NVIDIA","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16/resolve/9f5b80e4845884eb8da03353b0b176503631e034/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16/resolve/9f5b80e4845884eb8da03353b0b176503631e034/config.json","kind":"config"},"index":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16/resolve/9f5b80e4845884eb8da03353b0b176503631e034/model.safetensors.index.json","kind":"config"},"modeling_nemotron_h":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/nemotron_h/modeling_nemotron_h.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2606.15007v1","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019104511/https://artificialanalysis.ai/models/nvidia-nemotron-3-ultra-550b-a55b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-06-04","source":"card","at":"README.md; Release Date","quote":"Hugging Face - 06/04/2026","note":"Report on arXiv 2026-06-12; Hugging Face repository created 2026-06-03."},"meta.license":{"value":"other (openmdw-1.1)","source":"card","at":"README.md; YAML front matter","quote":"license_name: openmdw-1.1"},"meta.modality":{"value":"text","source":"card","at":"README.md; Input","quote":"Input Type(s): Text"},"params.active":{"value":55000000000,"source":"report","at":"PDF page 1; Abstract","quote":"a 550 billion total and 55 billion active parameter Mixture-of-Experts"},"params.note":{"value":"Rounded publisher figures; MTP inclusion not stated. The checkpoint index lists 560.5B parameters including the MTP block.","source":"index","at":"model.safetensors.index.json; metadata","quote":"\"total_parameters\": 560524578816"},"params.total":{"value":550000000000,"source":"report","at":"PDF page 1; Abstract","quote":"a 550 billion total and 55 billion active parameter Mixture-of-Experts"},"shape.d_model":{"value":8192,"source":"config","at":"/hidden_size"},"shape.layers":{"value":60,"source":"report","at":"PDF page 4; Table 1","quote":"Total Layers 108","basis":"inferred","note":"One block per token mixer: 48 Mamba-2 + 12 attention (config layers_block_type). 48 blocks are followed by a MoE FFN; the 12 Mamba-2 layers right before each attention layer have none. The publisher's 108 layers (Table 1) count mixer and MoE sublayers separately."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":131072,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":12,"source":"config","at":"/layers_block_type | count attention"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, Mamba-2","source":"report","at":"PDF page 3; §2.1","quote":"Nemotron 3 Ultra uses the same hybrid Mamba-Attention Mixture-of-Experts (MoE) architecture as Nemotron 3 Super","note":"Attention is GQA with 64 query and 2 KV heads (Table 1)."},"attn.kv_heads":{"value":2,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"48 Mamba-2 + 12 GQA","source":"config","at":"config.json; /layers_block_type","quote":"\"layers_block_type\": [ \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"attention\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\", \"mamba\", \"moe\" ]","basis":"inferred","note":"From config layers_block_type. Irregular: Mamba-2 runs before, between and after the 12 attention layers are 4, 3, 4, 4, 3, 4, 4, 3, 4, 4, 3, 4, 4. Ordered token mixers from /layers_block_type; standalone moe/MLP blocks are omitted, matching shape.layers. Their order is not a claim that the FFNs are absent.","sequence":["Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2"]},"attn.linear_layers":{"value":48,"source":"config","at":"/layers_block_type | count mamba"},"attn.linear_kind":{"value":"Mamba-2: 256 heads × head dim 64, state 128, 8 groups, conv kernel 4","source":"report","at":"PDF page 4; Table 1","quote":"Mamba State Dimension 128 Mamba Groups 8 Mamba Heads 256 Mamba Head Dimension 64","note":"Conv kernel from config /conv_kernel (use_conv_bias true); groups = n_groups, state = ssm_state_size; mamba_hidden_act silu; dt clamped below at time_step_min 0.001 (NemotronHMamba2Mixer time_step_limit)."},"attn.linear_heads":{"value":256,"source":"config","at":"/mamba_num_heads"},"attn.linear_head_dim":{"value":64,"source":"config","at":"/mamba_head_dim"},"attn.conv":{"value":4,"source":"config","at":"/conv_kernel"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.position":{"value":"NoPE","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHAttention.forward","quote":"key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)","note":"No rotary embedding is applied to q or k; config rope_theta is unused."},"attn.qk_norm":{"value":false,"source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHAttention.forward","quote":"query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)"},"ffn.activation":{"value":"ReLU²","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHExperts.forward","quote":"# Expert computation: down_proj(act_fn(up_proj(x)))","note":"Not gated; act_fn = ACT2FN[mlp_hidden_act] with config mlp_hidden_act relu2, for routed and shared experts."},"ffn.bias":{"value":false,"source":"config","at":"/mlp_bias"},"moe.balancing":{"value":"aux-loss-free bias (update rate 1e-3) + load-balancing loss 1e-4","source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'auxiliary-loss-free load balancing strategy ... with an update rate of 10−3, paired with a standard load balancing loss with coefficient of 10−4'."},"moe.expert_dim":{"value":5120,"source":"config","at":"/moe_intermediate_size","note":"Routed experts map the 2048-wide latent to 5120 and back (LatentMoE)."},"moe.experts":{"value":512,"source":"config","at":"/n_routed_experts"},"moe.latent_dim":{"value":2048,"source":"config","at":"/moe_latent_size","note":"fc1_latent_proj (8192→2048) and fc2_latent_proj (2048→8192) wrap the routed experts; router and shared expert act on the 8192-wide hidden state (NemotronHMoE.forward)."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":5.0,"source":"config","at":"/routed_scaling_factor"},"moe.router":{"value":"sigmoid","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHTopkRouter.forward","quote":"scores = router_logits.sigmoid()","note":"Expert-selection bias e_score_correction_bias is added for top-k choice only; n_group = topk_group = 1 (no group-limited routing)."},"moe.dense_layers":{"value":0,"source":"config","at":"/layers_block_type | count mlp"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":10240,"source":"config","at":"/moe_shared_expert_intermediate_size"},"moe.top_k":{"value":22,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":2,"source":"report","at":"PDF page 3; §2.1","quote":"native Multi-Token Prediction for inference acceleration with two heads during pre-training","note":"Both heads share one attention+MoE block; the checkpoint stores one block (num_nextn_predict_layers 1)."},"mtp.modules":{"value":1,"source":"report","at":"PDF page 3; §2.1","quote":"Both MTP heads share the same parameters to enable robust autoregressive drafting"},"mtp.block":{"value":"next-token embedding and hidden state each normed (enorm, hnorm), concatenated and projected 2×8192→8192 (eh_proj); then one GQA attention layer and one LatentMoE layer; final norm","source":"report","at":"PDF page 3; §2.1","quote":"consist of a single attention layer followed by a single MoE layer","note":"Config mtp_layers_block_type [attention, moe]; projection and norms from mtp.layers.0.{enorm,hnorm,eh_proj} in the index, eh_proj shape [8192, 16384] in the safetensors header."},"mtp.ffn":{"value":"MoE","source":"report","at":"PDF page 3; §2.1","quote":"consist of a single attention layer followed by a single MoE layer","note":"LatentMoE with the backbone's widths: expert up_proj [5120, 2048], latent proj [2048, 8192], shared expert [10240, 8192] (safetensors header)."},"mtp.shares_embedding":{"value":true,"source":"report","at":"PDF page 12; §2.7","quote":"the MTP blocks’ wgrad contribution to the shared output layer","note":"Output head stated shared; the MTP tensors (mtp.*) in the index also contain no embedding table."},"mtp.loss_weight":{"value":"0.1 (0.05 per MTP head)","source":"report","at":"PDF page 12; §2.7","quote":"Nemotron 3 Ultra uses 2 MTP blocks with a MTP loss scaling factor of 0.1 (0.05 for each MTP block)"},"mtp.released":{"value":true,"source":"index","at":"/weight_map | has mtp.layers.0.eh_proj"},"norm.eps":{"value":1e-05,"source":"config","at":"/layer_norm_epsilon"},"norm.placement":{"value":"pre","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHBlock docstring","quote":"Each block applies pre-normalization followed by the mixer, then adds a residual connection."},"norm.type":{"value":"RMSNorm","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHBlock.__init__","quote":"self.norm = NemotronHRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)"},"ctx.extension":{"value":"Long-context CPT at constant LR 2.5e-6 for 33B tokens: 92% of iterations at 1M, 8% at 4K; blend 46% long-context (document QA + SFT-style), 54% phase-2 data","source":"report","at":"PDF page 9; §2.5","quote":"We performed CPT on 1,048,576 (1M) context length for 92% of the iterations, while we trained on 4,096 (4K) for the remaining 8% of the time"},"ctx.final":{"value":1048576,"source":"report","at":"PDF page 9; §2.5","quote":"We performed CPT on 1,048,576 (1M) context length for 92% of the iterations"},"ctx.max":{"value":1048576,"source":"card","at":"README.md; Input","quote":"Maximum context length up to 1M tokens.","note":"Config max_position_embeddings is 262144; NoPE layers do not read it."},"ctx.pretrain":{"value":8192,"source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'The model was trained with a sequence length of 8,192'."},"train.averaging":{"value":"Checkpoint merge with a 500B-token window, selected from merges over 125B-1T windows and several orderings, used as the start of the long-context phase","source":"report","at":"PDF page 9; §2.4","quote":"A 500B token merge window checkpoint that exhibited a balanced trade-off between knowledge, math, and code was selected for the long-context phase."},"train.batch":{"value":"3,072 × 8,192 (25.17M tokens)","source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'batch size of 3,072 sequences, resulting in approximately 25.17 million tokens per batch'. The Ultra long-context phase states 25,165,824 tokens per iteration (§2.5)."},"train.batch_tokens":{"value":25165824,"source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Super recipe 3,072 × 8,192 = 25,165,824 (Super report §2.4), kept by Ultra §2.4; the Ultra long-context phase also states 25,165,824 tokens per iteration (§2.5)."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'momentum coefficients β1 = 0.9 and β2 = 0.95'."},"train.cutoff":{"value":"September 2025","source":"card","at":"README.md; Model Overview, Data Freshness","quote":"The pre-training data has a cutoff date of September 2025."},"train.data":{"value":"Two-phase curriculum over 19 categories, switching from diversity- to quality-biased mix after ~15T tokens (~75%); quality-filtered + synthetic web crawl ~49% of phase 1 and ~38% of phase 2; also finepdfs, math, code (refreshed to Sep 2025), Nemotron-CC-Code, Wikipedia, academic, legal, multilingual (11 languages), Crawl++, SFT-style data","source":"report","at":"PDF page 8; §2.3.6","quote":"The pretraining corpus spans 19 high level categories across both data mixtures."},"train.hardware":{"value":"GB200","source":"report","at":"PDF page 9; §2.5","quote":"128-way expert parallelism, and 2-way pipeline parallelism to train on GB200 GPUs.","note":"Stated only for the long-context phase."},"train.accelerators":{"value":"GB200","source":"report","at":"PDF page 9; §2.5","quote":"128-way expert parallelism, and 2-way pipeline parallelism to train on GB200 GPUs.","note":"Stated only for the long-context phase; count not stated."},"train.lr_min":{"value":2.5e-06,"source":"report","at":"PDF page 9; §2.4","quote":"minus-sqrt decay schedule to a minimum of 2.5 × 10−6"},"train.lr_peak":{"value":0.00025,"source":"report","at":"PDF page 8; §2.4","quote":"We warmup the learning rate for 200 billion tokens to a peak value of 2.5 × 10−4"},"train.optimizer":{"value":"AdamW","source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'We used AdamW (Loshchilov & Hutter, 2017) optimizer'."},"train.precision":{"value":"NVFP4 (E2M1; 2D block quantization on weights; RHT on wgrad inputs; stochastic rounding on gradients). Higher precision for the final 15% of the network (16 layers), Mamba output projections, latent projections, QKV/attention projections, MTP layers and embeddings; FP32 gradient reduction.","source":"report","at":"PDF pages 3-4; §2.2","quote":"NVFP4 layers use the E2M1 datatype with two-dimensional block quantization on weights, Random Hadamard Transforms on inputs to wgrad, and stochastic rounding on gradients","note":"The 16 layers count Mamba, attention and MoE sublayers separately (108 total)."},"train.schedule":{"value":"WSD: 200B-token warmup, stable to 15T, minus-sqrt decay over the final 5T tokens","source":"report","at":"PDF pages 8-9, 12; §2.4, §2.7","quote":"For the final 5 trillion tokens we then decay the learning rate","note":"After a divergence near 16T the run was rolled back to the 15T checkpoint and decay started there, cutting the horizon to 20T (§2.7)."},"train.stability":{"value":"Divergence at ~8T attributed to lowering local gradient accumulation for the output layer from FP32 to BF16 (MTP wgrad contribution lost): rollback + FP32 gradient reduction. Divergence at ~16T (cause undetermined): rollback to the 15T checkpoint + immediate LR decay.","source":"report","at":"PDF page 12; §2.7","quote":"Rolling back to an earlier checkpoint and moving back to the full FP32 gradient reduction recipe re-stabilized training"},"train.stages":{"value":[{"name":"phase-1","tokens":15000000000000,"seq_len":8192,"batch_sequences":3072,"note":"Diversity-biased mix; warmup and stable LR. seq_len and batch from the Super recipe, which Ultra §2.4 keeps.","quote":"with 15 trillion tokens of data in the first phase focusing on diversity and broad domain coverage"},{"name":"phase-2","tokens":5000000000000,"seq_len":8192,"batch_sequences":3072,"note":"Quality-biased mix; spans the minus-sqrt LR decay.","quote":"followed by 5 trillion tokens of data in the second phase focusing on high quality data"},{"name":"long-context","tokens":33000000000,"seq_len":1048576,"lr":2.5e-06,"batch_tokens":25165824,"note":"92% of iterations at 1M, 8% at 4K (math and code SFT-style data only); starts from a 500B-token checkpoint merge.","quote":"Eventually, the LC-Phase was trained for 33B tokens."}],"source":"report","at":"PDF pages 1, 9; §1, §2.5"},"train.tokens":{"value":20000000000000,"source":"report","at":"PDF page 1; Abstract","quote":"We pre-trained Nemotron 3 Ultra on 20 trillion text tokens","note":"Excludes the 33B-token long-context phase."},"train.warmup":{"value":"200B tokens","source":"report","at":"PDF page 8; §2.4","quote":"We warmup the learning rate for 200 billion tokens"},"train.weight_decay":{"value":0.1,"source":"report","at":"PDF page 9; §2.4","quote":"All other hyperparameters remain the same as for Nemotron 3 Super.","basis":"inferred","note":"Ultra §2.4 does not override it; Super report p.14 §2.4: 'optimizer with a weight decay of 0.1'."},"scaling.hp_rule":{"value":"Nemotron 3 Super recipe and hyperparameters, adjusted to a 20T WSD horizon and 2.5e-4 peak LR","source":"report","at":"PDF page 8; §2.4","quote":"We follow the same training recipe and hyperparameters as Nemotron 3 Super (NVIDIA, 2026), with a few adjustments"},"eval.aa_index":{"value":23,"source":"aa","at":"Artificial Analysis model page /models/nvidia-nemotron-3-ultra-550b-a55b, Wayback snapshot 2026-10-19","quote":"Nemotron 3 Ultra 550B A55B (Reasoning) scores 23 on the Artificial Analysis Intelligence Index","note":"Score of NVIDIA-Nemotron-3-Ultra-550B-A55B, post-trained from this base. Variant: Nemotron 3 Ultra 550B A55B (Reasoning). Index v4.3.2."}},"not_disclosed":["train.grad_clip","params.active_non_embedding"],"derived":{"params.counted_total":{"value":549308993536,"formula":"embedding + output head + all experts + other weights, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16@9f5b80e safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":2147483648,"formula":"input embedding + output head, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16@9f5b80e safetensors headers"},"params.counted_non_embedding":{"value":547161509888,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16@9f5b80e safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":10942947840,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16@9f5b80e safetensors headers"},"params.counted_active_non_embedding":{"value":53911359488.0,"formula":"non-expert weights + routed experts × 22/512, excluding embedding and output head, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16@9f5b80e safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":136.53333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":32.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.625,"formula":"moe.expert_dim / d_model"},"moe.latent_ratio":{"value":4.0,"formula":"d_model / moe.latent_dim"},"attn.linear_ratio":{"value":4.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":23.272727272727273,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.1,"formula":"params.active / params.total"},"scaling.tpp":{"value":36.36363636363637,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":370.97932958733406,"formula":"train.tokens / params.counted_active_non_embedding"},"scaling.adamw_tau":{"value":0.050331648,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":12288.0,"formula":"12288 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1610612736.0,"formula":"12288 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":12288.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":223.4181818181818,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":29283867.927272726,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":12288.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":12288.0,"tokens_per_entry":1}],"formula":"Global KV: 12 × (512 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"nemotron3-super","name":"Nemotron-3-Super-120B-A12B-Base","org":"NVIDIA","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16/resolve/b4cc202094ea3644bc4461c9fe58ec8b6a3695eb/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16/resolve/b4cc202094ea3644bc4461c9fe58ec8b6a3695eb/config.json","kind":"config"},"index":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16/resolve/b4cc202094ea3644bc4461c9fe58ec8b6a3695eb/model.safetensors.index.json","kind":"config"},"modeling_nemotron_h":{"url":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16/resolve/b4cc202094ea3644bc4461c9fe58ec8b6a3695eb/modeling_nemotron_h.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2604.12374v1","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20261019100953/https://blogs.nvidia.com/blog/nemotron-3-super-agentic-ai/","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019104702/https://artificialanalysis.ai/models/nvidia-nemotron-3-super-120b-a12b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-03-11","source":"blog","at":"blogs.nvidia.com; post header","quote":"March 11, 2026","note":"Hugging Face base repository created 2026-03-10; the card's Release Date line says 03/04/2026."},"meta.license":{"value":"other (nvidia-nemotron-open-model-license)","source":"card","at":"README.md; YAML front matter","quote":"license_name: nvidia-nemotron-open-model-license"},"meta.modality":{"value":"text","source":"card","at":"README.md; Input","quote":"Input Type(s): Text"},"params.active":{"value":12700000000,"source":"report","at":"PDF page 4; §2.1","quote":"maintaining a constrained active budget of 12.7B parameters (12.1B excluding embeddings)"},"params.active_non_embedding":{"value":12100000000,"source":"report","at":"PDF page 4; §2.1","quote":"maintaining a constrained active budget of 12.7B parameters (12.1B excluding embeddings)","note":"The 0.6B gap is close to one 131,072 × 4,096 matrix (0.54B), so the output head may still be counted."},"params.note":{"value":"Counts include embeddings (12.1B active without). MTP inclusion not stated; the checkpoint index lists 123.6B parameters with the MTP block.","source":"report","at":"PDF page 4; §2.1","quote":"maintaining a constrained active budget of 12.7B parameters (12.1B excluding embeddings)"},"params.total":{"value":120600000000,"source":"report","at":"PDF page 4; §2.1","quote":"to 120.6B total parameters"},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"config.json; hybrid_override_pattern","quote":"\"hybrid_override_pattern\": \"MEMEMEM*EMEMEMEM*EMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEM*EMEMEMEME\"","basis":"inferred","note":"One block per token mixer: 40 Mamba-2 (M) + 8 attention (*). 40 blocks are followed by a MoE FFN (E); the 8 Mamba-2 layers right before each attention layer have none. The publisher's 88 layers (Table 1, num_hidden_layers) count mixer and MoE sublayers separately."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":131072,"source":"config","at":"/vocab_size"},"attn.global_layers":{"value":8,"source":"config","at":"/hybrid_override_pattern | count *"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.heads":{"value":32,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"GQA, Mamba-2","source":"report","at":"PDF page 7; §2.1.3","quote":"The attention layers employ Grouped-Query Attention (GQA) with 32 query heads and 2 KV heads"},"attn.kv_heads":{"value":2,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"40 Mamba-2 + 8 GQA","source":"config","at":"config.json; /hybrid_override_pattern","quote":"\"hybrid_override_pattern\": \"MEMEMEM*EMEMEMEM*EMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEMEM*EMEMEMEM*EMEMEMEME\"","basis":"inferred","note":"Irregular: Mamba-2 runs before, between and after the 8 attention layers are 4, 4, 4, 5, 5, 5, 5, 4, 4. M = Mamba-2, * = GQA; E denotes a standalone MoE block and is omitted from the token-mixer diagram, matching shape.layers.","sequence":["Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2","GQA","Mamba-2","Mamba-2","Mamba-2","Mamba-2"]},"attn.linear_layers":{"value":40,"source":"config","at":"/hybrid_override_pattern | count M"},"attn.linear_kind":{"value":"Mamba-2: 128 heads × head dim 64, state 128, 8 groups, conv kernel 4","source":"report","at":"PDF page 3; Table 1","quote":"Mamba State Dimension 128 Mamba Groups 8 Mamba Heads 128 Mamba Head Dimension 64","note":"Conv kernel from config /conv_kernel (use_conv_bias true); groups = n_groups, state = ssm_state_size; mamba_hidden_act silu on x and the output gate."},"attn.linear_heads":{"value":128,"source":"config","at":"/mamba_num_heads"},"attn.linear_head_dim":{"value":64,"source":"config","at":"/mamba_head_dim"},"attn.conv":{"value":4,"source":"config","at":"/conv_kernel"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.position":{"value":"NoPE","source":"report","at":"PDF page 7; §2.1.3","quote":"Consistent with prior Nemotron models, we omit positional embeddings","note":"config rope_theta is unused; the modeling code applies no rotary embedding. Config max_position_embeddings is 1048576."},"attn.qk_norm":{"value":false,"source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHAttention.forward","quote":"query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)"},"ffn.activation":{"value":"ReLU²","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHMLP.forward","quote":"return self.down_proj(self.act_fn(self.up_proj(x)))","note":"Not gated (up → act → down); act_fn = ACT2FN[mlp_hidden_act] with config mlp_hidden_act relu2."},"ffn.bias":{"value":false,"source":"config","at":"/mlp_bias"},"moe.balancing":{"value":"aux-loss-free bias (update rate 1e-3) + load-balancing loss 1e-4","source":"report","at":"PDF page 14; §2.4","quote":"we adopted an auxiliary-loss-free load balancing strategy (Wang et al., 2024; DeepSeek-AI, 2025c) with an update rate of 10−3 , paired with a standard load balancing loss with coefficient of 10−4"},"moe.expert_dim":{"value":2688,"source":"config","at":"/moe_intermediate_size","note":"Routed experts map the 1024-wide latent to 2688 and back (LatentMoE)."},"moe.latent_dim":{"value":1024,"source":"config","at":"/moe_latent_size","note":"One 4096→1024 down-projection and 1024→4096 up-projection per MoE layer wrap the routed experts; the router and shared expert act on the 4096-wide hidden state (§2.1.1; NemotronHMoE.forward)."},"moe.experts":{"value":512,"source":"config","at":"/n_routed_experts"},"moe.router":{"value":"sigmoid","source":"report","at":"PDF page 14; §2.4","quote":"We utilized a sigmoid router score function complemented by expert biasing.","note":"Config n_group = topk_group = 1: no group-limited routing."},"moe.dense_layers":{"value":0,"source":"config","at":"/hybrid_override_pattern | count -"},"moe.routed_scale":{"value":5.0,"source":"config","at":"/routed_scaling_factor"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.shared_dim":{"value":5376,"source":"config","at":"/moe_shared_expert_intermediate_size"},"moe.top_k":{"value":22,"source":"config","at":"/num_experts_per_tok"},"mtp.depth":{"value":2,"source":"report","at":"PDF page 3; Table 1","quote":"MTP layers (shared weight) 2","basis":"inferred","note":"One attention+MoE MTP block applied at two offsets with shared weights (§2.1.2: one head exposed to multiple offsets). The checkpoint stores one block (num_nextn_predict_layers 1)."},"mtp.modules":{"value":1,"source":"report","at":"PDF page 6; §2.1.2","quote":"sharing parameters across multiple MTP heads during training, yielding a unified prediction head"},"mtp.block":{"value":"next-token embedding and hidden state each normed (enorm, hnorm), concatenated and projected 2×4096→4096 (eh_proj); then one GQA attention layer and one LatentMoE layer; final norm","source":"config","at":"config.json; mtp_hybrid_override_pattern","quote":"\"mtp_hybrid_override_pattern\": \"*E\"","note":"Projection and norms from mtp.layers.0.{enorm,hnorm,eh_proj} in model.safetensors.index.json; eh_proj shape [4096, 8192] in the safetensors header."},"mtp.ffn":{"value":"MoE","source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"mtp.layers.1.mixer.fc1_latent_proj.weight\"","note":"LatentMoE with the backbone's widths: expert up_proj [2688, 1024], latent proj [1024, 4096], shared expert [5376, 4096] (safetensors header)."},"mtp.shares_embedding":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"lm_head.weight\"","basis":"inferred","note":"The MTP tensors (mtp.*) contain no embedding or output head; the only ones are backbone.embeddings and lm_head."},"mtp.loss_weight":{"value":"0.3","source":"report","at":"PDF page 14; §2.4","quote":"we used an MTP objective with loss scaling factor of 0.3."},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"mtp.layers.0.eh_proj.weight\""},"norm.placement":{"value":"pre","source":"modeling_nemotron_h","at":"modeling_nemotron_h.py; NemotronHBlock docstring","quote":"Each block applies pre-normalization followed by the mixer, then adds a residual connection."},"norm.eps":{"value":1e-05,"source":"config","at":"/layer_norm_epsilon"},"norm.type":{"value":"RMSNorm","source":"report","at":"PDF page 7; §2.1.3","quote":"use RMSNorm for normalization"},"ctx.extension":{"value":"Long-context CPT at constant LR 4.5e-6: 34B tokens at 1M, then 17B tokens alternating 1M and 4K sequences","source":"report","at":"PDF page 16; §2.6","quote":"We initially performed CPT on 1,048,576 (1m) context length."},"ctx.final":{"value":1048576,"source":"report","at":"PDF page 16; §2.6","quote":"We initially performed CPT on 1,048,576 (1m) context length."},"ctx.max":{"value":1048576,"source":"report","at":"PDF page 7; §2.1.3","quote":"This configuration supports context lengths of up to 1M tokens."},"ctx.pretrain":{"value":8192,"source":"report","at":"PDF page 14; §2.4","quote":"The model was trained with a sequence length of 8,192"},"train.batch":{"value":"3,072 × 8,192 (25.17M tokens)","source":"report","at":"PDF page 14; §2.4","quote":"batch size of 3,072 sequences, resulting in approximately 25.17 million tokens per batch."},"train.batch_tokens":{"value":25165824,"source":"report","at":"PDF page 15; §2.5","quote":"our global batch size of 3,072×8,192 tokens","basis":"inferred","note":"3,072 × 8,192 = 25,165,824; no batch ramp stated."},"train.betas":{"value":"(0.9, 0.95)","source":"report","at":"PDF page 14; §2.4","quote":"momentum coefficients β1 = 0.9 and β2 = 0.95"},"train.data":{"value":"Nemotron 3 Nano mixture, 16 categories: web crawl in 5 quality tiers (incl. synthetic rephrasings), math, Wikipedia, code, academic, multilingual, finepdfs, SFT-style incl. reasoning; phase 1 diversity-focused, phase 2 (from 80% of tokens) high-quality","source":"report","at":"PDF page 13; §2.3.7","quote":"Our pretraining corpus spans 16 high-level categories."},"train.cutoff":{"value":"December 2025","source":"card","at":"README.md; Model Overview, Data Freshness","quote":"The pre-training data has a cutoff date of December 2025."},"train.hardware":{"value":"GB200","source":"report","at":"PDF page 16; §2.6","quote":"context parallelism, 2-way tensor parallelism, and 64-way expert parallelism to train on GB200 GPUs.","note":"Stated only for the long-context phase."},"train.averaging":{"value":"Final base checkpoint is a 500B-token merge (weighted average over a sliding window of checkpoints, minus-sqrt decay-emulation coefficients)","source":"report","at":"PDF page 15; §2.5","quote":"The final base model checkpoint selected for downstream alignment was itself a 500B merge","note":"Ordering relative to the long-context phase is not stated."},"train.lr_min":{"value":4.5e-06,"source":"report","at":"PDF page 14; §2.4","quote":"annealing the LR to a minimum of 4.5 × 10−6 ."},"train.lr_peak":{"value":0.00045,"source":"report","at":"PDF page 14; §2.4","quote":"The learning rate (LR) was warmed up over the initial 200 billion tokens to a peak value of 4.5 × 10−4 ."},"train.optimizer":{"value":"AdamW","source":"report","at":"PDF page 14; §2.4","quote":"We used AdamW (Loshchilov & Hutter, 2017) optimizer with a weight decay of 0.1"},"train.precision":{"value":"NVFP4 (E2M1, 16-element blocks, E4M3 block scales + FP32 global scale; 2D weight blocks; RHT on wgrad inputs; stochastic rounding on gradients). BF16 for the final 15% of layers, latent projections, MTP, QKV/attention projections and embeddings; MXFP8 for the Mamba output projection.","source":"report","at":"PDF page 8; §2.2, Table 3","quote":"The NVFP4 format utilizes an E2M1 element format with 16-element micro-blocks, E4M3 micro-block scaling factors, and a second-level FP32 global scale."},"train.schedule":{"value":"WSD: 200B-token warmup, stable, minus-sqrt decay over the final 5T tokens","source":"report","at":"PDF page 14; §2.4","quote":"we implemented a minus-sqrt decay schedule for the final 5 trillion tokens"},"train.stages":{"value":[{"name":"phase-1","tokens":20000000000000,"seq_len":8192,"batch_sequences":3072,"note":"Diversity-focused blend; warmup and stable LR.","quote":"The first phase accounted for 80% of pre-training (20 trillion tokens)"},{"name":"phase-2","tokens":5000000000000,"seq_len":8192,"batch_sequences":3072,"note":"High-quality blend; spans the minus-sqrt LR decay.","quote":"the second phase accounted for 20% of pre-training (5 trillion tokens)"},{"name":"long-context-1","tokens":34000000000,"seq_len":1048576,"lr":4.5e-06,"batch_sequences":16,"note":"20% long-context document QA, 80% downscaled phase-2 data.","quote":"Such stage lasted for 34 billion tokens."},{"name":"long-context-2","tokens":17000000000,"seq_len":1048576,"lr":4.5e-06,"batch_sequences":16,"note":"Alternates 1M and 4K sequences.","quote":"The second stage lasted for 17 billion tokens."}],"source":"report","at":"PDF pages 1, 14, 16; §1, §2.4, §2.6"},"train.tokens":{"value":25000000000000,"source":"report","at":"PDF page 14; §2.4","quote":"learning rate schedule over a total horizon of 25 trillion tokens","note":"Excludes the 51B-token long-context phase."},"train.warmup":{"value":"200B tokens","source":"report","at":"PDF page 14; §2.4","quote":"warmed up over the initial 200 billion tokens"},"train.weight_decay":{"value":0.1,"source":"report","at":"PDF page 14; §2.4","quote":"optimizer with a weight decay of 0.1"},"eval.aa_index":{"value":13,"source":"aa","at":"Artificial Analysis model page /models/nvidia-nemotron-3-super-120b-a12b, Wayback snapshot 2026-10-19","quote":"Nemotron 3 Super 120B A12B (Reasoning) scores 13 on the Artificial Analysis Intelligence Index","note":"Score of NVIDIA-Nemotron-3-Super-120B-A12B, post-trained from this base. Variant: Nemotron 3 Super 120B A12B (Reasoning). Index v4.3.2."}},"not_disclosed":["train.grad_clip"],"derived":{"params.counted_total":{"value":120668707840,"formula":"embedding + output head + all experts + other weights, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16@b4cc202 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1073741824,"formula":"input embedding + output head, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16@b4cc202 safetensors headers"},"params.counted_non_embedding":{"value":119594966016,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16@b4cc202 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":2873102848,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16@b4cc202 safetensors headers"},"params.counted_active_non_embedding":{"value":11696495616.0,"formula":"non-expert weights + routed experts × 22/512, excluding embedding and output head, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16@b4cc202 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":85.33333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.65625,"formula":"moe.expert_dim / d_model"},"moe.latent_ratio":{"value":4.0,"formula":"d_model / moe.latent_dim"},"attn.linear_ratio":{"value":5.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":23.272727272727273,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.10530679933665009,"formula":"params.active / params.total"},"scaling.tpp":{"value":207.29684908789386,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":2137.3923285023698,"formula":"train.tokens / params.counted_active_non_embedding"},"scaling.adamw_tau":{"value":0.022369621333333332,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":8192.0,"formula":"8192 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":1073741824.0,"formula":"8192 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":8192.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":645.0393700787401,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":84546600.31496063,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":8192.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":8192.0,"tokens_per_entry":1}],"formula":"Global KV: 8 × (512 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"olmo-3-32b","name":"Olmo-3-1125-32B","org":"Ai2","country":"United States","kind":"base","weights":"open","note":null,"sources":{"config":{"url":"https://huggingface.co/allenai/Olmo-3-1125-32B/resolve/c2b61dae89a1ad10e4ad5653d0e46b590902607b/config.json","kind":"config"},"report":{"url":"https://arxiv.org/pdf/2512.13961v2","kind":"paper"},"pretrain_script":{"url":"https://raw.githubusercontent.com/allenai/OLMo-core/5f6f58a133e7ef577d596295f2c8db4651c27857/src/scripts/official/OLMo3/OLMo-3-1025-32B-pretrain.py","kind":"code"},"modeling_olmo3":{"url":"https://raw.githubusercontent.com/huggingface/transformers/8cb5963cc22174954e7dca2c0a3320b7dc2f4edc/src/transformers/models/olmo3/modeling_olmo3.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20251120152225/https://allenai.org/blog/olmo3","kind":"blog"},"card":{"url":"https://huggingface.co/allenai/Olmo-3-1125-32B/resolve/c2b61dae89a1ad10e4ad5653d0e46b590902607b/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019104913/https://artificialanalysis.ai/models/olmo-3-32b-think","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-11-20","source":"blog","at":"Olmo 3 announcement; header date","quote":"November 20, 2025"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; Model Description","quote":"Model type: a Transformer style autoregressive language model.","note":"Language model with no vision or audio components (Olmo3ForCausalLM)."},"params.total":{"value":32000000000,"source":"report","at":"p.1 abstract","quote":"We introduce Olmo 3, a family of state-of-the-art, fully-open language models at the 7B and 32B","note":"Size designation; no exact parameter count is reported."},"params.active":{"value":32000000000,"source":"report","at":"p.1 abstract","quote":"We introduce Olmo 3, a family of state-of-the-art, fully-open language models at the 7B and 32B","note":"Dense model; all parameters active."},"shape.d_model":{"value":5120,"source":"config","at":"/hidden_size"},"shape.layers":{"value":64,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":100278,"source":"config","at":"/vocab_size","note":"Training script pads the embedding to a multiple of 128 (100,352 rows); the released checkpoint is trimmed to 100,278."},"shape.tokenizer":{"value":"OLMo 2 tokenizer, derived from OpenAI cl100k","source":"report","at":"p.10 §3.2 Tokenizer","quote":"We process data for each stage using the same tokenizer as OLMo 2, which is derived from OpenAI"},"attn.global_layers":{"value":16,"source":"config","at":"/layer_types | count full_attention"},"attn.local_layers":{"value":48,"source":"config","at":"/layer_types | count sliding_attention"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.9 §3.2; Table 33 caption p.85","quote":"the 32B model uses grouped-query attention (Ainslie et al., 2023) for increased efficiency."},"attn.layout":{"value":"3 SWA : 1 global (last layer global)","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Ordered 64-layer backbone from /layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.window":{"value":4096,"source":"config","at":"/sliding_window"},"attn.local_heads":{"value":40,"source":"config","at":"/num_attention_heads","note":"Olmo3Attention reads the same num_attention_heads, num_key_value_heads and head_dim for both layer types; only sliding_window and the rotary embedding differ."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"modeling_olmo3","at":"modeling_olmo3.py; Olmo3Attention.__init__","quote":"self.head_dim = getattr(config, \"head_dim\", config.hidden_size // config.num_attention_heads)","basis":"inferred","note":"Same Olmo3Attention class for both layer types; hidden_size 5120 / 40 heads = 128."},"attn.local_rope_theta":{"value":500000,"source":"config","at":"/rope_theta","note":"Sliding layers use Olmo3RotaryEmbedding(config, rope_type=\"default\"), i.e. rope_theta without YaRN."},"attn.heads":{"value":40,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"modeling_olmo3","at":"modeling_olmo3.py line 146; Olmo3Attention.__init__","quote":"self.head_dim = getattr(config, \"head_dim\", config.hidden_size // config.num_attention_heads)","basis":"inferred","note":"config.json has no head_dim, so hidden_size 5120 / 40 heads = 128."},"attn.position":{"value":"RoPE","source":"report","at":"p.85 Table 33; §3.6.4","quote":"RoPE scaling YaRN on full attn. layers"},"attn.rope_theta":{"value":500000,"source":"config","at":"/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.qk_norm":{"value":true,"source":"modeling_olmo3","at":"modeling_olmo3.py lines 164–184; Olmo3Attention","quote":"self.q_norm = Olmo3RMSNorm(config.num_attention_heads * self.head_dim, config.rms_norm_eps)","note":"RMSNorm over the full Q and K projections (all heads jointly), not per head; paper Table 33 lists QK-Norm."},"ffn.activation":{"value":"SwiGLU","source":"report","at":"p.85 Table 33","quote":"Activation SwiGLU","note":"Config hidden_act silu; Olmo3MLP computes down_proj(act_fn(gate_proj(x)) * up_proj(x))."},"ffn.dim":{"value":27648,"source":"config","at":"/intermediate_size"},"ffn.bias":{"value":false,"source":"modeling_olmo3","at":"modeling_olmo3.py; Olmo3MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"norm.type":{"value":"RMSNorm","source":"report","at":"p.85 Table 33","quote":"Layer norm RMSNorm"},"norm.placement":{"value":"post (on sublayer outputs, inside the residual branch)","source":"report","at":"p.85 Table 33","quote":"Layer norm applied to Outputs","note":"OLMo 2-style reordered norm: residual + norm(sublayer(x)); see Olmo3DecoderLayer."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.pretrain":{"value":8192,"source":"report","at":"p.9 §3.2 Architecture","quote":"We train with a context window of 8192 tokens (increased from 4096 tokens for OLMo 2) during pretraining and midtraining stages."},"ctx.final":{"value":65536,"source":"report","at":"p.30 §3.6","quote":"the methodology we used to scale Olmo 3’s context window from 8,192 to 65,536 tokens."},"ctx.max":{"value":65536,"source":"report","at":"p.5 §2.1","quote":"Olmo 3 is our first model with long-context capabilities, supporting up to 65K context after extension.","note":"Config max_position_embeddings 65536."},"ctx.extension":{"value":"YaRN on full-attention layers only (SWA RoPE unchanged); 8K → 65K over 100B tokens after midtraining, 34% long / 66% midtraining data, best-fit packing, intra-document masking","source":"report","at":"p.30 §3.6","quote":"During long-context extension, we apply YaRN (Peng et al., 2023) to full attention layers, and do not adjust positional embeddings on sliding-window attention layers","note":"Released config rope_scaling: rope_type yarn, factor 8, original_max_position_embeddings 8192, attention_factor 1.2079441541679836, beta_fast 32, beta_slow 1."},"mtp.depth":{"value":0,"source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"model_config = TransformerConfig.olmo3_32B(","note":"Official pretraining config builds a plain decoder with next-token loss; OLMo-core at this commit has no MTP module."},"train.tokens":{"value":5700000000000,"source":"report","at":"p.85 Table 35","quote":"Total training tokens 5.5T 100B (twice) 100B","basis":"inferred","note":"5.5T pretraining + 100B midtraining (two seeds, averaged) + 100B long-context along the released model's lineage; Table 13 lists 6.2T for 32B Stage 3."},"train.optimizer":{"value":"AdamW (OLMo-core SkipStepAdamW)","source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"optim=SkipStepAdamWConfig("},"train.lr_peak":{"value":0.0006,"source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; module constants","quote":"LR = 6e-4","note":"Paper Table 35: 6.0 × 10^-4."},"train.lr_min":{"value":6.21e-05,"source":"report","at":"p.10 Figure 4 caption","quote":"Due to the truncation, the real final learning rate is 6.210 × 10","note":"End of stage-1 pretraining (exponent −5 is a superscript in the PDF). Midtraining and long-context stages each decay linearly from 2.071e-4 to 0 (Table 35)."},"train.schedule":{"value":"2000-step warmup; cosine over one epoch (5.93T) targeting 10% of peak, truncated at 5.5T; midtraining and long-context: linear decay 2.071e-4 → 0","source":"report","at":"p.10 Figure 4; p.85 Table 35","quote":"The learning rate schedule is a cosine schedule over one epoch (5.93T tokens), truncated at 5.5T tokens."},"train.warmup":{"value":"2000 steps","source":"report","at":"p.10 Figure 4 caption","quote":"Warm-up is 2000 steps, and the peak learning rate is 6 × 10","note":"Long-context stage has its own 200-step warmup; midtraining none (Table 35)."},"train.batch":{"value":"1,024 × 8,192 (8.39M tokens) pretraining; 512 × 8,192 (4.19M tokens) midtraining; 128 × 65,536 (8.39M tokens) long-context","source":"report","at":"p.85 Table 35","quote":"Batch size (# tokens) 8,388,608 4,194,304 8,388,608"},"train.batch_tokens":{"value":8388608,"source":"report","at":"p.85 Table 35, Olmo 3 Base 32B pretraining column","quote":"Batch size (# tokens) 8,388,608 4,194,304 8,388,608","note":"1,024 instances × 8,192 tokens; no ramp."},"train.weight_decay":{"value":0.1,"source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"weight_decay=0.1,","note":"Embeddings excluded (weight_decay=0.0 override; paper Table 33: no weight decay on embeddings)."},"train.betas":{"value":"(0.9, 0.95)","source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"betas=(0.9, 0.95),"},"train.grad_clip":{"value":1.0,"source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"max_grad_norm=1.0,"},"train.precision":{"value":"BF16","source":"report","at":"p.9 §3.2 Training","quote":"using bfloat16 precision throughout.","note":"Pretraining script sets Float8Config(enabled=False); FSDP params bf16, reductions fp32."},"train.stability":{"value":"z-loss 1e-5; QK-norm; no weight decay on embeddings","source":"pretrain_script","at":"OLMo-3-1025-32B-pretrain.py; build_config","quote":"z_loss_multiplier=1e-5,","note":"Paper Table 33 lists the same z-loss weight, QK-Norm and no embedding weight decay."},"train.masking":{"value":"intra-document masking during long-context extension","source":"report","at":"p.35 §3.6.3","quote":"During long-context extension, we apply intra-document masking to ensure that each training sequence attends only to tokens originating from the same underlying document","note":"Not stated for pretraining or midtraining."},"train.data":{"value":"Dolma 3 Mix, 5.93T: Common Crawl 76.1%, olmOCR science PDFs 13.6%, Stack-Edu code 6.89%, FineMath 3+ 2.56%, arXiv 0.86%, Wikipedia/Wikibooks 0.04%; upsampled from a 9.31T pool","source":"report","at":"p.14 §3.4, Table 4","quote":"Common Crawl Web pages 8.14T 9.67B 4.51T (76.1%)","note":"The 32B run stopped at 5.5T of this one-epoch mix."},"train.cutoff":{"value":"December 2024","source":"report","at":"p.15 §3.4.1 Text extraction","quote":"We start with 104 dumps from the CommonCrawl corpus, with a cutoff date of December 31, 2024.","note":"olmOCR science PDFs also cut off December 2024; smaller sources earlier."},"train.hardware":{"value":"H100 80GB; pretraining 512 GPUs for ~9.5 days then 1024 for ~35 days; midtraining 2 × 512; long-context 1024","source":"report","at":"p.7 §2 runtime breakdown; p.85 Table 34","quote":"The initial pretraining phase on 5.5T tokens took about 9.5 days on 512 GPUs, followed by an additional 35 days on 1024 GPUs.","note":"Table 34: 8× NVIDIA H100 (80GB HBM3) nodes; durations include crash resumptions."},"train.accelerators":{"value":"1,024 H100","source":"report","at":"p.7 §2 runtime breakdown","quote":"The initial pretraining phase on 5.5T tokens took about 9.5 days on 512 GPUs, followed by an additional 35 days on 1024 GPUs.","note":"First ~9.5 days on 512 GPUs; Table 34 (p.85) gives 8× NVIDIA H100 (80GB HBM3) nodes."},"train.parallelism":{"value":"HSDP (DP-shard 64 · DP-rep 16)","source":"report","at":"p.85 Table 34; Olmo 3 Base 32B, Pretraining column","quote":"Olmo 3 Base 32B Pretraining Midtraining Long-context ext DP-rep 16 8 16 DP-shard 64 64 8"},"train.averaging":{"value":"midtraining: weights of two 100B runs with different data-order seeds averaged; long-context: checkpoints at steps 10,000, 11,000, 11,921 averaged","source":"report","at":"p.35 §3.6.3 Model souping; p.85 Table 35","quote":"we merge the last three checkpoints from the end of the extension run (at steps 10,000, 11,000, and 11,921) to produce our final long-context Olmo 3 Base 32B."},"train.stages":{"value":[{"name":"pretraining","tokens":5500000000000,"seq_len":8192,"lr":0.0006,"batch_tokens":8388608,"note":"Dolma 3 Mix; cosine to 5.93T truncated at 5.5T.","quote":"The initial pretraining phase on 5.5T tokens took about 9.5 days on 512 GPUs, followed by an additional 35 days on 1024 GPUs."},{"name":"midtraining","tokens":100000000000,"seq_len":8192,"lr":0.0002071,"batch_tokens":4194304,"note":"Dolma 3 Dolmino Mix; no warmup, linear decay to 0; two runs with different data-order seeds, weights averaged.","quote":"Midtraining consisted of two parallel runs on 512 GPUs each, covering 100B tokens per run"},{"name":"long-context","tokens":100000000000,"seq_len":65536,"lr":0.0002071,"batch_tokens":8388608,"steps":11921,"note":"Dolma 3 Longmino Mix; 200-step warmup, linear decay to 0; final model averages checkpoints at steps 10,000, 11,000, 11,921.","quote":"Long-context extension was executed as a single run on 1024 GPUs"}],"source":"report","at":"p.3 §2; p.30–35 §3.5–3.6; p.85 Table 35"},"scaling.ladder":{"value":"Data ablations up to 1B params at 100B tokens; mix swarms of 30M models at 3B tokens (5× Chinchilla)","source":"report","at":"p.11 §3.3; p.18 §3.4 swarm construction","quote":"up to our largest experiment scale of 1B parameters at 100B tokens"},"eval.aa_index":{"value":6,"source":"aa","at":"Artificial Analysis model page /models/olmo-3-32b-think, Wayback snapshot 2026-10-19","quote":"Olmo 3 32B Think scores 6 on the Artificial Analysis Intelligence Index","note":"Score of Olmo-3-32B-Think, post-trained from this base. Variant: Olmo 3 32B Think. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding"],"derived":{"params.counted_total":{"value":32233522176,"formula":"embedding + output head + all experts + other weights, allenai/Olmo-3-1125-32B@c2b61da safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1026846720,"formula":"input embedding + output head, allenai/Olmo-3-1125-32B@c2b61da safetensors headers"},"params.counted_non_embedding":{"value":31206675456,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, allenai/Olmo-3-1125-32B@c2b61da safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":487604224,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, allenai/Olmo-3-1125-32B@c2b61da safetensors headers"},"params.counted_active_non_embedding":{"value":31206675456.0,"formula":"all weights except embedding and output head, allenai/Olmo-3-1125-32B@c2b61da safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":80.0,"formula":"d_model / layers"},"attn.qk_width":{"value":5120,"formula":"heads × head_dim"},"attn.group_size":{"value":5.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":5.4,"formula":"ffn.dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"scaling.tpp":{"value":178.125,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":182.65322776970402,"formula":"train.tokens / params.counted_active_non_embedding"},"scaling.adamw_tau":{"value":0.024528093567251465,"formula":"train.batch_tokens / (train.lr_peak × train.weight_decay × train.tokens)"},"kv.growth_per_token":{"value":65536.0,"formula":"65536 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9395240960.0,"formula":"65536 B × 131072 (Global KV) + 196608 B × min(131072, 4096) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":71680.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2240.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":293601280.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":65536.0,"windows":[[196608.0,4096]],"groups":[{"name":"Global KV","bytes_per_entry":65536.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":196608.0,"tokens_per_entry":1,"window":4096}],"formula":"Global KV: 16 × (2048 × 16/8) B/entry; SWA KV: 48 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen3-next","name":"Qwen3-Next-80B-A3B-Instruct","org":"Qwen","country":"China","kind":"instruct","weights":"open","note":"Describes the Instruct checkpoint; the card gives the pretraining token count of the shared base.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/resolve/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/resolve/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","kind":"config"},"transformers-code":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_next/modeling_qwen3_next.py","kind":"code"},"hf-config-code":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_next/configuration_qwen3_next.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/resolve/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/model.safetensors.index.json","kind":"config"},"blog":{"url":"https://web.archive.org/web/20260826125316id_/https://docs.qwenlm.ai/research/qwen3-next/index.json","kind":"blog"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019105058/https://artificialanalysis.ai/models/qwen3-next-80b-a3b-instruct","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2025-09-11","source":"github","at":"QwenLM GitHub README; News","quote":"2025-09-11: We release Qwen3-Next-80B-A3B"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; YAML front matter","quote":"pipeline_tag: text-generation","note":"Text-generation pipeline tag; Qwen3NextForCausalLM config has no vision components."},"params.active":{"value":3000000000,"source":"card","at":"README.md; Model Overview","quote":"80B in total and 3B activated"},"params.note":{"value":"Rounded card figures. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Number of Paramaters (Non-Embedding): 79B"},"params.total":{"value":80000000000,"source":"card","at":"README.md; Model Overview","quote":"80B in total and 3B activated"},"params.non_embedding":{"value":79000000000,"source":"card","at":"README.md; Model Overview","quote":"Number of Paramaters (Non-Embedding): 79B"},"shape.d_model":{"value":2048,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":151936,"source":"config","at":"/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)"},"attn.global_layers":{"value":12,"source":"card","at":"README.md; Model Overview; Hybrid Layout","quote":"12 \\* (3 \\* (Gated DeltaNet -> MoE) -> 1 \\* (Gated Attention -> MoE))"},"attn.head_dim":{"value":256,"source":"config","at":"/head_dim"},"attn.heads":{"value":16,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Highlights","quote":"Replaces standard attention with the combination of Gated DeltaNet and Gated Attention","note":"Gated Attention is GQA with 16 Q / 2 KV heads."},"attn.kv_heads":{"value":2,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"hf-config-code","at":"configuration_qwen3_next.py; Qwen3NextConfig.__post_init__, lines 132–137","quote":"\"linear_attention\" if bool((i + 1) % interval_pattern) else \"full_attention\"","note":"Config full_attention_interval 4; layer_types is derived from it in configuration_qwen3_next.py. The pinned config omits layer_types and full_attention_interval; Qwen3NextConfig defaults the interval to 4.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.bias":{"value":false,"source":"hf-config-code","at":"configuration_qwen3_next.py; Qwen3NextConfig defaults","quote":"attention_bias: bool = False","note":"config.json does not set attention_bias; the class default applies to q/k/v/o projections."},"attn.linear_layers":{"value":36,"source":"card","at":"README.md; Model Overview; Hybrid Layout","quote":"12 \\* (3 \\* (Gated DeltaNet -> MoE) -> 1 \\* (Gated Attention -> MoE))","basis":"inferred","note":"12 repeats × 3 Gated DeltaNet layers."},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated."},"attn.qk_norm":{"value":true,"source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextAttention.__init__","quote":"self.q_norm = Qwen3NextRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_theta"},"attn.conv":{"value":4,"source":"config","at":"/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/linear_value_head_dim"},"attn.linear_heads":{"value":32,"source":"config","at":"/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 32 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 32 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from linear_conv_kernel_dim; SiLU output gate from Qwen3NextRMSNormGated in modeling_qwen3_next.py."},"ffn.activation":{"value":"SwiGLU","source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"moe.expert_dim":{"value":512,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/num_experts"},"moe.router":{"value":"softmax","source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture of Experts","quote":"Number of Shared Experts: 1"},"moe.shared_dim":{"value":512,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":10,"source":"config","at":"/num_experts_per_tok"},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hybrid Layout","quote":"12 \\* (3 \\* (Gated DeltaNet -> MoE) -> 1 \\* (Gated Attention -> MoE))","note":"Config decoder_sparse_step 1 and empty mlp_only_layers: every layer is MoE."},"moe.balancing":{"value":"global-batch load-balancing loss","source":"blog","at":"Qwen3-Next blog; Key Features; Ultra-Sparse MoE","quote":"Experiments show that, with global load balancing","note":"Cites 'Demons in the Detail' (global-batch balancing); coefficient not stated."},"mtp.block":{"value":"one decoder layer (gated attention + MoE); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, MoE mlp), norm; no mtp.layers.1. Card: MTP trained with multi-step training."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"transformers-code","at":"modeling_qwen3_next.py; Qwen3NextDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"card","at":"README.md; Highlights; Stability Optimizations","quote":"zero-centered and weight-decayed layernorm","note":"Card: zero-centered gains with weight decay; code applies 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN, factor 4."},"train.tokens":{"value":15000000000000,"source":"card","at":"README.md; Model Overview","quote":"Pretraining (15T tokens)"},"train.data":{"value":"uniformly sampled 15T-token subset of the Qwen3 36T-token corpus","source":"blog","at":"Qwen3-Next blog; Pre-training","quote":"Qwen3-Next is trained on a uniformly sampled subset (15T tokens) of Qwen3"},"train.compute":{"value":"<80% of Qwen3-30B-A3B GPU hours; 9.3% of Qwen3-32B compute cost","source":"blog","at":"Qwen3-Next blog; Pre-training","quote":"It uses less than 80% of the GPU hours needed by Qwen3-30A-3B, and only 9.3% of the compute cost of Qwen3-32B"},"train.init":{"value":"MoE router parameters normalized at initialization","source":"blog","at":"Qwen3-Next blog; Key Features; Training-Stability-Friendly Designs","quote":"We also normalize MoE router parameters during initialization"},"train.stability":{"value":"zero-centered, weight-decayed norm gains","source":"card","at":"README.md; Highlights; Stability Optimizations","quote":"zero-centered and weight-decayed layernorm","note":"Card also cites unspecified other stabilizing enhancements."},"eval.aa_index":{"value":10,"source":"aa","at":"Artificial Analysis model page /models/qwen3-next-80b-a3b-instruct, Wayback snapshot 2026-10-19","quote":"Qwen3 Next 80B A3B Instruct scores 10 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3 Next 80B A3B Instruct. Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","ctx.pretrain","ctx.final","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":79674391296,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3-Next-80B-A3B-Instruct@9c7f2fb safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":622329856,"formula":"input embedding + output head, Qwen/Qwen3-Next-80B-A3B-Instruct@9c7f2fb safetensors headers"},"params.counted_non_embedding":{"value":79052061440,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3-Next-80B-A3B-Instruct@9c7f2fb safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":1648531648,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3-Next-80B-A3B-Instruct@9c7f2fb safetensors headers"},"params.counted_active_non_embedding":{"value":3252599552.0,"formula":"non-expert weights + routed experts × 10/512, excluding embedding and output head, Qwen/Qwen3-Next-80B-A3B-Instruct@9c7f2fb safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":42.666666666666664,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":51.2,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.0375,"formula":"params.active / params.total"},"scaling.tpp":{"value":187.5,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":4611.695894373658,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":24576.0,"formula":"24576 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3221225472.0,"formula":"24576 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":24576.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":8192.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":1073741824.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":24576.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":24576.0,"tokens_per_entry":1}],"formula":"Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen35-122b","name":"Qwen3.5-122B-A10B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.5-122B-A10B/resolve/dc4d348443bc740c68e2d77492492c11606384d5/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.5-122B-A10B/resolve/dc4d348443bc740c68e2d77492492c11606384d5/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.5-122B-A10B/resolve/dc4d348443bc740c68e2d77492492c11606384d5/model.safetensors.index.json","kind":"config"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"flash_next_report":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/69885871a64393807d988b27b1b5e380e8f28526/tech_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019105242/https://artificialanalysis.ai/models/qwen3-5-122b-a10b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-02-24","source":"github","at":"QwenLM GitHub README; News","quote":"2026-02-24: Qwen3.5-122B-A10B, Qwen3.5-35B-A3B, and Qwen3.5-27B are released"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder","note":"Card's quickstart gives text, image and video input examples; config defines image and video tokens."},"params.active":{"value":10000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"122B in total and 10B activated"},"params.note":{"value":"Rounded card figure, listed under the Language Model block; the vision encoder is separate. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder"},"params.total":{"value":122000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"122B in total and 10B activated"},"shape.d_model":{"value":3072,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":12,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"12 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Gated Attention is GQA with 32 Q / 2 KV heads."},"attn.kv_heads":{"value":2,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 48-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":36,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.__init__","quote":"self.q_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":64,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 64 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 64 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5MoeRMSNormGated in modeling_qwen3_5_moe.py."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"moe.expert_dim":{"value":1024,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/text_config/num_experts"},"moe.router":{"value":"softmax","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture of Experts","quote":"8 Routed + 1 Shared"},"moe.shared_dim":{"value":1024,"source":"config","at":"/text_config/shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"12 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Every layer is MoE; Qwen3_5Moe code has no dense-layer option."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_top_value /= router_top_value.sum(dim=-1, keepdim=True)","note":"Unconditional in this code path."},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multi-steps."},"mtp.block":{"value":"one decoder layer (gated attention + MoE); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeRMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"train.objective":{"value":"early-fusion training on vision-language tokens","source":"card","at":"README.md; Qwen3.5 Highlights","quote":"Early fusion training on multimodal tokens"},"scaling.hp_rule":{"value":"LR and batch from a fitted hyperparameter scaling law (coefficients not published)","source":"flash_next_report","at":"Qwen3.8-Flash-Next report p.3 §1; p.16 §3.2","quote":"so we refit the scaling law (Kaplan et al., 2020) used for the Qwen3.5 series","note":"§3.2 of the same report: the Qwen scaling laws cover learning rate and batch size."},"eval.aa_index":{"value":16,"source":"aa","at":"Artificial Analysis model page /models/qwen3-5-122b-a10b, Wayback snapshot 2026-10-19","quote":"Qwen3.5 122B A10B (Reasoning) scores 16 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.5 122B A10B (Reasoning). Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","moe.balancing","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":122111526912,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.5-122B-A10B@dc4d348 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1525678080,"formula":"input embedding + output head, Qwen/Qwen3.5-122B-A10B@dc4d348 safetensors headers"},"params.counted_non_embedding":{"value":120585848832,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.5-122B-A10B@dc4d348 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":2514674944,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.5-122B-A10B@dc4d348 safetensors headers"},"params.counted_active_non_embedding":{"value":8245610496.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, Qwen/Qwen3.5-122B-A10B@dc4d348 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":64.0,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.3333333333333333,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.08196721311475409,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":24576.0,"formula":"24576 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3221225472.0,"formula":"24576 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":24576.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2457.6,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":322122547.2,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":24576.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":24576.0,"tokens_per_entry":1}],"formula":"Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen35-27b","name":"Qwen3.5-27B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/fc05daec18b0a78c049392ed2e771dde82bdf654/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/fc05daec18b0a78c049392ed2e771dde82bdf654/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5/modeling_qwen3_5.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/fc05daec18b0a78c049392ed2e771dde82bdf654/model.safetensors.index.json","kind":"config"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"flash_next_report":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/69885871a64393807d988b27b1b5e380e8f28526/tech_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019105412/https://artificialanalysis.ai/models/qwen3-5-27b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-02-24","source":"github","at":"QwenLM GitHub README; News","quote":"2026-02-24: Qwen3.5-122B-A10B, Qwen3.5-35B-A3B, and Qwen3.5-27B are released"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder","note":"Card's quickstart gives text, image and video input examples; config defines image and video tokens."},"params.active":{"value":27000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Parameters: 27B","note":"Dense model; all parameters active."},"params.note":{"value":"Rounded card figure, listed under the Language Model block; the vision encoder is separate. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder"},"params.total":{"value":27000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Parameters: 27B"},"shape.d_model":{"value":5120,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":64,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5Attention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":16,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":24,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"16 × (3 × (Gated DeltaNet → FFN) → 1 × (Gated Attention → FFN))","note":"Gated Attention is GQA with 24 Q / 4 KV heads."},"attn.kv_heads":{"value":4,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 64-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":48,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5Attention.__init__","quote":"self.q_norm = Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":48,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 48 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 48 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5RMSNormGated in modeling_qwen3_5.py."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.dim":{"value":17408,"source":"config","at":"/text_config/intermediate_size"},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multi-steps."},"mtp.block":{"value":"one decoder layer (gated attention + dense SwiGLU FFN); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"dense","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.gate_proj.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5RMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"train.objective":{"value":"early-fusion training on vision-language tokens","source":"card","at":"README.md; Qwen3.5 Highlights","quote":"Early fusion training on multimodal tokens"},"scaling.hp_rule":{"value":"LR and batch from a fitted hyperparameter scaling law (coefficients not published)","source":"flash_next_report","at":"Qwen3.8-Flash-Next report p.3 §1; p.16 §3.2","quote":"so we refit the scaling law (Kaplan et al., 2020) used for the Qwen3.5 series","note":"§3.2 of the same report: the Qwen scaling laws cover learning rate and batch size."},"eval.aa_index":{"value":23,"source":"aa","at":"Artificial Analysis model page /models/qwen3-5-27b, Wayback snapshot 2026-10-19","quote":"Qwen3.5 27B (Reasoning) scores 23 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.5 27B (Reasoning). Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":26895998464,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.5-27B@fc05dae safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":2542796800,"formula":"input embedding + output head, Qwen/Qwen3.5-27B@fc05dae safetensors headers"},"params.counted_non_embedding":{"value":24353201664,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.5-27B@fc05dae safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":383273184,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.5-27B@fc05dae safetensors headers"},"params.counted_active_non_embedding":{"value":24353201664.0,"formula":"all weights except embedding and output head, Qwen/Qwen3.5-27B@fc05dae safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":80.0,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":3.4,"formula":"ffn.dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"kv.growth_per_token":{"value":65536.0,"formula":"65536 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":8589934592.0,"formula":"65536 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":65536.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2427.259259259259,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":318145725.6296296,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":65536.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":65536.0,"tokens_per_entry":1}],"formula":"Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen35-35b","name":"Qwen3.5-35B-A3B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"Describes the post-trained checkpoint; the base weights are public separately as Qwen3.5-35B-A3B-Base.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B/resolve/59d61f3ce65a6d9863b86d2e96597125219dc754/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B/resolve/59d61f3ce65a6d9863b86d2e96597125219dc754/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B/resolve/59d61f3ce65a6d9863b86d2e96597125219dc754/model.safetensors.index.json","kind":"config"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"flash_next_report":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/69885871a64393807d988b27b1b5e380e8f28526/tech_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019105531/https://artificialanalysis.ai/models/qwen3-5-35b-a3b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-02-24","source":"github","at":"QwenLM GitHub README; News","quote":"2026-02-24: Qwen3.5-122B-A10B, Qwen3.5-35B-A3B, and Qwen3.5-27B are released"},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder","note":"Card's quickstart gives text, image and video input examples; config defines image and video tokens."},"params.active":{"value":3000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"35B in total and 3B activated"},"params.note":{"value":"Rounded card figure, listed under the Language Model block; the vision encoder is separate. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder"},"params.total":{"value":35000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"35B in total and 3B activated"},"shape.d_model":{"value":2048,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":40,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":10,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":16,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"10 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Gated Attention is GQA with 16 Q / 2 KV heads."},"attn.kv_heads":{"value":2,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 40-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":30,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.__init__","quote":"self.q_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":32,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 32 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 32 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5MoeRMSNormGated in modeling_qwen3_5_moe.py."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"moe.expert_dim":{"value":512,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":256,"source":"config","at":"/text_config/num_experts"},"moe.router":{"value":"softmax","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture of Experts","quote":"8 Routed + 1 Shared"},"moe.shared_dim":{"value":512,"source":"config","at":"/text_config/shared_expert_intermediate_size"},"moe.top_k":{"value":8,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"10 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Every layer is MoE; Qwen3_5Moe code has no dense-layer option."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_top_value /= router_top_value.sum(dim=-1, keepdim=True)","note":"Unconditional in this code path."},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multi-steps."},"mtp.block":{"value":"one decoder layer (gated attention + MoE); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeRMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"train.objective":{"value":"early-fusion training on vision-language tokens","source":"card","at":"README.md; Qwen3.5 Highlights","quote":"Early fusion training on multimodal tokens"},"scaling.hp_rule":{"value":"LR and batch from a fitted hyperparameter scaling law (coefficients not published)","source":"flash_next_report","at":"Qwen3.8-Flash-Next report p.3 §1; p.16 §3.2","quote":"so we refit the scaling law (Kaplan et al., 2020) used for the Qwen3.5 series","note":"§3.2 of the same report: the Qwen scaling laws cover learning rate and batch size."},"eval.aa_index":{"value":19,"source":"aa","at":"Artificial Analysis model page /models/qwen3-5-35b-a3b, Wayback snapshot 2026-10-19","quote":"Qwen3.5 35B A3B (Reasoning) scores 19 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.5 35B A3B (Reasoning). Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","moe.balancing","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":34660610688,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.5-35B-A3B@59d61f3 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_embedding":{"value":1017118720,"formula":"input embedding + output head, Qwen/Qwen3.5-35B-A3B@59d61f3 safetensors headers"},"params.counted_non_embedding":{"value":33643491968,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.5-35B-A3B@59d61f3 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_max_layer":{"value":842700992,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.5-35B-A3B@59d61f3 safetensors headers"},"params.counted_active_non_embedding":{"value":2437870208.0,"formula":"non-expert weights + routed experts × 8/256, excluding embedding and output head, Qwen/Qwen3.5-35B-A3B@59d61f3 safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"shape.aspect":{"value":51.2,"formula":"d_model / layers"},"attn.qk_width":{"value":4096,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":32.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.08571428571428572,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":20480.0,"formula":"20480 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":2684354560.0,"formula":"20480 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":20480.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":6826.666666666667,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":894784853.3333334,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":20480.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":20480.0,"tokens_per_entry":1}],"formula":"Global KV: 10 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen35-397b","name":"Qwen3.5-397B-A17B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/resolve/8472618112abcbd45acbcdc58436aff4233c23f7/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/resolve/8472618112abcbd45acbcdc58436aff4233c23f7/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20260906222106id_/https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.5&type=qwen_ai","kind":"blog"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/resolve/8472618112abcbd45acbcdc58436aff4233c23f7/model.safetensors.index.json","kind":"config"},"flash_next_report":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/69885871a64393807d988b27b1b5e380e8f28526/tech_report.pdf","kind":"paper"},"aa":{"url":"https://web.archive.org/web/20261019105706/https://artificialanalysis.ai/models/qwen3-5-397b-a17b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-02-16","source":"blog","at":"qwen.ai article API; data.extra.date","quote":"2026-02-16T04:00:00+08:00","note":"Qwen3.5 blog date (Beijing time)."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder","note":"Card's quickstart gives text, image and video input examples; config defines image and video tokens."},"params.active":{"value":17000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"397B in total and 17B activated"},"params.note":{"value":"Rounded card figure, listed under the Language Model block; the vision encoder is separate. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder"},"params.total":{"value":397000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"397B in total and 17B activated"},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":60,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":15,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":32,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"15 \\* (3 \\* (Gated DeltaNet -> MoE) -> 1 \\* (Gated Attention -> MoE))","note":"Gated Attention is GQA with 32 Q / 2 KV heads."},"attn.kv_heads":{"value":2,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 60-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":45,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.__init__","quote":"self.q_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/rope_parameters/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":64,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 64 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 64 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5MoeRMSNormGated in modeling_qwen3_5_moe.py."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"moe.expert_dim":{"value":1024,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/text_config/num_experts"},"moe.router":{"value":"softmax","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture of Experts","quote":"10 Routed + 1 Shared"},"moe.shared_dim":{"value":1024,"source":"config","at":"/text_config/shared_expert_intermediate_size"},"moe.top_k":{"value":10,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"15 \\* (3 \\* (Gated DeltaNet -> MoE) -> 1 \\* (Gated Attention -> MoE))","note":"Every layer is MoE; Qwen3_5Moe code has no dense-layer option."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_top_value /= router_top_value.sum(dim=-1, keepdim=True)","note":"Unconditional in this code path."},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multi-steps."},"mtp.block":{"value":"one decoder layer (gated attention + MoE); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeRMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"train.precision":{"value":"FP8 (activations, MoE routing, GEMMs); BF16 kept in sensitive layers by runtime monitoring","source":"blog","at":"Qwen3.5 blog; Infrastructure","quote":"a native FP8 pipeline applies low precision to activations, MoE routing, and GEMM operations—with runtime monitoring preserving BF16 in sensitive layers","note":"Stated for the Qwen3.5 training infrastructure in the 397B-A17B release post; scaling granularity not stated."},"train.objective":{"value":"early-fusion training on vision-language tokens","source":"card","at":"README.md; Qwen3.5 Highlights","quote":"Early fusion training on multimodal tokens"},"scaling.hp_rule":{"value":"LR and batch from a fitted hyperparameter scaling law (coefficients not published)","source":"flash_next_report","at":"Qwen3.8-Flash-Next report p.3 §1; p.16 §3.2","quote":"so we refit the scaling law (Kaplan et al., 2020) used for the Qwen3.5 series","note":"§3.2 of the same report: the Qwen scaling laws cover learning rate and batch size."},"eval.aa_index":{"value":18,"source":"aa","at":"Artificial Analysis model page /models/qwen3-5-397b-a17b, Wayback snapshot 2026-10-19","quote":"Qwen3.5 397B A17B (Reasoning) scores 18 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.5 397B A17B (Reasoning). Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","moe.balancing","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","params.active_non_embedding"],"derived":{"params.counted_total":{"value":396346350336,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.5-397B-A17B@8472618 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"params.counted_embedding":{"value":2034237440,"formula":"input embedding + output head, Qwen/Qwen3.5-397B-A17B@8472618 safetensors headers"},"params.counted_non_embedding":{"value":394312112896,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.5-397B-A17B@8472618 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"params.counted_max_layer":{"value":6575157504,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.5-397B-A17B@8472618 safetensors headers"},"params.counted_active_non_embedding":{"value":15314803456.0,"formula":"non-expert weights + routed experts × 10/512, excluding embedding and output head, Qwen/Qwen3.5-397B-A17B@8472618 safetensors headers; reproduces stated total (other modalities) and active (with embedding and output head)"},"shape.aspect":{"value":68.26666666666667,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":51.2,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.042821158690176324,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":30720.0,"formula":"30720 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":4026531840.0,"formula":"30720 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":30720.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1807.0588235294117,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":236854814.11764705,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":30720.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":30720.0,"tokens_per_entry":1}],"formula":"Global KV: 15 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen38-24t","name":"Qwen3.8-2.4T-A95B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/resolve/207bd685a7e3696cfaff12ded7c6a7ea0f88c996/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/resolve/207bd685a7e3696cfaff12ded7c6a7ea0f88c996/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/resolve/207bd685a7e3696cfaff12ded7c6a7ea0f88c996/model.safetensors.index.json","kind":"config"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019105741/https://artificialanalysis.ai/models/qwen3-8-2-4t-a95b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-12","source":"github","at":"QwenLM GitHub README; News","quote":"2026-08-12: Qwen3.8-2.4T-A95B is now available on","note":"Hugging Face repository created 2026-08-08, before public release."},"meta.license":{"value":"other (qwen3.8-max)","source":"card","at":"README.md; YAML front matter","quote":"license_name: qwen3.8-max"},"meta.modality":{"value":"text","source":"card","at":"README.md; API Usage","quote":"Qwen3.8-2.4T-A95B is a text-only model that requires thinking mode for all interactions. Multimodal inputs are not supported"},"params.active":{"value":95000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"2.4T in total and 95B activated"},"params.note":{"value":"Rounded card figure for a text-only model. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Quickstart; API Usage","quote":"Qwen3.8-2.4T-A95B is a text-only model"},"params.total":{"value":2400000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"2.4T in total and 95B activated"},"shape.d_model":{"value":8192,"source":"config","at":"/hidden_size"},"shape.layers":{"value":92,"source":"config","at":"/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":23,"source":"config","at":"/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/head_dim"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"23 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Gated Attention is GQA with 64 Q / 4 KV heads."},"attn.kv_heads":{"value":4,"source":"config","at":"/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 92-layer backbone from /layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":69,"source":"config","at":"/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeAttention.__init__","quote":"self.q_norm = Qwen3_5MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/linear_value_head_dim"},"attn.linear_heads":{"value":128,"source":"config","at":"/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 128 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 128 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5MoeRMSNormGated in modeling_qwen3_5_moe.py; config output_gate_type swish (not read by the code)."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"moe.expert_dim":{"value":2048,"source":"config","at":"/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/num_experts"},"moe.router":{"value":"softmax","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture of Experts","quote":"10 Routed + 1 Shared"},"moe.shared_dim":{"value":2048,"source":"config","at":"/shared_expert_intermediate_size"},"moe.top_k":{"value":10,"source":"config","at":"/num_experts_per_tok"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"23 × (3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE))","note":"Every layer is MoE; Qwen3_5Moe code has no dense-layer option."},"moe.norm_topk":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeTopKRouter.forward","quote":"router_top_value /= router_top_value.sum(dim=-1, keepdim=True)","note":"Unconditional in this code path."},"mtp.modules":{"value":1,"source":"config","at":"/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multiple steps."},"mtp.block":{"value":"one decoder layer (gated attention + MoE); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5_moe.py; Qwen3_5MoeRMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/rms_norm_eps"},"ctx.max":{"value":1010000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,010,000 tokens","note":"262,144 native (config max_position_embeddings); extension method not stated."},"eval.aa_index":{"value":40,"source":"aa","at":"Artificial Analysis model page /models/qwen3-8-2-4t-a95b, Wayback snapshot 2026-10-19","quote":"Qwen3.8 2.4T A95B scores 40 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.8 2.4T A95B. Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","moe.balancing","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":2419804697984,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.8-2.4T-A95B@207bd68 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":4068474880,"formula":"input embedding + output head, Qwen/Qwen3.8-2.4T-A95B@207bd68 safetensors headers"},"params.counted_non_embedding":{"value":2415736223104,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.8-2.4T-A95B@207bd68 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":26262741376,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.8-2.4T-A95B@207bd68 safetensors headers"},"params.counted_active_non_embedding":{"value":91219391872.0,"formula":"non-expert weights + routed experts × 10/512, excluding embedding and output head, Qwen/Qwen3.8-2.4T-A95B@207bd68 safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":89.04347826086956,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":51.2,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.03958333333333333,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":94208.0,"formula":"94208 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":12348030976.0,"formula":"94208 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":94208.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":991.6631578947369,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":129979273.43157895,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":94208.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":94208.0,"tokens_per_entry":1}],"formula":"Global KV: 23 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen38-27b","name":"Qwen3.8-27B","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.8-27B/resolve/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.8-27B/resolve/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/config.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen3_5/modeling_qwen3_5.py","kind":"code"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.8-27B/resolve/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/model.safetensors.index.json","kind":"config"},"github":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8/2ea10dc725823bf7c3e21ce8557cbe15245132ae/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019105913/https://artificialanalysis.ai/models/qwen3-8-27b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-14","source":"github","at":"QwenLM GitHub README; News","quote":"2026-08-14: Qwen3.8-27B is now available on","note":"Hugging Face repository created 2026-08-05, before public release."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image + video","source":"card","at":"README.md; introduction","quote":"a native vision-language model that understands images and videos"},"params.active":{"value":27000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Parameters: 27B","note":"Dense model; all parameters active."},"params.note":{"value":"Rounded card figure, listed under the Language Model block; the vision encoder is separate. Whether the MTP module is counted is not stated.","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder"},"params.total":{"value":27000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"Number of Parameters: 27B"},"shape.d_model":{"value":5120,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":64,"source":"config","at":"/text_config/num_hidden_layers"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5Attention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)","note":"Config attn_output_gate true; the code applies the gate unconditionally."},"attn.global_layers":{"value":16,"source":"config","at":"/text_config/layer_types | count full_attention"},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":24,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, GQA","source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"16 × (3 × (Gated DeltaNet → FFN) → 1 × (Gated Attention → FFN))","note":"Gated Attention is GQA with 24 Q / 4 KV heads."},"attn.kv_heads":{"value":4,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 GQA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 64-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","GQA"]},"attn.linear_layers":{"value":48,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Gated Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5Attention.__init__","quote":"self.q_norm = Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the full-attention layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":48,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 48 V heads, head dim 128, short conv 4, SiLU-gated RMSNorm on output","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 48 for V and 16 for QK","note":"QK heads/dim from config linear_num_key_heads and linear_key_head_dim; conv kernel from config linear_conv_kernel_dim; SiLU output gate from Qwen3_5RMSNormGated in modeling_qwen3_5.py; config output_gate_type swish (not read by the code)."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5MLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.dim":{"value":17408,"source":"config","at":"/text_config/intermediate_size"},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); card: MTP trained with multiple steps."},"mtp.block":{"value":"one decoder layer (gated attention + dense SwiGLU FFN); RMSNormed hidden state and RMSNormed next-token embedding fused by linear fc; final RMSNorm","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.pre_fc_norm_embedding.weight","note":"From weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc, layers.0 (self_attn with q_norm/k_norm, mlp), norm."},"mtp.ffn":{"value":"dense","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.gate_proj.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc.weight"},"norm.placement":{"value":"pre","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen3_5.py; Qwen3_5RMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Gain initialized at 0 and applied as 1 + weight."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"ctx.max":{"value":1000000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,000,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"eval.aa_index":{"value":34,"source":"aa","at":"Artificial Analysis model page /models/qwen3-8-27b, Wayback snapshot 2026-10-19","quote":"Qwen3.8 27B (Xhigh) scores 34 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.8 27B (Xhigh). Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":26895998464,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.8-27B@1d4bf0f safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":2542796800,"formula":"input embedding + output head, Qwen/Qwen3.8-27B@1d4bf0f safetensors headers"},"params.counted_non_embedding":{"value":24353201664,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.8-27B@1d4bf0f safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":383273184,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.8-27B@1d4bf0f safetensors headers"},"params.counted_active_non_embedding":{"value":24353201664.0,"formula":"all weights except embedding and output head, Qwen/Qwen3.8-27B@1d4bf0f safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":80.0,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":3.4,"formula":"ffn.dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"kv.growth_per_token":{"value":65536.0,"formula":"65536 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":8589934592.0,"formula":"65536 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":65536.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2427.259259259259,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":318145725.6296296,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":65536.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":65536.0,"tokens_per_entry":1}],"formula":"Global KV: 16 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"qwen38-flash-next","name":"Qwen3.8-Flash-Next","org":"Qwen","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint. Recipe facts are from the technical report.","sources":{"card":{"url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/de4b8e4d43b917e7706784d8bb445c9af86a3540/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/de4b8e4d43b917e7706784d8bb445c9af86a3540/config.json","kind":"config"},"weights-index":{"url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/de4b8e4d43b917e7706784d8bb445c9af86a3540/model.safetensors.index.json","kind":"config"},"hf-transformers":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py","kind":"code"},"hf-config-code":{"url":"https://raw.githubusercontent.com/huggingface/transformers/d4804f9d76b8c071b993d813e9e87067b780cbee/src/transformers/models/qwen4_exp/configuration_qwen4_exp.py","kind":"code"},"report":{"url":"https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/69885871a64393807d988b27b1b5e380e8f28526/tech_report.pdf","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20260922022512id_/https://qwen.ai/api/v2/article/?language=en-US&path=qwen3.8-flash-next&type=qwen_ai","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019105948/https://artificialanalysis.ai/models/qwen3-8-flash-next","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-08-26","source":"report","at":"p.1 header date","quote":"2026-08-26","note":"Same date as the qwen.ai blog API entry; Hugging Face repository created 2026-08-24."},"meta.license":{"value":"other (qwen-community-1.0)","source":"card","at":"README.md; YAML front matter","quote":"license_name: qwen-community-1.0"},"meta.modality":{"value":"text + image + video","basis":"inferred","source":"card","at":"README.md; Model Overview","quote":"Type: Causal Language Model with Vision Encoder","note":"Card's quickstart gives text, image and video input examples; config defines image and video tokens."},"params.active":{"value":6000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"125B with 6B activated, plus 51B n-gram embedding and 4B MTP"},"params.note":{"value":"Language-model count; excludes the 51B n-gram embedding tables and the 4B MTP module.","source":"card","at":"README.md; Model Overview; Language Model","quote":"125B with 6B activated, plus 51B n-gram embedding and 4B MTP"},"params.total":{"value":125000000000,"source":"report","at":"p.1 §1 Introduction","quote":"with 125B total parameters, 6B activated per token, and additional 51B parameters of n-gram embedding tables"},"shape.d_model":{"value":2560,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/text_config/num_hidden_layers"},"shape.memory":{"value":"Hashed bigram/trigram embedding at the input of layer 2, 51B params held in host memory: 16 hash heads (8 per order) of ~20M rows × 160 dims; residual-stream-gated value plus dilated short conv added to all 4 residual branches","source":"card","at":"README.md; Model Overview; Language Model","quote":"N-gram Embedding: 20,000,000 (bigrams/trigrams at layer 2)","note":"Head layout and dims from config ngram_size=3, heads_per_ngram=8, ngram_vocab_size_base, ple_embed_dim=2560, ple_layer_ids [2], make_ngram_vocab_size_divisible_by 128, ple_conv_kernel_size 4 (dilation ngram_size) and Qwen4ExpTextNGramEmbedding / Qwen4ExpTextPLELayer."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/text_config/tie_word_embeddings"},"shape.vocab":{"value":248320,"source":"config","at":"/text_config/vocab_size"},"attn.gate":{"value":"sigmoid, elementwise on the attention output before o_proj; gate projected jointly with Q","source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextAttention.forward","quote":"attn_output = attn_output * torch.sigmoid(gate)"},"attn.global_layers":{"value":12,"source":"config","at":"/text_config/layer_types | count full_attention","note":"The checkpoint labels the QSA layers full_attention."},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.heads":{"value":24,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"Gated DeltaNet, QSA","source":"card","at":"README.md; Highlights; Hybrid Attention with QSA","quote":"has been reworked into Gated DeltaNet and Qwen Sparse Attention (QSA)","note":"QSA is GQA (24 Q / 2 KV heads) restricted to indexer-selected micro-blocks."},"attn.kv_heads":{"value":2,"source":"config","at":"/text_config/num_key_value_heads"},"attn.layout":{"value":"3 Gated DeltaNet : 1 QSA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\", \"linear_attention\", \"linear_attention\", \"linear_attention\", \"full_attention\" ]","note":"Config full_attention_interval 4; layer_types lists the same pattern. Ordered 48-layer backbone from /text_config/layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA","Gated DeltaNet","Gated DeltaNet","Gated DeltaNet","QSA"]},"attn.linear_layers":{"value":36,"source":"config","at":"/text_config/layer_types | count linear_attention"},"attn.position":{"value":"partial RoPE","source":"card","at":"README.md; Model Overview; Qwen Sparse Attention","quote":"Rotary Position Embedding Dimension: 64","note":"64 of 256 head dims rotated; config rope_parameters mrope_section [11, 11, 10], interleaved in code."},"attn.qk_norm":{"value":true,"source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextAttention.__init__","quote":"self.q_norm = Qwen4ExpTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K in the QSA layers."},"attn.rope_fraction":{"value":0.25,"source":"config","at":"/text_config/partial_rotary_factor"},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_parameters/rope_theta"},"attn.bias":{"value":false,"source":"config","at":"/text_config/attention_bias"},"attn.conv":{"value":4,"source":"config","at":"/text_config/linear_conv_kernel_dim"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/text_config/linear_value_head_dim"},"attn.linear_heads":{"value":48,"source":"config","at":"/text_config/linear_num_value_heads"},"attn.linear_kind":{"value":"Gated DeltaNet: 16 QK heads, 48 V heads, head dim 128, short conv 4, sigmoid output gate after RMSNorm","source":"card","at":"README.md; Model Overview; Gated DeltaNet","quote":"Number of Linear Attention Heads: 48 for V and 16 for QK","note":"Sigmoid output gate replaces the SiLU gate of original GDN (report §2.1.1, Eq. 11; config output_gate_type). QK heads/dim from config linear_num_key_heads and linear_key_head_dim."},"attn.sparse":{"value":"QSA: MQA indexer (4 query heads, 1 key head) scores 4-token average-pooled key blocks; top-512 blocks (2048-token budget) per query plus the trailing partial block","source":"report","at":"p.7 §2.1.2 Qwen Sparse Attention; Implementation Details","quote":"With a token budget of K = 2048 and a compression ratio of r = 4, QSA selects up to 512 complete blocks for each query","note":"Config indexer_n_heads 4, indexer_kv_heads 1, indexer_head_dim 128, indexer_budget 2048, indexer_compress_ratio 4. The MTP layer also uses QSA."},"ffn.activation":{"value":"SwiGLU","source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextExperts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (config hidden_act silu)."},"ffn.bias":{"value":false,"source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)","note":"Shared-expert MLP; routed experts are bias-free weight tensors."},"ffn.clamp":{"value":"none","source":"report","at":"p.2 §1 Introduction; Training Stability","quote":"without relying on explicit clipping methods such as qk-clip (Kimi Team, 2025) or SwiGLU-clip","note":"Stated for the full-scale run; the released code applies no clamp."},"moe.expert_dim":{"value":640,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":512,"source":"config","at":"/text_config/num_experts"},"moe.router":{"value":"softmax","source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextTopKRouter.forward","quote":"router_probs = torch.nn.functional.softmax(router_logits, dtype=torch.float, dim=-1)","note":"Top-k taken after softmax; shared expert has its own sigmoid gate."},"moe.shared":{"value":1,"source":"card","at":"README.md; Model Overview; Mixture Of Experts","quote":"10 Routed + 1 Shared"},"moe.shared_dim":{"value":640,"source":"config","at":"/text_config/shared_expert_intermediate_size"},"moe.top_k":{"value":10,"source":"config","at":"/text_config/num_experts_per_tok"},"moe.dense_layers":{"value":0,"source":"card","at":"README.md; Model Overview; Hidden Layout","quote":"12 × (3 × (Gated DeltaNet → MoE) → 1 × (Qwen Sparse Attention → MoE))","note":"Every decoder layer builds Qwen4ExpTextSparseMoeBlock."},"moe.balancing":{"value":"global-batch load-balancing loss","source":"blog","at":"Qwen3.8-Flash-Next blog; Other Architecture Optimizations; Ultra-sparse MoE","quote":"Ultra-sparse MoE : With global load balancing","note":"Cites 'Demons in the Detail' (global-batch balancing); coefficient not stated."},"moe.norm_topk":{"value":true,"source":"hf-config-code","at":"configuration_qwen4_exp.py; Qwen4ExpTextConfig defaults","quote":"norm_topk_prob: bool = True","note":"config.json does not set norm_topk_prob, so the class default applies; read by Qwen4ExpTextTopKRouter."},"mtp.modules":{"value":1,"source":"config","at":"/text_config/mtp_num_hidden_layers","note":"One MTP decoder layer (mtp.layers.0 only in the weight index); report: multi-step MTP."},"mtp.block":{"value":"one decoder layer (QSA + MoE, each through Gated Residual); RMSNormed hidden state and RMSNormed next-token embedding projected by separate fc_hidden and fc_embedding","source":"report","at":"p.7 §2.1.2 Qwen Sparse Attention; Implementation Details","quote":"all full-attention layers in the backbone and MTP module are replaced with QSA","note":"Weight names: pre_fc_norm_hidden, pre_fc_norm_embedding, fc_hidden, fc_embedding, hyper_connection_mixer, layers.0 (self_attn with indexer, attn/mlp hyper_connection, MoE mlp)."},"mtp.ffn":{"value":"MoE","source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.layers.0.mlp","quote":"mtp.layers.0.mlp.shared_expert_gate.weight"},"mtp.shares_embedding":{"value":true,"source":"config","at":"config.json; text_config.mtp_use_dedicated_embeddings","quote":"\"mtp_use_dedicated_embeddings\": false","note":"The weight index has no MTP embedding or output head."},"mtp.params":{"value":4000000000,"source":"card","at":"README.md; Model Overview; Language Model","quote":"125B with 6B activated, plus 51B n-gram embedding and 4B MTP"},"mtp.released":{"value":true,"source":"weights-index","at":"model.safetensors.index.json; weight_map mtp.* entries","quote":"mtp.fc_hidden.weight"},"norm.placement":{"value":"pre","source":"report","at":"p.12 §2.2 Gated Residual","quote":"GR also replaces the block’s pre-normalization rather than sitting in front of it","note":"Normalization sits inside the Gated Residual read: per-branch RMSNorm over 4 residual branches, then a sigmoid elementwise gate."},"norm.type":{"value":"RMSNorm (1+w gain)","source":"hf-transformers","at":"modeling_qwen4_exp.py; Qwen4ExpTextRMSNorm.forward","quote":"output = output * (1.0 + self.weight.float())","note":"Report §2.1.1: zero-centered RMSNorm is applied to all RMSNorm layers."},"norm.eps":{"value":1e-06,"source":"config","at":"/text_config/rms_norm_eps"},"norm.residual":{"value":"Gated Residual: 4 residual branches, each RMSNormed; elementwise sigmoid read gate (rank-320 bottleneck) averages branches into the block input; output written to every branch with per-branch scalar 2·sigmoid; no branch mixing","source":"report","at":"p.11 §2.2 Residual; Gated Residual","quote":"We use nr = 4 branches, with a separate GR module for the attention block and the MLP block of every layer.","note":"Separate GR per attention and MoE sublayer; a final GR read replaces the norm before the LM head. Branches = config hc_count 4; rank 320 per card/config hc_lowrank."},"ctx.final":{"value":262144,"source":"report","at":"p.7 §2.1.2 Qwen Sparse Attention; Training Details","quote":"Each step comprises 96 sequences of 256K tokens","note":"256K-token sequences in the final continued-pretraining stage."},"ctx.max":{"value":1000000,"source":"card","at":"README.md; Model Overview","quote":"262,144 natively and extensible up to 1,000,000 tokens","note":"262,144 native (config max_position_embeddings); beyond that via inference-time YaRN."},"train.batch":{"value":"constant (no batch-size warmup); size not disclosed","source":"report","at":"p.18 §3.2 Hyperparameter Scaling; Batch-Size Warmup Is No Longer Necessary","quote":"we do not employ batch-size warmup in our production runs"},"train.compute":{"value":"~1/9 the training FLOPs of the 397B-A17B predecessor (~1/3 its tokens); absolute figure not disclosed","source":"report","at":"p.1 Abstract","quote":"at 1/3 the activated parameters, 1/3 the training tokens, and roughly 1/9 the training FLOPs"},"train.optimizer":{"value":"Muon + AdamW","source":"report","at":"p.16 §3.1 Optimizer","quote":"We use Muon as the main optimizer","note":"Muon (Nesterov 0.95, Polar Express, 8 Newton–Schulz steps, update scaled by 0.2·sqrt(max(A,B)); qkv and GDN input projections split per head, fc1 into gate/up) on linear maps; AdamW on embeddings, output head, router and GR low-rank projections; Adam without weight decay on n-gram tables."},"train.stages":{"value":[{"name":"QSA dense distillation (indexer only)","tokens":"~2B","seq_len":262144,"lr":0.001,"batch_sequences":8,"steps":1000,"quote":"only the indexer is trained for 1,000 steps, using a learning rate of 1 × 10−3 . Each step comprises 8 sequences of 256K tokens, amounting to approximately 2B training tokens in total."},{"name":"QSA sparse training","tokens":"~200B","seq_len":262144,"lr":2.5e-05,"batch_sequences":96,"steps":8000,"quote":"the backbone and indexer are jointly trained for 8,000 steps using a learning rate of 2.5 × 10−5 . Each step comprises 96 sequences of 256K tokens, totaling roughly 200B training tokens."}],"source":"report","at":"p.6–7 §2.1.2 Qwen Sparse Attention; Training Details","note":"Only the final 256K continued-pretraining stages are disclosed; earlier stages and the total token count are not."},"train.init":{"value":"MoE router parameters normalized at initialization","source":"blog","at":"Qwen3.8-Flash-Next blog; Other Architecture Optimizations; Training stability","quote":"normalized MoE router initialization are retained"},"train.stability":{"value":"no QK-clip or SwiGLU-clip","source":"report","at":"p.2 §1 Introduction; Training Stability","quote":"without relying on explicit clipping methods such as qk-clip (Kimi Team, 2025) or SwiGLU-clip","note":"Report: the full-scale run had no loss spikes or anomalous gradient-norm fluctuations."},"scaling.hp_rule":{"value":"LR and batch from a hyperparameter scaling law refitted for the new architecture and Muon; target batch from step 0 (no batch-size warmup)","source":"card","at":"README.md; Highlights, Tailored Training Recipe","quote":"Guided by refitted scaling laws, we eliminate traditional batch-size warmups and start directly at the target batch size","note":"Report §3.2: the law gives LR and batch size; coefficients are not published."},"scaling.ladder":{"value":"28-layer 25B-A3B MoE (400B tokens at 4K + 80B at 32K) for architecture ablations; 35B-A3B for QSA; HP law checked on 20-layer 10.8B-A0.89B (4T tokens) and 48-layer 156B-A7B (419B tokens)","source":"report","at":"p.5 §2.1.1 Architecture Ablation; p.8 §2.1.2; pp.17–18 §3.2","quote":"Each checkpoint corresponds to a 28-layer 25B-A3B MoE model based on the Qwen3.5 architecture, pretrained first on 400B tokens with a 4K context length, and subsequently on 80B tokens with a 32K context length."},"eval.aa_index":{"value":40,"source":"aa","at":"Artificial Analysis model page /models/qwen3-8-flash-next, Wayback snapshot 2026-10-19","quote":"Qwen3.8-Flash-Next scores 40 on the Artificial Analysis Intelligence Index","note":"Variant: Qwen3.8-Flash-Next. Index v4.3.2."}},"not_disclosed":["mtp.depth","mtp.loss_weight","ctx.pretrain","train.tokens","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":125743653763,"formula":"embedding + output head + all experts + other weights, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":1271398400,"formula":"input embedding + output head, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers"},"params.counted_non_embedding":{"value":124472255363,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":2626818787,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers"},"params.counted_active_non_embedding":{"value":6035596163.0,"formula":"non-expert weights + routed experts × 10/512, excluding embedding and output head, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_lookup":{"value":51200245792,"formula":"n-gram / per-layer embedding tables, Qwen/Qwen3.8-Flash-Next@de4b8e4 safetensors headers"},"shape.aspect":{"value":53.333333333333336,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":12.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.25,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":51.2,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.048,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":24576.0,"formula":"24576 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":3221225472.0,"formula":"24576 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":24576.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":4096.0,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":536870912.0,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":24576.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":24576.0,"tokens_per_entry":1}],"formula":"Global KV: 12 × (1024 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"solar-open-2","name":"Solar-Open2-250B","org":"Upstage","country":"South Korea","kind":"reasoning","weights":"open","note":"No base checkpoint is public; the record describes the post-trained Solar-Open2-250B release.","sources":{"card":{"url":"https://huggingface.co/upstage/Solar-Open2-250B/resolve/9190fbe63a2ad8e17fc766ccceb36de7c66f004b/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/upstage/Solar-Open2-250B/resolve/9190fbe63a2ad8e17fc766ccceb36de7c66f004b/config.json","kind":"config"},"index":{"url":"https://huggingface.co/upstage/Solar-Open2-250B/resolve/9190fbe63a2ad8e17fc766ccceb36de7c66f004b/model.safetensors.index.json","kind":"config"},"modeling_solar_open2":{"url":"https://raw.githubusercontent.com/upstageAI/transformers/ef5c37485091057070b36dd0620798f74988eb01/src/transformers/models/solar_open2/modeling_solar_open2.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2607.20062v2","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20261019101128/https://www.upstage.ai/blog/en/solar-open-2","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019110119/https://artificialanalysis.ai/models/solar-open2-250b","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-07-22","source":"blog","at":"upstage.ai/blog/en/solar-open-2; header","quote":"July 22, 2026","note":"arXiv v1 of the report is also dated 22 Jul 2026."},"meta.license":{"value":"other (Upstage Solar License)","source":"card","at":"README.md; YAML front matter","quote":"license_name: upstage-solar-license"},"meta.modality":{"value":"text","basis":"inferred","source":"card","at":"README.md; YAML front matter","quote":"pipeline_tag: text-generation","note":"Text-generation pipeline tag; SolarOpen2ForCausalLM config has no vision components."},"params.total":{"value":250287794944,"source":"card","at":"README.md; Model Overview table","quote":"250B (250,287,794,944)"},"params.active":{"value":15000000000,"source":"report","at":"p.2 §2","quote":"Solar Open 2 is a sparse Mixture-of-Experts (MoE) Transformer with 250B total parameters and 15B active parameters per token."},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":48,"source":"config","at":"/num_hidden_layers"},"shape.vocab":{"value":196608,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.tokenizer":{"value":"byte-level BPE, 196,608 vocab, digit splitting; inherited unchanged from Solar Open 1","source":"report","at":"p.2 §2.1","quote":"It is a custom byte-level BPE tokenizer with a 196,608-token vocabulary, trained on a corpus that oversamples Korean and the target domains, with digit splitting and whitespace preservation for arithmetic and code fidelity."},"attn.kind":{"value":"GQA, KDA","source":"report","at":"p.3 §2.2","quote":"It adopts GQA (Ainslie et al., 2023) for the softmax layers and KDA (Kimi Delta Attention) (Yang et al., 2025; Kimi Team, 2025) for the linear layers"},"attn.layout":{"value":"3 KDA : 1 GQA (GQA first in each block of 4)","source":"config","at":"config.json; /gqa_layers","quote":"\"gqa_layers\": [ 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44 ]","note":"Config /gqa_layers puts GQA at layers 0, 4, …, 44 (gqa_interval 3 KDA layers between). /gqa_layers gives zero-based global-attention indices; other layers use KDA.","basis":"inferred","sequence":["GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA","GQA","KDA","KDA","KDA"]},"attn.global_layers":{"value":12,"source":"config","at":"/gqa_layers | len"},"attn.linear_layers":{"value":36,"source":"report","at":"p.3 §2.2","quote":"so that 12 of the 48 layers (25%) are softmax and 36 (75%) are linear"},"attn.heads":{"value":64,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.position":{"value":"NoPE","source":"report","at":"p.4 §2.2","quote":"Solar Open 2 removes positional encoding entirely","note":"Config /use_rope is false, so SolarOpen2Attention skips RoPE and /rope_theta is unused; KDA layers carry token order."},"attn.qk_norm":{"value":false,"source":"index","at":"/weight_map | has q_norm","note":"GQA layers have no q/k norm weights; KDA layers L2-normalize q and k inside the kernel."},"attn.gate":{"value":"elementwise sigmoid gate on SDPA output (g_proj from layer input), GQA layers","source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2Attention.forward","quote":"attn_output = attn_output * torch.sigmoid(self.g_proj(hidden_states))","note":"Config use_gqa_gate true. KDA layers use a sigmoid-gated RMSNorm on their output (SolarOpen2RMSNormGated)."},"attn.bias":{"value":false,"source":"index","at":"/weight_map | has q_proj.bias"},"attn.linear_kind":{"value":"KDA with negative eigenvalues (β = 2σ(·) ∈ (0, 2)); 64 heads × 128, no KV grouping; short conv 4 on q/k/v; low-rank (rank 128) decay and output-gate projections","source":"report","at":"p.4 §2.2","quote":"Solar Open 2 instead sets allow_neg_eigval=True: widening the write strength to β = 2σ(·) ∈ (0, 2)","note":"Config kda_allow_neg_eigval true. Heads, dims and conv from config /linear_attn_config; low-rank gates from /kda_use_full_proj false (f_a_proj/g_a_proj project to head_dim)."},"attn.linear_heads":{"value":64,"source":"config","at":"/linear_attn_config/num_heads"},"attn.linear_head_dim":{"value":128,"source":"config","at":"/linear_attn_config/head_dim"},"attn.conv":{"value":4,"source":"config","at":"/linear_attn_config/short_conv_kernel_size"},"ffn.activation":{"value":"SwiGLU","source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2Experts.forward","quote":"current_hidden_states = self.act_fn(gate) * up","note":"act_fn is SiLU (report Table 1); the shared expert uses the same gate × up form (SolarOpen2MLP)."},"ffn.bias":{"value":false,"source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.experts":{"value":320,"source":"config","at":"/n_routed_experts"},"moe.top_k":{"value":8,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":1,"source":"config","at":"/n_shared_experts"},"moe.expert_dim":{"value":1280,"source":"config","at":"/moe_intermediate_size"},"moe.shared_dim":{"value":1280,"source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2MoE.__init__","quote":"config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts","note":"moe_intermediate_size 1280 × 1 shared expert."},"moe.dense_layers":{"value":0,"source":"config","at":"/first_k_dense_replace"},"moe.router":{"value":"sigmoid","source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2TopkRouter.forward","quote":"scores = router_logits.sigmoid()","note":"Expert selection adds a per-expert e_score_correction_bias (present in the weights); the report does not describe the balancing method."},"moe.norm_topk":{"value":true,"source":"config","at":"/norm_topk_prob"},"moe.routed_scale":{"value":1.0,"source":"config","at":"/routed_scaling_factor"},"norm.type":{"value":"RMSNorm","source":"modeling_solar_open2","at":"modeling_solar_open2.py; SolarOpen2DecoderLayer.__init__","quote":"self.input_layernorm = SolarOpen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"report","at":"p.2 §2","quote":"a single shared expert with no dense layers, and pre-norm residuals."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"ctx.final":{"value":1048576,"source":"report","at":"p.9 §3.3","quote":"augments it with long-document data including repo-level code, and extends training to a 1M context length."},"ctx.max":{"value":1048576,"source":"report","at":"p.5 Table 1","quote":"Context Length 131,072 (128K) 1,048,576 (1M)","note":"Config max_position_embeddings 1048576."},"ctx.extension":{"value":"Stage 4 Length Expansion (865B tokens): continued training to 1M on a higher-quality corpus plus long documents and repo-level code; NoPE","source":"report","at":"p.6 §3","quote":"Length Expansion closes pre-training with 0.9T tokens of continued training that extend the context window to 1M and inject reasoning and agent data."},"train.tokens":{"value":"~12T","source":"report","at":"p.6 Figure 5","quote":"Stages 2–4 then train on about 12T tokens in total","note":"Stages sum to 11.865T (10T + 1T + 865B, Figure 8); excludes Solar Open 1 training behind the 5.69B transferred weights."},"train.init":{"value":"selective weight transfer from Solar Open 1: 5.69B params (2.3%) transferred (embedding/output, norms, GQA q/k/v/o, shared expert; KDA q/o re-mapped from GQA q/o); routed experts, router, KDA k/v, decay/write gates, conv and GQA output gate randomly initialized","source":"report","at":"p.7 §3.1, Table 2","quote":"Only the compatible portion is reused: about 2.3% of parameters, or 5.69B."},"train.data":{"value":"10T tokens curated from a 20T deduplicated pool by quality and rarity; real:synthetic 4:6; math and code each ≥15%; English >80%","source":"report","at":"p.8 §3.2","quote":"the resulting recipe sets (1) a real-to-synthetic ratio of 4:6, (2) a domain mix in which math and code each occupy at least 15%, and (3) an English share above 80%."},"train.averaging":{"value":"merge of four intermediate checkpoints from Stage 4 (Length Expansion)","source":"report","at":"p.9 §3.3","quote":"To compensate, we selected four intermediate checkpoints and merged them, obtaining a pre-trained language model (PLM) stronger than any single checkpoint.","note":"The merged PLM is the post-training starting point and is not released; merge weights are not stated."},"train.hardware":{"value":"NVIDIA B200 GPUs","source":"card","at":"README.md; Model Overview table","quote":"Training Hardware | NVIDIA B200 GPUs"},"train.accelerators":{"value":"B200","source":"card","at":"README.md; Model Overview table","quote":"Training Hardware | NVIDIA B200 GPUs"},"train.compute":{"value":"2M GPU hours","source":"card","at":"README.md; Model Overview table","quote":"Training GPU Time | 2M GPU Hours","note":"Scope (pretraining only or including post-training) not stated."},"train.stages":{"value":[{"name":"general pre-training","tokens":10000000000000,"note":"Curated 10T mixture.","quote":"General Pre-training is the main stage, training on 10T tokens of a broad-coverage, high-quality data mixture"},{"name":"intensive pre-training","tokens":1000000000000,"note":"Best 1T tokens of the stage-2 data re-filtered at a higher quality threshold.","quote":"Stage 3 (Intensive Pre-training) raises the quality threshold and trains again on the best 1T tokens of the Stage-2 data"},{"name":"length expansion","tokens":865000000000,"seq_len":1048576,"note":"Higher-quality base corpus plus long documents, repo-level code, reasoning and agent data; four checkpoints merged.","quote":"and Stage 4 (Length Expansion, 865B)."}],"source":"report","at":"pp.6–9 §3, §3.3, Figure 8"},"scaling.ladder":{"value":"10B-A1B proxy for architecture ablations; 200B-A15B proxy for the weight-transfer initialization","source":"report","at":"p.6 §2.2; p.7 §3.1","quote":"We ablated each of these changes on a 10B-A1B proxy before committing to the final design."},"eval.aa_index":{"value":25,"source":"aa","at":"Artificial Analysis model page /models/solar-open2-250b, Wayback snapshot 2026-10-19","quote":"Solar Open2 250B scores 25 on the Artificial Analysis Intelligence Index","note":"Variant: Solar Open2 250B. Index v4.3.2."}},"not_disclosed":["moe.balancing","mtp.depth","ctx.pretrain","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision","params.active_non_embedding"],"derived":{"params.counted_total":{"value":250287810304,"formula":"embedding + output head + all experts + other weights, upstage/Solar-Open2-250B@9190fbe safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_embedding":{"value":1610612736,"formula":"input embedding + output head, upstage/Solar-Open2-250B@9190fbe safetensors headers"},"params.counted_non_embedding":{"value":248677197568,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, upstage/Solar-Open2-250B@9190fbe safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"params.counted_max_layer":{"value":5187944960,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, upstage/Solar-Open2-250B@9190fbe safetensors headers"},"params.counted_active_non_embedding":{"value":13125084928.0,"formula":"non-expert weights + routed experts × 8/320, excluding embedding and output head, upstage/Solar-Open2-250B@9190fbe safetensors headers; reproduces stated total (backbone) and active (with embedding and output head)"},"shape.aspect":{"value":85.33333333333333,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"moe.granularity":{"value":0.3125,"formula":"moe.expert_dim / d_model"},"attn.linear_ratio":{"value":3.0,"formula":"attn.linear_layers / attn.global_layers"},"moe.sparsity":{"value":40.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.05993100863490422,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":49152.0,"formula":"49152 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6442450944.0,"formula":"49152 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":49152.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":3276.8,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":429496729.6,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":49152.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":49152.0,"tokens_per_entry":1}],"formula":"Global KV: 12 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"step-3-5-flash","name":"Step-3.5-Flash-Base","org":"StepFun","country":"China","kind":"base","weights":"open","note":null,"sources":{"config":{"url":"https://huggingface.co/stepfun-ai/Step-3.5-Flash-Base/resolve/19623ae192b589db21eb7793e75a5d3ea3ef0dae/config.json","kind":"config"},"index":{"url":"https://huggingface.co/stepfun-ai/Step-3.5-Flash-Base/resolve/19623ae192b589db21eb7793e75a5d3ea3ef0dae/model.safetensors.index.json","kind":"config"},"modeling_step3p5":{"url":"https://huggingface.co/stepfun-ai/Step-3.5-Flash/resolve/ab446a3de5e171ea341227e24bb1f090e1b771f7/modeling_step3p5.py","kind":"code"},"configuration_step3p5":{"url":"https://huggingface.co/stepfun-ai/Step-3.5-Flash/resolve/ab446a3de5e171ea341227e24bb1f090e1b771f7/configuration_step3p5.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2602.10604v2","kind":"paper"},"report_v1":{"url":"https://arxiv.org/pdf/2602.10604v1","kind":"paper"},"card":{"url":"https://huggingface.co/stepfun-ai/Step-3.5-Flash-Base/resolve/19623ae192b589db21eb7793e75a5d3ea3ef0dae/README.md","kind":"model-card"},"aa":{"url":"https://web.archive.org/web/20261019110223/https://artificialanalysis.ai/models/step-3-5-flash-0202","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-02-11","source":"report_v1","at":"p.1 arXiv stamp","quote":"arXiv:2602.10604v1 [cs.CL] 11 Feb 2026","note":"Report predates the Base weights (Hugging Face repo created 2026-03-02)."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; YAML front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text","basis":"inferred","source":"config","at":"config.json; /architectures/0","quote":"Step3p5ForCausalLM","note":"Causal-LM class with no vision config."},"params.total":{"value":196000000000,"source":"report","at":"p.6 §2.2","quote":"This configuration maintains an extensive knowledge capacity (196B total parameters)"},"params.active":{"value":11000000000,"source":"report","at":"p.6–7 §2.2","quote":"restricting per-token activation to just 11B"},"params.active_non_embedding":{"value":11000000000,"source":"report","at":"p.27 Table 6","quote":"Activated params / token (backbone) 11B","note":"Table 6 states activated params exclude embedding/output matrices; backbone without the 3 MTP heads."},"params.note":{"value":"Backbone only: 198B total / 13B active with the 3 MTP heads; active count excludes embedding and output matrices.","source":"report","at":"p.27 Table 6","quote":"“Activated params” are reported per token and exclude embedding/output matrices."},"shape.d_model":{"value":4096,"source":"config","at":"/hidden_size"},"shape.layers":{"value":45,"source":"config","at":"/num_hidden_layers"},"shape.vocab":{"value":128815,"source":"config","at":"/vocab_size","note":"Paper Table 6 gives 128,896."},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.7 §2.2","quote":"This configuration repeats a four-layer motif consisting of three SWA layers (W =512) followed by a single full GQA-8 layer."},"attn.layout":{"value":"3 SWA : 1 global (leading global layer + 11 × [3 SWA, 1 global])","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\" ]","note":"Ordered 45-layer backbone from /layer_types; auxiliary modules are excluded. The array has 46 entries; only the first num_hidden_layers=45 are backbone layers.","basis":"inferred","sequence":["GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":12,"source":"config","at":"/layer_types/0:45 | count full_attention","note":"/layer_types has 46 entries; the last is the MTP layer."},"attn.local_layers":{"value":33,"source":"config","at":"/layer_types/0:45 | count sliding_attention"},"attn.heads":{"value":64,"source":"report","at":"p.27 Table 6","quote":"Query heads (full / SWA) 64 / 96","note":"Full-attention layers; SWA layers use 96 (attn.local_heads)."},"attn.local_heads":{"value":96,"source":"config","at":"/attention_other_setting/num_attention_heads","note":"Step3p5Attention.__init__ reads it for sliding_attention layers."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/attention_other_setting/num_attention_groups","note":"Step3p5Attention.__init__ reads it for sliding_attention layers."},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim","note":"Step3p5Attention uses config head_dim for both layer types; /attention_other_setting/head_dim (128) is not read."},"attn.kv_heads":{"value":8,"source":"config","at":"/num_attention_groups","note":"Step3p5Attention reads num_attention_groups as the KV head count."},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.window":{"value":512,"source":"config","at":"/sliding_window"},"attn.position":{"value":"partial RoPE (global, 64 of 128 dims), RoPE (SWA, all 128 dims)","source":"report","at":"p.27 Table 6","quote":"RoPE dims (full / SWA) 64 / 128"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_theta/1","note":"Per-layer list read by Step3p5RotaryEmbedding; 10,000 on every sliding_attention layer."},"attn.rope_fraction":{"value":0.5,"source":"config","at":"/partial_rotary_factors/0","note":"Global layers; SWA layers use 1.0. Step3p5RotaryEmbedding reads the per-layer factor."},"attn.rope_theta":{"value":1000000,"source":"config","at":"/rope_theta/0","note":"Global layers; SWA layers 10,000. 10,000 for all layers at 4K, raised to 1M for full attention in the 32K portion of annealing (p.15 §4.2.3)."},"attn.qk_norm":{"value":true,"source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5Attention.__init__","quote":"self.q_norm = Step3p5RMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head zero-centered RMSNorm on Q and K, before RoPE; applied regardless of /use_qk_norm."},"attn.gate":{"value":"head-wise scalar sigmoid gate on each head's output, before o_proj; all layers","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5Attention.forward","quote":"self.head_dim) * gate_states.unsqueeze(-1).sigmoid()","note":"Enabled by /use_head_wise_attn_gate; g_proj maps hidden_size → num_heads; paper App. A.1. Used instead of attention sinks (/sink false)."},"attn.bias":{"value":false,"source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5Attention.__init__","quote":"self.q_proj = nn.Linear(config.hidden_size, self.q_size, bias=False)"},"ffn.activation":{"value":"SwiGLU","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5MLP.forward","quote":"return self.down_proj(gate * up)","note":"gate = SiLU(gate_proj(x)); routed experts use the same form (Step3p5MoEMLP.get_expert_output)."},"ffn.dim":{"value":11264,"source":"config","at":"/intermediate_size"},"ffn.bias":{"value":false,"source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5MLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"ffn.clamp":{"value":"gate capped at 7, up clamped to [−7, 7] in routed experts of layers 43–44; shared expert of layer 44 limit 16","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5MoEMLP.get_expert_output","quote":"up = up.clamp(min=-self.limit, max=self.limit)","note":"Limits from /swiglu_limits and /swiglu_limits_shared (0-indexed, 0 = no clamp); paper §4.1.3 adopts activation clipping in deep MoE layers against expert activation blow-up."},"moe.experts":{"value":288,"source":"config","at":"/moe_num_experts"},"moe.top_k":{"value":8,"source":"config","at":"/moe_top_k"},"moe.shared":{"value":1,"source":"report","at":"p.6 §2.2","quote":"Each MoE layer contains 288 routed experts plus one shared expert"},"moe.expert_dim":{"value":1280,"source":"config","at":"/moe_intermediate_size"},"moe.shared_dim":{"value":1280,"source":"config","at":"/share_expert_dim"},"moe.dense_layers":{"value":3,"source":"report","at":"p.6 §2.2","quote":"Step 3.5 Flash adopts a 45-layer sparse-MoE Transformer backbone (3 dense layers and 42 MoE layers)","note":"Config moe_layers_enum lists layers 3–44."},"moe.router":{"value":"sigmoid","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5MoEMLP.router_bias_func","quote":"gate_prob = torch.sigmoid(gating_output.float())","note":"Bias (use_moe_router_bias) added for top-k selection only; moe_router_activation sigmoid; router logits computed in FP32 (/need_fp32_gate)."},"moe.norm_topk":{"value":true,"source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5MoEMLP.forward","quote":"router_logits, self.top_k, renormalize=True)","note":"Hard-coded in code; config norm_expert_weight true agrees."},"moe.routed_scale":{"value":3.0,"source":"config","at":"/moe_router_scaling_factor","note":"Applied to routed weights in Step3p5MoEMLP.forward; paper §4.1.2 motivates an explicit routed/shared scale against expert collapse."},"moe.balancing":{"value":"aux-loss-free bias (update rate 0.001 for the first 14.6T, decayed to 0 during annealing) + EP-group balance loss 0.001","source":"report","at":"p.15 §4.2.3","quote":"the bias update rate is 0.001 for the first 14.6T tokens and decays to 0.0 during annealing, and an EP-group balance loss with coefficient 0.001 is applied throughout pre-training.","note":"EP-group loss balances load across expert-parallel rank groups (Eq. 1, §2.2)."},"mtp.depth":{"value":1,"source":"report","at":"p.7 §2.2","quote":"we activate and optimize only MTP-1 in most training stages.","note":"MTP-2 and MTP-3 are cloned from MTP-1 and trained in a lightweight post-training phase; each head is SWA + dense FFN."},"mtp.modules":{"value":1,"source":"config","at":"/num_nextn_predict_layers","note":"Only MTP-1 is trained in pre-training and shipped in the Base checkpoint; MTP-2/3 are cloned from it in post-training."},"mtp.block":{"value":"gated SWA attention + dense SwiGLU FFN; H-Norm(hidden) and E-Norm(next-token embedding) concatenated, then linear projection","source":"report","at":"p.6 Figure 2","quote":"MTP modules use SWA and dense FFNs.","note":"Concat/norm layout from Figure 2; checkpoint has hnorm, enorm, eh_proj, g_proj in layer 45."},"mtp.ffn":{"value":"dense","source":"report","at":"p.7 §2.2","quote":"Each MTP head consists of a SWA and a dense FFN"},"mtp.shares_embedding":{"value":false,"basis":"inferred","source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.45.transformer.shared_head.output.weight\"","note":"Input embedding shared (Figure 2), output head not: the MTP layer has its own 128,896 × 4,096 head (lm_head has 128,815 rows), and with it the layer holds 809,390,336 params, the paper's 0.81B per head."},"mtp.params":{"value":810000000,"source":"report","at":"p.7 §2.2","quote":"Each MTP head consists of a SWA and a dense FFN, adding only 0.81B parameters","note":"Per head (~0.41% of the backbone), including its own output head; the Base checkpoint contains MTP-1 only."},"mtp.loss_weight":{"value":"0.3 in pre-training stage 1; 0.1 in stage 2","source":"report","at":"p.15 §4.2.3","quote":"The MTP loss weight is set to 0.3 in Pretraining Stage 1 and 0.1 in Pre-training Stage 2"},"mtp.released":{"value":true,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.45.eh_proj.weight\"","note":"One MTP layer (index 45; /num_nextn_predict_layers 1)."},"norm.type":{"value":"RMSNorm (1+w gain)","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5RMSNorm.forward","quote":"normed = normed * (self.weight.float() + 1)","note":"Paper Fig. 2: zero-centered RMSNorm throughout; applied regardless of /zero_centered."},"norm.placement":{"value":"pre","source":"modeling_step3p5","at":"modeling_step3p5.py; Step3p5DecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.eps":{"value":1e-05,"source":"configuration_step3p5","at":"configuration_step3p5.py; Step3p5Config.__init__","quote":"rms_norm_eps: float = 1e-5,","note":"config.json does not set rms_norm_eps, so the class default applies."},"ctx.pretrain":{"value":4096,"source":"report","at":"p.14 §4.2.2","quote":"Pre-training Stage 1: Open-domain pre-training (14.6T tokens, 4k context)."},"ctx.final":{"value":32768,"source":"report","at":"p.14 §4.2.2","quote":"then transitions to 1T tokens at 32k context under the same annealed mixture","note":"End of pre-training; mid-training (750B tokens) then extends to 128K. Released Base config: max_seq_len 32768."},"ctx.max":{"value":262144,"source":"card","at":"README.md; 4.1 Technical Specifications","quote":"Context Window | 256K","note":"Base config.json sets max_seq_len 32768."},"ctx.extension":{"value":"last 1T tokens of annealing at 32K; full-attention RoPE θ 10K → 1M, SWA θ stays 10K","source":"report","at":"p.15 §4.2.3","quote":"set θFull = 1,000,000 only for full attention and maintain θSWA = 10,000 for the 32k portion of annealing.","note":"Mid-training goes to 128K with θFull 5M."},"train.tokens":{"value":17600000000000,"source":"report","at":"p.13 §4.2","quote":"Overall, we train on approximately 17.6T tokens for pre-training and 750B tokens for mid-training.","note":"Pre-training only (sum of stages: 14.6T + 2T + 1T); §1 says 17.2T; mid-training adds 750B."},"train.optimizer":{"value":"Muon (Polar Express orthogonalization, 6 iterations)","source":"report","at":"p.11 §4.1.1","quote":"We therefore adopt the Polar Express [73] iteration and run a fixed T =6 steps to balance optimization quality and throughput."},"train.lr_peak":{"value":0.00025,"source":"report","at":"p.15 §4.2.3","quote":"The learning rate is linearly warmed up from 0 to 2.5 × 10−4 over the first 2,000 steps"},"train.lr_min":{"value":2e-05,"source":"report","at":"p.15 §4.2.3","quote":"keep the learning rate fixed at 2 × 10−5 for the 32k portion (1T tokens)."},"train.warmup":{"value":"2,000 steps","source":"report","at":"p.15 §4.2.3","quote":"The learning rate is linearly warmed up from 0 to 2.5 × 10−4 over the first 2,000 steps"},"train.schedule":{"value":"2,000-step linear warmup; cosine 2.5e-4 → 5e-5 over stage 1 (14.6T); cosine 5e-5 → 2e-5 over the 2T 4K anneal; constant 2e-5 for the 1T 32K portion","source":"report","at":"p.15 §4.2.3","quote":"In Pre-training Stage 2, we apply a secondary cosine decay from 5 × 10−5 to 2 × 10−5 over the 4k portion (2T tokens)"},"train.batch":{"value":"4,096 → 16,384 × 4,096 (67.1M tokens) over the first 400B tokens, then constant; 2,048 × 32,768 (67.1M tokens) in the 32K anneal","source":"report","at":"p.15 §4.2.3; p.11 Figure 3","quote":"The global batch size gradually increases from 4096 to 16384 over the first 400B tokens, and keeps 16384 in the remaining training, and is set to 2k for the 32k portion of annealing.","note":"The report does not give the unit; read as sequences because both stages then give the same tokens per step."},"train.batch_tokens":{"value":67108864,"basis":"inferred","source":"report","at":"p.15 §4.2.3","quote":"The global batch size gradually increases from 4096 to 16384 over the first 400B tokens, and keeps 16384 in the remaining training","note":"16,384 sequences × 4,096 tokens (stage 1 context); read as sequences since 2,048 × 32K in the 32K anneal gives the same tokens per step."},"train.weight_decay":{"value":0.1,"source":"report","at":"p.15 §4.2.3","quote":"We use the Muon optimizer [34] throughout pre-training, set weight decay to 0.1"},"train.grad_clip":{"value":1.0,"source":"report","at":"p.15 §4.2.3","quote":"set weight decay to 0.1 and gradeint clip to 1.0."},"train.precision":{"value":"mixed precision; Muon Polar Express iteration (state and intermediates) in FP16","source":"report","at":"p.11–12 §4.1.1","quote":"We therefore cast only the Polar Express iteration (state and intermediates) to float16","note":"Rest of training stays mixed-precision (p.12); BF16 Polar Express caused rare unrecoverable loss spikes; the mixed-precision format is not stated."},"train.stability":{"value":"FP16 Polar Express in Muon; SwiGLU activation clipping in deep MoE layers; zero-centered RMSNorm; QK-norm; head-wise attention gate; per-expert activation/parameter-norm monitoring","source":"report","at":"p.11–13 §4.1; p.31 App. B","quote":"This observation leads us to prefer activation clipping over weight clipping","note":"One transient loss spike over the run (Figure 3)."},"train.objective":{"value":"metadata prefix (content type, language, domain, source) on every sample; its loss masked after ~3.8T tokens; PR-Dialogue template scaffolding masked during annealing","source":"report","at":"p.29 App. A.3","quote":"After an initial phase of approximately 3.8T tokens, we keep M in the context but mask out its positions from the loss"},"train.data":{"value":"StepCrawl web and ePub/PDF, code (OpenCoder filters relaxed to ≤6 violations), GitHub PR/issue/commit (incl. 90B PR-Dialogue tokens), math/STEM, synthetic tool-use and reasoning; annealing upweights code, PR/issue/commit and high-quality knowledge","source":"report","at":"pp.13–14 §4.2.1–4.2.2; App. C","quote":"We anneal the data mixture toward code and PR/Issue/Commit-centric sources, while increasing the share of higher-quality knowledge and reasoning-dense samples."},"train.hardware":{"value":"4,096 NVIDIA H800; 8 GPUs per node, 8×200 Gbps RoCE","source":"report","at":"p.9 §3.1","quote":"Step 3.5 Flash is trained on a large-scale cluster with 4,096 NVIDIA H800 GPUs.","note":"PP8 with VPP, EP8, ZeRO-1 DP (§3.2)."},"train.accelerators":{"value":"4,096 H800","source":"report","at":"p.9 §3.1","quote":"Step 3.5 Flash is trained on a large-scale cluster with 4,096 NVIDIA H800 GPUs."},"train.parallelism":{"value":"PP8 · VPP · EP8 · ZeRO-1 DP","source":"report","at":"p.9 §3.2","quote":"including 8-way pipeline parallelism (PP) [68] with virtual pipeline stages (VPP), and 8-way expert parallelism (EP) [25], and ZeRO-1 Data Parallelism"},"train.stages":{"value":[{"name":"pre-training stage 1","tokens":14600000000000,"seq_len":4096,"lr":0.00025,"batch_sequences":16384,"note":"Open-domain mix; batch ramps from 4,096 over the first 400B tokens; cosine to 5e-5; MTP weight 0.3.","quote":"Pre-training Stage 1: Open-domain pre-training (14.6T tokens, 4k context)."},{"name":"pre-training stage 2 (annealing, 4K)","tokens":2000000000000,"seq_len":4096,"lr":5e-05,"batch_sequences":16384,"note":"Annealed mix; cosine 5e-5 → 2e-5; MTP weight 0.1; expert-bias update rate decays to 0.","quote":"This stage starts with 2T tokens at 4k context"},{"name":"pre-training stage 2 (annealing, 32K)","tokens":1000000000000,"seq_len":32768,"lr":2e-05,"batch_sequences":2048,"note":"Same annealed mix; constant LR; full-attention RoPE θ 1M.","quote":"then transitions to 1T tokens at 32k context under the same annealed mixture to initialize long-context training."}],"source":"report","at":"p.14 §4.2.2; p.15 §4.2.3"},"scaling.ladder":{"value":"30B-A3B (48 layers, d 2048, 1.4T tokens) for attention-layout and data ablations; 100B-A10B (43 layers, d 4096, 250B tokens) for head-wise gate vs sink","source":"report","at":"p.30 Appendix A.4 Table 9; p.35 Appendix C.5","quote":"Table 9: Training configuration for the 100B-A10B and the 30B-A3B architecture ablation suite."},"eval.aa_index":{"value":17,"source":"aa","at":"Artificial Analysis model page /models/step-3-5-flash-0202, Wayback snapshot 2026-10-19","quote":"Step 3.5 Flash scores 17 on the Artificial Analysis Intelligence Index","note":"Score of Step-3.5-Flash, post-trained from this base. Variant: Step 3.5 Flash. AA entry for the original February 2026 release (Step 3.5 Flash 2603 has its own entry). Index v4.3.2."}},"not_disclosed":["train.betas"],"derived":{"params.counted_total":{"value":196955466816,"formula":"embedding + output head + all experts + other weights, stepfun-ai/Step-3.5-Flash-Base@19623ae safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_embedding":{"value":1055252480,"formula":"input embedding + output head, stepfun-ai/Step-3.5-Flash-Base@19623ae safetensors headers"},"params.counted_non_embedding":{"value":195900214336,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, stepfun-ai/Step-3.5-Flash-Base@19623ae safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"params.counted_max_layer":{"value":4656210464,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, stepfun-ai/Step-3.5-Flash-Base@19623ae safetensors headers"},"params.counted_active_non_embedding":{"value":10931407936.0,"formula":"non-expert weights + routed experts × 8/288, excluding embedding and output head, stepfun-ai/Step-3.5-Flash-Base@19623ae safetensors headers; reproduces stated total (backbone) and active (non-embedding)"},"shape.aspect":{"value":91.02222222222223,"formula":"d_model / layers"},"attn.qk_width":{"value":8192,"formula":"heads × head_dim"},"attn.group_size":{"value":8.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.75,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.3125,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":2.75,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":36.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.05612244897959184,"formula":"params.active / params.total"},"scaling.tpp":{"value":89.79591836734694,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1610.0396310376977,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":49152.0,"formula":"49152 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":6511656960.0,"formula":"49152 B × 131072 (Global KV) + 135168 B × min(131072, 512) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":49680.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":4516.363636363636,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":591968814.5454545,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":49152.0,"windows":[[135168.0,512]],"groups":[{"name":"Global KV","bytes_per_entry":49152.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":135168.0,"tokens_per_entry":1,"window":512}],"formula":"Global KV: 12 × (2048 × 16/8) B/entry; SWA KV: 33 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"step-3","name":"Step-3","org":"StepFun","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint (stepfun-ai/step3, a VLM); architecture facts are for the language model.","sources":{"card":{"url":"https://huggingface.co/stepfun-ai/step3/resolve/e2d67fc936dc60f8f606f2429728f1dea503e10d/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/stepfun-ai/step3/resolve/e2d67fc936dc60f8f606f2429728f1dea503e10d/config.json","kind":"config"},"index":{"url":"https://huggingface.co/stepfun-ai/step3/resolve/e2d67fc936dc60f8f606f2429728f1dea503e10d/model.safetensors.index.json","kind":"config"},"configuration_step3":{"url":"https://huggingface.co/stepfun-ai/step3/resolve/e2d67fc936dc60f8f606f2429728f1dea503e10d/configuration_step3.py","kind":"code"},"modeling_step3":{"url":"https://huggingface.co/stepfun-ai/step3/resolve/e2d67fc936dc60f8f606f2429728f1dea503e10d/modeling_step3.py","kind":"code"},"report":{"url":"https://arxiv.org/pdf/2507.19427v1","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20250812171427/https://stepfun.ai/research/en/step3","kind":"blog"}},"facts":{"meta.released":{"value":"2025-07-25","source":"report","at":"p.1 arXiv stamp","quote":"arXiv:2507.19427v1 [cs.LG] 25 Jul 2025","note":"Report predates the weights (release blog dated July 31, 2025)."},"meta.license":{"value":"apache-2.0","source":"card","at":"README.md; front matter","quote":"license: apache-2.0"},"meta.modality":{"value":"text + image","source":"blog","at":"Step3 blog; Pretrain, multimodal pathway","quote":"our 5B-parameter vision encoder extracts dense image features"},"params.total":{"value":321000000000,"source":"card","at":"README.md; Introduction","quote":"with 321B total parameters and 38B active"},"params.active":{"value":38000000000,"source":"card","at":"README.md; Introduction","quote":"with 321B total parameters and 38B active"},"params.note":{"value":"321B includes the 5B vision encoder; the LLM alone is 316B. 38B active per text token.","source":"card","at":"README.md; Step3 model card table","quote":"Total Parameters (LLM) | 316B"},"shape.layers":{"value":61,"source":"config","at":"/text_config/num_hidden_layers"},"shape.d_model":{"value":7168,"source":"config","at":"/text_config/hidden_size"},"shape.vocab":{"value":128815,"source":"config","at":"/text_config/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"index","at":"model.safetensors.index.json; weight_map","quote":"\"lm_head.weight\": \"model-00005.safetensors\"","basis":"inferred","note":"Inferred from the checkpoint: lm_head.weight (shard 5) and model.embed_tokens.weight (shard 1) are separate [128815, 7168] tensors with different bytes; no config key or code states tying (config.json has no tie_word_embeddings)."},"shape.tokenizer":{"value":"DeepSeek-V3 tokenizer","source":"card","at":"README.md; Step3 model card table","quote":"Tokenizer | Deepseek V3"},"attn.kind":{"value":"MFA","source":"card","at":"README.md; Step3 model card table","quote":"Attention Mechanism | MFA","note":"Multi-matrix Factorization Attention: query down-projected 7168 → 2048 (/text_config/share_q_dim), RMSNorm, up-projected to 64×256; one K and one V head shared by all query heads."},"attn.global_layers":{"value":61,"source":"card","at":"README.md; Step3 model card table","quote":"Number of Layers (Dense layer included)|61","note":"Every layer uses full MFA attention (Step3vDecoderLayer)."},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kv_heads":{"value":1,"source":"report","at":"p.2 §2","quote":"Key and a Value head, all with a dimension of 256.","note":"Step3vAttention hardcodes num_key_value_heads = 1 (/text_config/num_attention_groups 1 is not read). The cache is 512 elements per token per layer, matching report Table 3 (1.02e9 bytes at 32K in FP8 = 32,768 × 61 × 512)."},"attn.head_dim":{"value":256,"source":"config","at":"/text_config/head_dim"},"attn.position":{"value":"RoPE","source":"modeling_step3","at":"modeling_step3.py; Step3vAttention.forward","quote":"query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)","note":"All 256 dims rotated; no rope_scaling in config.json."},"attn.rope_theta":{"value":500000,"source":"config","at":"/text_config/rope_theta"},"attn.qk_norm":{"value":false,"source":"modeling_step3","at":"modeling_step3.py; Step3vAttention.forward","quote":"query_states = self.inter_norm(query_states)","note":"The only attention norm is RMSNorm on the 2048-dim query latent before up-projection; per-head Q and K are not normalized."},"attn.bias":{"value":false,"source":"modeling_step3","at":"modeling_step3.py; Step3vAttention.__init__","quote":"self.k_proj = nn.Linear(config.hidden_size, self.head_dim, bias=False)","note":"q_proj, wq, v_proj and o_proj are also bias-free."},"ffn.activation":{"value":"SwiGLU","source":"modeling_step3","at":"modeling_step3.py; Step3vMLP.forward","quote":"down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))","note":"act_fn is SiLU; routed experts use the same form (Step3vMoEMLP.get_expert_output)."},"ffn.dim":{"value":18432,"source":"config","at":"/text_config/intermediate_size","note":"Dense FFN of layers 0–3 and 60."},"ffn.bias":{"value":false,"source":"modeling_step3","at":"modeling_step3.py; Step3vMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.experts":{"value":48,"source":"config","at":"/text_config/moe_num_experts"},"moe.top_k":{"value":3,"source":"config","at":"/text_config/moe_top_k"},"moe.shared":{"value":1,"source":"card","at":"README.md; Step3 model card table","quote":"Number of Shared Experts| 1"},"moe.expert_dim":{"value":5120,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.shared_dim":{"value":5120,"source":"config","at":"/text_config/share_expert_dim"},"moe.dense_layers":{"value":4,"source":"config","at":"config.json text_config.moe_layers_enum","quote":"\"moe_layers_enum\": \"4,5,6,7,","note":"moe_layers_enum lists layers 4–59, so the last layer (60) is dense too: 5 dense layers in total (card)."},"moe.router":{"value":"softmax","source":"modeling_step3","at":"modeling_step3.py; Step3vMoEMLP.forward","quote":"routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)"},"moe.norm_topk":{"value":false,"source":"modeling_step3","at":"modeling_step3.py; Step3vMoEMLP.forward","quote":"routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) # we cast back to the input dtype","note":"Top-k softmax weights are used without renormalization, consistent with /text_config/norm_expert_weight false (which the code does not read)."},"norm.type":{"value":"RMSNorm","source":"modeling_step3","at":"modeling_step3.py; Step3vDecoderLayer.__init__","quote":"self.input_layernorm = Step3vRMSNorm(config.hidden_size, eps=config.rms_norm_eps)"},"norm.placement":{"value":"pre","source":"modeling_step3","at":"modeling_step3.py; Step3vDecoderLayer.forward","quote":"hidden_states = self.input_layernorm(hidden_states)"},"norm.eps":{"value":1e-05,"source":"configuration_step3","at":"configuration_step3.py; Step3TextConfig.__init__","quote":"rms_norm_eps: float = 1e-5,","note":"config.json sets no rms_norm_eps, so the Step3TextConfig default applies."},"ctx.max":{"value":65536,"source":"card","at":"README.md; Step3 model card table","quote":"Max Context Length | 65536","note":"/text_config/max_seq_len and /text_config/max_position_embedding (RoPE cache length) are also 65536."},"train.tokens":{"value":">20T","source":"blog","at":"Pretrain Data","quote":"During pretraining, Step3 processes over 20T text tokens across more than ten languages","note":"Text tokens; 4T image-text mixed tokens are counted separately."},"train.data":{"value":">20T text tokens in >10 languages, 3.7T high-quality tokens reserved for annealing; web and licensed publisher data, MinHash-deduplicated; 4T image-text mixed tokens","source":"blog","at":"Pretrain Data","quote":"of which 3.7T high-quality tokens are reserved for the annealing phases, while incorporating 4T image-text mixed tokens"},"train.objective":{"value":"multimodal: 4T image-text mixed tokens in pretraining; connector and LLM trained on 1.4T multimodal tokens with the 5B vision encoder frozen","source":"blog","at":"Multimodal","quote":"In the second stage, the vision encoder is frozen, and the Connector and the LLM are trained on the full dataset, totaling 1.4T tokens."},"train.stages":{"value":[{"name":"pretraining","note":">20T text tokens in total, including annealing.","quote":"During pretraining, Step3 processes over 20T text tokens"},{"name":"annealing","tokens":3700000000000,"note":"Several annealing phases on high-quality data.","quote":"of which 3.7T high-quality tokens are reserved for the annealing phases"}],"source":"blog","at":"Pretrain Data"}},"not_disclosed":["params.active_non_embedding","moe.balancing","mtp.depth","ctx.pretrain","ctx.final","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip","train.precision"],"derived":{"params.counted_total":{"value":316684620800,"formula":"embedding + output head + all experts + other weights, stepfun-ai/step3@e2d67fc safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_embedding":{"value":1846691840,"formula":"input embedding + output head, stepfun-ai/step3@e2d67fc safetensors headers"},"params.counted_non_embedding":{"value":314837928960,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, stepfun-ai/step3@e2d67fc safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"params.counted_max_layer":{"value":5564628992,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, stepfun-ai/step3@e2d67fc safetensors headers"},"params.counted_active_non_embedding":{"value":37384719360.0,"formula":"non-expert weights + routed experts × 3/48, excluding embedding and output head, stepfun-ai/step3@e2d67fc safetensors headers; reproduces stated total (other modalities) and active (with output head)"},"shape.aspect":{"value":117.50819672131148,"formula":"d_model / layers"},"attn.qk_width":{"value":16384,"formula":"heads × head_dim"},"attn.group_size":{"value":64.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":2.5714285714285716,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.7142857142857143,"formula":"moe.expert_dim / d_model"},"moe.sparsity":{"value":16.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.11838006230529595,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":62464.0,"formula":"62464 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":8187281408.0,"formula":"62464 B × 131072 (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":62464.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":1643.7894736842106,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":215454773.89473686,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":62464.0,"windows":[],"groups":[{"name":"Global KV","bytes_per_entry":62464.0,"tokens_per_entry":1}],"formula":"Global KV: 61 × (512 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"step-5-preview","name":"Step-5-Preview","org":"StepFun","country":"China","kind":"reasoning","weights":"open","note":"No base checkpoint is public; facts describe the released post-trained checkpoint.","sources":{"card":{"url":"https://huggingface.co/stepfun-ai/Step-5-Preview/resolve/364b3b03fed2477aa0953fe855a2c4c8c48ee3cc/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/stepfun-ai/Step-5-Preview/resolve/364b3b03fed2477aa0953fe855a2c4c8c48ee3cc/config.json","kind":"config"},"hf-index":{"url":"https://huggingface.co/stepfun-ai/Step-5-Preview/resolve/364b3b03fed2477aa0953fe855a2c4c8c48ee3cc/model.safetensors.index.json","kind":"config"},"inference-model":{"url":"https://huggingface.co/stepfun-ai/Step-5-Preview/resolve/364b3b03fed2477aa0953fe855a2c4c8c48ee3cc/inference/model.py","kind":"code"},"inference-kernel":{"url":"https://huggingface.co/stepfun-ai/Step-5-Preview/resolve/364b3b03fed2477aa0953fe855a2c4c8c48ee3cc/inference/kernel.py","kind":"code"},"blog":{"url":"https://web.archive.org/web/20261019095405/https://platform.stepfun.ai/blog/inside-step-5-preview","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019110359/https://artificialanalysis.ai/models/step-5","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-10-15","source":"blog","at":"Inside Step 5 Preview: Architecture and Training for Agentic Work; byline","quote":"Oct 15, 2026","note":"Weights and technical blog published the same day; the Hugging Face repo, created 2026-09-20 at API launch, was private until then."},"meta.license":{"value":"other (StepFun Model License 1.0)","source":"card","at":"README.md; License","quote":"is licensed under the StepFun Model License 1.0."},"meta.modality":{"value":"text + image + video","source":"card","at":"README.md; introduction","quote":"accepts text, images, and video","note":"The card's specification table lists Input as \"Text, image\"; the config defines image tokens only."},"params.active":{"value":27000000000,"source":"card","at":"README.md; Introduction","quote":"Step 5 Preview is a sparse Mixture-of-Experts (MoE) model with 600B total parameters and 27B active parameters."},"params.note":{"value":"Rounded publisher figures; inclusion of the 3 MTP layers and the vision encoder is not stated.","source":"card","at":"README.md; Model Specifications","quote":"Total Parameters 600B"},"params.total":{"value":600000000000,"source":"card","at":"README.md; Introduction","quote":"Step 5 Preview is a sparse Mixture-of-Experts (MoE) model with 600B total parameters and 27B active parameters."},"shape.d_model":{"value":4096,"source":"config","at":"/text_config/hidden_size"},"shape.layers":{"value":92,"source":"config","at":"/text_config/num_hidden_layers"},"shape.vocab":{"value":128896,"source":"config","at":"/text_config/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"inference-model","at":"inference/model.py; Step5ForConditionalGenerationConfig.__init__","quote":"kwargs.setdefault(\"tie_word_embeddings\", False)","note":"Step5ForCausalLM builds a separate lm_head (_tied_weights_keys empty); config.json has no tie_word_embeddings key."},"attn.bias":{"value":false,"source":"inference-model","at":"inference/model.py; Step5Attention.__init__","quote":"config.hidden_size, self.kv_size, bias=False, **factory","note":"q, k, v, o and gate projections are all built with bias=False."},"attn.gate":{"value":"head-wise sigmoid on the attention output, before o_proj","source":"inference-model","at":"inference/model.py; Step5Attention.forward","quote":"gate = self.g_proj(hidden_states).sigmoid().unsqueeze(-1)","note":"Enabled by /text_config/use_head_wise_attn_gate; applies to both layer types."},"attn.global_layers":{"value":23,"source":"config","at":"/text_config/layer_types/0:92 | count full_attention"},"attn.head_dim":{"value":192,"source":"config","at":"/text_config/head_dim","note":"Step5Attention reads /text_config/head_dim for both layer types."},"attn.heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads"},"attn.kind":{"value":"SWA, DSA (Sparse GQA)","source":"blog","at":"Architectural Design Philosophy","quote":"we propose Sparse GQA, introducing a sparse indexing mechanism into standard GQA inspired by DSA","note":"Sparse GQA runs on the \"full_attention\" layers (inference/model.py Step5Config.is_sparse_layer); the others are 512-token sliding-window GQA."},"attn.kv_heads":{"value":4,"source":"config","at":"/text_config/num_attention_groups","note":"Step5Attention reads /text_config/num_attention_groups for both layer types."},"attn.local_heads":{"value":64,"source":"config","at":"/text_config/num_attention_heads","note":"Step5Attention reads num_attention_heads, num_attention_groups and head_dim for both layer types."},"attn.local_kv_heads":{"value":4,"source":"config","at":"/text_config/num_attention_groups"},"attn.local_head_dim":{"value":192,"source":"config","at":"/text_config/head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/text_config/rope_theta/0","note":"Per-layer list; 10000 on every sliding_attention layer."},"attn.indexer_dim":{"value":256,"source":"config","at":"/text_config/sparse_config/proxy_dim","note":"One indexer key (sparse_indexer_num_k_heads 1) per 8-token region, not per token; KV fields are computed from attn.kv_groups."},"attn.kv_groups":{"value":[{"name":"DSA K/V","layers":23,"elements":1536,"tokens_per_entry":1,"note":"4 KV heads × 192 dims, K and V.","quote":"retain full K/V and completed-region summaries"},{"name":"DSA indexer region summaries","layers":23,"elements":256,"tokens_per_entry":8,"note":"proxy_dim 256 per 8-token region (region_block_size); stored FP8 in kernel.py DSALayerCache.","quote":"retain full K/V and completed-region summaries"},{"name":"SWA K/V","layers":69,"elements":1536,"tokens_per_entry":1,"window":512,"note":"4 KV heads × 192 dims, K and V.","quote":"Ring buffer of the last ``window`` tokens for one sliding-attention layer."}],"source":"inference-model","at":"inference/model.py; Step5Cache, DenseLayerCache"},"attn.layout":{"value":"3 SWA : 1 DSA","source":"config","at":"config.json; /text_config/layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\" ]","note":"Repeats 23 times; the last main layer (91) is DSA. Ordered 92-layer backbone from /text_config/layer_types; auxiliary modules are excluded. The array has 95 entries; only the first num_hidden_layers=92 are backbone layers.","basis":"inferred","sequence":["SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA","SWA","SWA","SWA","DSA"]},"attn.local_layers":{"value":69,"source":"config","at":"/text_config/layer_types/0:92 | count sliding_attention","note":"/text_config/layer_types has 95 entries; the last 3 are the MTP layers."},"attn.position":{"value":"partial RoPE (DSA, 64 of 192 dims), RoPE (SWA)","source":"inference-model","at":"inference/model.py; Step5Attention.__init__","quote":"rotary_span = int(round(self.head_dim * config.partial_rotary_factors[layer_idx]))","note":"/text_config/partial_rotary_factors is 1/3 on full_attention layers and 1.0 on sliding_attention layers."},"attn.qk_norm":{"value":true,"source":"inference-model","at":"inference/model.py; Step5Attention.forward","quote":"query = self._apply_rope(self.q_norm(query), positions)"},"attn.rope_fraction":{"value":0.3333333333333333,"source":"config","at":"/text_config/partial_rotary_factors/3","note":"Full-attention (DSA) layers: 64 of 192 dims. SWA layers rotate all dims (1.0)."},"attn.rope_theta":{"value":10000000,"source":"config","at":"/text_config/rope_theta/3","note":"Full-attention (DSA) layers; rope_scaling (rope_type llama3, factor 1.0, original_max_position_embeddings 1048576, low_freq_factor 1.0, high_freq_factor 32.0) applies only to yarn_only_types full_attention and is an identity at factor 1.0."},"attn.sparse":{"value":"DSA indexer (16 heads, proxy dim 256) scores 8-token region summaries; each query attends to its top-512 regions","source":"inference-kernel","at":"inference/kernel.py; REGION_BLOCK_SIZE","quote":"Step-5 compresses each region of 8 consecutive tokens into one summary vector.","note":"/text_config/sparse_config (enabled, apply_to_layer_types full_attention, attention_impl sparse_gqa, compression_method csa_block_compress): topk 512 regions of region_block_size 8; sparse_indexer_num_heads 16 in num_provider_groups 4 aligned with KV groups; sparse_indexer_num_k_heads 1; proxy_dim 256; sparse_indexer_use_rope with sparse_indexer_rope_dim 32; sparse_indexer_q_norm_type rmsnorm, sparse_indexer_k_norm_type layernorm, sparse_indexer_csa_z_norm_type none; sparse_indexer_softmax_variant ssmax and sparse_indexer_ssmax_s_granularity are unused (indexer scores with weighted ReLU)."},"attn.window":{"value":512,"source":"config","at":"/text_config/sliding_window"},"ffn.activation":{"value":"SwiGLU","source":"inference-model","at":"inference/model.py; clamped_swiglu","quote":"silu(gate).clamp(max=limit) * up.clamp(-limit, limit)","note":"See ffn.clamp."},"ffn.bias":{"value":false,"source":"inference-model","at":"inference/model.py; Step5MLP.__init__","quote":"self.gate_proj = ColumnParallelLinear(config.hidden_size, intermediate_size, tp_size=tp, bias=False, **factory)"},"ffn.clamp":{"value":"SwiGLU limit 7 (silu(gate) ≤ 7, up in [−7, 7]) on all dense, shared and routed FFNs","source":"inference-model","at":"inference/model.py; clamped_swiglu","quote":"activated = torch.nn.functional.silu(gate.float()).clamp(max=limit)","note":"Released inference code; the docstring motivates it by FP8 expert-GEMM range, and use in training is not stated. /text_config/swiglu_limits and swiglu_limits_shared are 7.0 on all layers."},"ffn.dim":{"value":13824,"source":"config","at":"/text_config/intermediate_size","note":"Dense FFN in layers 0–2 and the last layer (91)."},"moe.dense_layers":{"value":3,"source":"config","at":"/text_config/moe_layer_list/0","note":"MoE layers are 3–90 (use_moe true); layer 91 is also dense."},"moe.expert_dim":{"value":1536,"source":"config","at":"/text_config/moe_intermediate_size"},"moe.experts":{"value":352,"source":"config","at":"/text_config/moe_num_experts"},"moe.norm_topk":{"value":true,"source":"config","at":"/text_config/norm_expert_weight","note":"Step5MoE.route renormalizes the unbiased top-k sigmoid scores."},"moe.routed_scale":{"value":3.0,"source":"config","at":"/text_config/moe_router_scaling_factor","note":"Step5MoE.route returns weights * scaling_factor."},"moe.router":{"value":"sigmoid","source":"inference-model","at":"inference/model.py; Step5MoE.forward","quote":"gate_prob = torch.sigmoid(logits)","note":"FP32 router (need_fp32_gate); selection-only expert bias (use_moe_router_bias) added for top-k and removed from the weights."},"moe.shared":{"value":1,"source":"inference-model","at":"inference/model.py; Step5DecoderLayer.__init__","quote":"self.share_expert = Step5MLP("},"moe.shared_dim":{"value":1536,"source":"config","at":"/text_config/share_expert_dim"},"moe.top_k":{"value":8,"source":"config","at":"/text_config/moe_top_k","note":"Step5Config drops the duplicate num_experts_per_tok and reads moe_top_k."},"mtp.depth":{"value":0,"source":"blog","at":"Agentic RL; High-Throughput and Low-Redundancy Infrastructure","quote":"an MTP-3 head trained after SFT provides roughly 30% end-to-end speedup","note":"The released 3-layer MTP head was trained after SFT for speculative decoding, not during pretraining."},"mtp.released":{"value":true,"source":"hf-index","at":"model.safetensors.index.json; weight_map","quote":"\"model.layers.94.eh_proj.weight\"","note":"Three MTP layers (92–94, /text_config/num_nextn_predict_layers), trained after SFT (see mtp.depth)."},"norm.placement":{"value":"pre","source":"inference-model","at":"inference/model.py; Step5DecoderLayer.forward","quote":"hidden = self.input_layernorm(residual)"},"norm.eps":{"value":1e-05,"source":"config","at":"/text_config/rms_norm_eps"},"norm.residual":{"value":"FP32 residual stream (hidden states and their gradients)","source":"blog","at":"Training Stability; FP32 hidden state","quote":"we find that using FP32 for hidden states [8] and their corresponding gradients works well","note":"Inference code keeps it via /text_config/fp32_residual_connection (Step5DecoderLayer, Step5Model.forward)."},"norm.type":{"value":"RMSNorm (1+w gain)","source":"inference-model","at":"inference/model.py; Step5RMSNorm","quote":"Zero-centered RMSNorm: ``x / rms(x) * (1 + w)``, reduced in fp32.","note":"norm_dtype float32."},"train.precision":{"value":"FP8 mixed precision","source":"blog","at":"Training Stability","quote":"we adopt a mixed-precision FP8 strategy to train Step 5 Preview"},"train.stability":{"value":"FP32 hidden states and gradients; FP32 RMSNorm gains; fine-grained gradient clamp (top 0.00004% of gradient values per layer)","source":"blog","at":"Training Stability; Fine-grained gradient clamp","quote":"We find clamping only 0.00004% of gradient values per layer leads to excellent gradient spike suppression","note":"FP32 gains replace BF16 because zero-centered gains near −1 oscillated in BF16."},"ctx.max":{"value":1048576,"source":"config","at":"/text_config/max_position_embeddings","note":"Card: \"It supports up to 1 million tokens of context\"."},"eval.aa_index":{"value":44,"source":"aa","at":"Artificial Analysis model page /models/step-5, Wayback snapshot 2026-10-19","quote":"Step 5 Preview scores 44 on the Artificial Analysis Intelligence Index","note":"Variant: Step 5 Preview. Index v4.3.2."}},"not_disclosed":["params.active_non_embedding","moe.balancing","ctx.pretrain","ctx.final","train.tokens","train.optimizer","train.lr_peak","train.lr_min","train.schedule","train.warmup","train.batch","train.weight_decay","train.betas","train.grad_clip"],"derived":{"params.counted_total":{"value":599054343616,"formula":"embedding + output head + all experts + other weights, stepfun-ai/Step-5-Preview@364b3b0 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_embedding":{"value":1583874048,"formula":"input embedding + output head, stepfun-ai/Step-5-Preview@364b3b0 safetensors headers"},"params.counted_non_embedding":{"value":597470469568,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, stepfun-ai/Step-5-Preview@364b3b0 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"params.counted_max_layer":{"value":6790260256,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, stepfun-ai/Step-5-Preview@364b3b0 safetensors headers"},"params.counted_active_non_embedding":{"value":26105601472.0,"formula":"non-expert weights + routed experts × 8/352, excluding embedding and output head, stepfun-ai/Step-5-Preview@364b3b0 safetensors headers; reproduces stated total (backbone) and active (with embedding)"},"shape.aspect":{"value":44.52173913043478,"formula":"d_model / layers"},"attn.qk_width":{"value":12288,"formula":"heads × head_dim"},"attn.group_size":{"value":16.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":3.375,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":0.375,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":44.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.045,"formula":"params.active / params.total"},"kv.growth_per_token":{"value":72128.0,"formula":"70656 B / 1 tokens (DSA K/V) + 11776 B / 8 tokens (DSA indexer region summaries); uniform BF16, payload only; quantization overhead excluded. DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":9562488832.0,"formula":"70656 B × 131072 (DSA K/V) + 11776 B × floor(131072 / 8) (DSA indexer region summaries) + 211968 B × min(131072, 512) (SWA K/V); uniform BF16, payload only; quantization overhead excluded. DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":72956.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":2702.074074074074,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":354166253.037037,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":72128.0,"windows":[[211968.0,512]],"groups":[{"name":"DSA K/V","bytes_per_entry":70656.0,"tokens_per_entry":1},{"name":"DSA indexer region summaries","bytes_per_entry":11776.0,"tokens_per_entry":8},{"name":"SWA K/V","bytes_per_entry":211968.0,"tokens_per_entry":1,"window":512}],"formula":"DSA K/V: 23 × (1536 × 16/8) B/entry; DSA indexer region summaries: 23 × (256 × 16/8) B/entry; SWA K/V: 69 × (1536 × 16/8) B/entry","formula_version":"kv-2"}},{"id":"trinity-large","name":"Trinity-Large-Base","org":"Arcee AI","country":"United States","kind":"base","weights":"open","note":null,"sources":{"card":{"url":"https://huggingface.co/arcee-ai/Trinity-Large-Base/resolve/b2a665b40e5d67b05fb164205cf63155f7958e2a/README.md","kind":"model-card"},"config":{"url":"https://huggingface.co/arcee-ai/Trinity-Large-Base/resolve/b2a665b40e5d67b05fb164205cf63155f7958e2a/config.json","kind":"config"},"modeling_afmoe":{"url":"https://raw.githubusercontent.com/huggingface/transformers/c633787a7acb9bd11e57cd8d599be2b767bbdbaa/src/transformers/models/afmoe/modeling_afmoe.py","kind":"code"},"report":{"url":"https://raw.githubusercontent.com/arcee-ai/trinity-large-tech-report/51bc374cbf4aa5f3ece618edd3e3488455664f77/Arcee%20Trinity%20Large.pdf","kind":"paper"},"blog":{"url":"https://web.archive.org/web/20261001124052/https://www.arcee.ai/blog/trinity-large","kind":"blog"},"aa":{"url":"https://web.archive.org/web/20261019110527/https://artificialanalysis.ai/models/trinity-large-thinking","kind":"leaderboard"}},"facts":{"meta.released":{"value":"2026-01-27","source":"blog","at":"Trinity Large blog; header","quote":"Jan 27, 2026"},"meta.license":{"value":"other (OpenMDW-1.1)","source":"card","at":"README.md; YAML front matter","quote":"license_name: openmdw-1.1"},"meta.modality":{"value":"text","basis":"inferred","source":"config","at":"config.json; /architectures/0","quote":"AfmoeForCausalLM","note":"Causal-LM class with no vision config."},"params.total":{"value":398000000000,"source":"card","at":"README.md; Architecture table","quote":"Total parameters | ~398B","note":"Report rounds to 400B total."},"params.active":{"value":13000000000,"source":"report","at":"p.1 Abstract","quote":"with 400B total parameters and 13B activated per token."},"params.note":{"value":"Card gives ~398B total, report 400B; embedding scope not stated.","source":"card","at":"README.md; Introduction","quote":"It is a 398B-parameter sparse Mixture-of-Experts (MoE) model with approximately 13B active parameters per token."},"shape.d_model":{"value":3072,"source":"config","at":"/hidden_size"},"shape.layers":{"value":60,"source":"config","at":"/num_hidden_layers"},"shape.vocab":{"value":200192,"source":"config","at":"/vocab_size"},"shape.tied_embeddings":{"value":false,"source":"config","at":"/tie_word_embeddings"},"shape.tokenizer":{"value":"byte-level BPE, 200K vocab trained on ~10B tokens; DeepSeek-V3-style pretokenizer with right-aligned 3-digit number chunks and extended script isolation","source":"report","at":"p.2 §2.1; p.3 §2.1.1","quote":"We train a custom 200,000-token BPE [Shibata et al., 1999] vocabulary using the tokenizers library"},"attn.kind":{"value":"GQA, SWA","source":"report","at":"p.4 §2.2","quote":"We combine grouped-query attention (GQA) [Ainslie et al., 2023], QK-normalization (QK-norm) [Henry et al., 2020], gated attention"},"attn.layout":{"value":"3 SWA : 1 global","source":"config","at":"config.json; /layer_types","quote":"\"layer_types\": [ \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\", \"sliding_attention\", \"sliding_attention\", \"sliding_attention\", \"full_attention\" ]","note":"Config global_attn_every_n_layers 4. Ordered 60-layer backbone from /layer_types; auxiliary modules are excluded.","basis":"inferred","sequence":["SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA","SWA","SWA","SWA","GQA"]},"attn.global_layers":{"value":15,"source":"config","at":"/layer_types | count full_attention"},"attn.local_layers":{"value":45,"source":"config","at":"/layer_types | count sliding_attention"},"attn.heads":{"value":48,"source":"config","at":"/num_attention_heads"},"attn.kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.window":{"value":4096,"source":"config","at":"/sliding_window"},"attn.local_heads":{"value":48,"source":"config","at":"/num_attention_heads","note":"AfmoeAttention uses the same projections for both layer types; only RoPE and the window differ."},"attn.local_kv_heads":{"value":8,"source":"config","at":"/num_key_value_heads"},"attn.local_head_dim":{"value":128,"source":"config","at":"/head_dim"},"attn.local_rope_theta":{"value":10000,"source":"config","at":"/rope_theta","note":"Only sliding_attention layers apply RoPE; rope_scaling null."},"attn.position":{"value":"RoPE (SWA), NoPE (global)","source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeAttention.forward","quote":"if self.is_local_attention:","note":"RoPE applied only in sliding_attention layers (config rope_theta 10,000)."},"attn.qk_norm":{"value":true,"source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeAttention.__init__","quote":"self.q_norm = AfmoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)","note":"Per-head RMSNorm on Q and K, all layers."},"attn.gate":{"value":"elementwise sigmoid gate σ(W_G x) on the SDPA output before o_proj, all layers","source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeAttention.forward","quote":"output = output * torch.sigmoid(gate_states)","note":"W_G is d × d (one gate per head channel), computed from the attention input (report eq. 11–14)."},"attn.bias":{"value":false,"source":"report","at":"p.2 §2","quote":"with no biases on the linear layers."},"ffn.activation":{"value":"SwiGLU","source":"report","at":"p.6 §2.3","quote":"We use the SwiGLU [Shazeer, 2020] activation function as the nonlinearity.","note":"Config hidden_act silu."},"ffn.dim":{"value":12288,"source":"config","at":"/intermediate_size","note":"Dense FFN in the first 6 layers."},"ffn.bias":{"value":false,"source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeMLP.__init__","quote":"self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)"},"moe.experts":{"value":256,"source":"config","at":"/num_experts"},"moe.top_k":{"value":4,"source":"config","at":"/num_experts_per_tok"},"moe.shared":{"value":1,"source":"config","at":"/num_shared_experts"},"moe.expert_dim":{"value":3072,"source":"config","at":"/moe_intermediate_size"},"moe.shared_dim":{"value":3072,"source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeSparseMoeBlock.__init__","quote":"self.shared_experts = AfmoeMLP(config, config.moe_intermediate_size * config.num_shared_experts)","basis":"inferred","note":"moe_intermediate_size 3072 × num_shared_experts 1."},"moe.dense_layers":{"value":6,"source":"config","at":"/num_dense_layers","note":"Raised from 3 to 6 after early unstable runs (report §6)."},"moe.router":{"value":"normalized sigmoid","source":"report","at":"p.6 §2.3 Routing","quote":"We use normalized sigmoid routing scores instead of softmax-based routing","note":"Config score_func sigmoid. Top-k chosen on score + expert bias; gates use unbiased scores."},"moe.norm_topk":{"value":true,"source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeTokenChoiceRouter.forward","quote":"top_scores = top_scores / denominator","note":"Config route_norm true."},"moe.routed_scale":{"value":2.448,"source":"report","at":"p.12 Table 2","quote":"Route scale 2.826 2.826 2.448","note":"Config route_scale 2.448; applied after top-k renormalization (AfmoeTokenChoiceRouter.forward)."},"moe.balancing":{"value":"SMEBU expert bias (tanh soft-clamped, momentum; λ=5e-4, β=0.5, κ=2) + sequence-wise aux loss 1e-4","source":"report","at":"p.7 §2.3; p.15 §6","quote":"For our Trinity Large run, we use SMEBU with λ = 5 × 10−4 , β = 0.5, and κ = 2.","note":"Sequence-wise loss weight from §6: 1 × 10^-4; config load_balance_coeff 5e-5 is not cited by the report."},"norm.type":{"value":"RMSNorm","source":"report","at":"p.8 §2.4","quote":"For layer normalization, we use a simplified depth-scaled sandwich norm"},"norm.placement":{"value":"sandwich","source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeDecoderLayer.forward","quote":"hidden_states = self.post_attention_layernorm(hidden_states)","note":"RMSNorm on sublayer input and output, both inside the residual branch; final RMSNorm before the LM head."},"norm.eps":{"value":1e-05,"source":"config","at":"/rms_norm_eps"},"norm.residual":{"value":"depth-scaled sandwich norm: output-RMSNorm gains initialized to 1/√L","source":"report","at":"p.8 §2.4","quote":"We depth-scale the RMSNorm gain parameters for the second RMSNorm in each layer."},"norm.embedding":{"value":"× sqrt(d_model)","source":"modeling_afmoe","at":"modeling_afmoe.py; AfmoeModel.forward","quote":"hidden_states = hidden_states * (self.config.hidden_size**0.5)","note":"Gated by config mup_enabled (true); report eq. 39."},"ctx.pretrain":{"value":8192,"source":"report","at":"p.12 §3.4.1","quote":"We use local/global attention with sliding-window size for the local layers set to 4096 and pre-training sequence length 8192."},"ctx.final":{"value":262144,"source":"report","at":"p.13 §3.5","quote":"We trained Trinity Large at a sequence length of 256K, targeting a final context window size of 256K","note":"Card states Base includes context extension but also lists no tuning beyond 8K; config max_position_embeddings is 262,144."},"ctx.max":{"value":524288,"source":"report","at":"p.13 §3.4.2","quote":"We train to 256k context for inference at 512k."},"ctx.extension":{"value":"single stage at 256K from the final pretrained checkpoint; only global (NoPE) layers extended, SWA window 4096 and RoPE unchanged; ~117B-token long-context mix; LR cosine continued 1/10 → 1/20 of peak; CP 4","source":"report","at":"p.13 §3.5","quote":"Adjusting only the global layers resulted in a much quicker loss recovery"},"train.tokens":{"value":17000000000000,"source":"report","at":"p.1 Abstract","quote":"Trinity Large was pre-trained on 17 trillion tokens.","note":"Excludes context extension (blog: 33-day pretraining figure excludes it)."},"train.optimizer":{"value":"Muon + AdamW (Muon for hidden layers, AdamW for embedding and output; Muon LR × max(1, √(fan_out/fan_in)))","source":"report","at":"p.12 §3.4.2","quote":"we use the Muon optimizer [Jordan et al., 2024a] for hidden layers and the AdamW optimizer [Loshchilov and Hutter, 2019] for the embedding and output layers.","note":"Distributed Muon from the Dion repository; expert gradients orthogonalized batched per expert (§3.3)."},"train.lr_peak":{"value":0.0008,"source":"report","at":"p.13 §3.4.2","quote":"The peak learning rates are 8.0 × 10−4 for Muon and 2.0 × 10−4 for AdamW.","note":"Muon; AdamW peak 2.0e-4."},"train.lr_min":{"value":4e-05,"source":"report","at":"p.13 §3.4.2","quote":"we continue cosine decay from 10 1 1 to 20 of the peak learning rate","note":"Muon, end of context extension (1/20 of peak); end of pretraining 8.0e-5. AdamW: 2.0e-5, then 1.0e-5."},"train.schedule":{"value":"WSD: 2000-step linear warmup, constant, cosine decay to 1/10 of peak; context extension continues cosine 1/10 → 1/20","source":"report","at":"p.13 §3.4.2","quote":"During the decay phase, we use a cosine decay to 10 1 of the peak learning rate","note":"The 10T TrueBase checkpoint has no LR anneal (blog), so decay starts after 10T."},"train.warmup":{"value":"2000 steps","source":"report","at":"p.13 §3.4.2","quote":"Trinity Large. We use a linear warmup of 2000 steps."},"train.batch":{"value":"12,288 × 8,192 (101M tokens), raised to 16,384 × 8,192 (134M tokens) after 4.9T tokens","source":"report","at":"p.13 §3.4.2; p.1 Figure 1","quote":"We train with global batch size 12288 at sequence length 8192, and increase the global batch size to 16384 after crossing 4.9T tokens"},"train.batch_tokens":{"value":134217728,"source":"report","at":"p.13 §3.4.2","quote":"We train with global batch size 12288 at sequence length 8192, and increase the global batch size to 16384 after crossing 4.9T tokens","basis":"inferred","note":"16,384 sequences × 8192 tokens, after the step-up at 4.9T (12,288 × 8192 = 100,663,296 before)."},"train.precision":{"value":"BF16 (MXFP8 linear and grouped-GEMM kernels disabled after early unstable runs)","source":"report","at":"p.15 §6","quote":"We disabled our MXFP8 kernels for linear layers and grouped GEMMs, falling back to BF16."},"train.init":{"value":"truncated normal σ=0.009 on [−3σ, 3σ] for all trainable parameters (σ = 0.5/√d); output-norm gains 1/√L","source":"report","at":"p.12 §3.4.1; p.8 §2.5","quote":"We initialize all trainable parameters from a truncated normal distribution with standard deviation 0.009."},"train.stability":{"value":"QK-norm; sigmoid attention-output gate; depth-scaled sandwich norm; z-loss 1e-6 (introduced mid-training); sequence-wise aux loss 1e-4; SMEBU; 6 dense layers; intra-document masking; BF16; random sequential document buffer from phase 3","source":"report","at":"pp.15–16 §6; p.9 §3.2","quote":"and had to reduce the weight to 1 × 10−6 , as larger values destabilized the network.","note":"A bug left z-loss at 0 until it was introduced mid-run; the six §6 fixes were applied together after collapsed-expert restarts."},"train.masking":{"value":"intra-document masking","source":"report","at":"p.9 §3.2","quote":"but Trinity Large was pre-trained using intra-doc attention masking."},"train.data":{"value":"DatologyAI 20T-token mix in 3 phases (13T/4T/3T), shifting from web toward code, math, STEM; >8T synthetic (~6.5T rephrased web, ~1T multilingual, ~800B code); 14 non-English languages","source":"report","at":"p.9 §3.1","quote":"DatologyAI generated over 8 trillion tokens of synthetic web, code, and STEM data.","note":"13T/4T/3T is the 20T mix; the report says the 17T were sampled 13:4:3 from it, the blog gives trained phases of 10T, 4T and 3T."},"train.hardware":{"value":"2048 B300; HSDP (FSDP group 128) + EP 8; pretraining 33 days","source":"blog","at":"Trinity Large blog; Training at scale","quote":"The full pretraining run finished in 33 days.","note":"Report §3.3: 2048 B300, FSDP group 128, EP 8, CP 4 for context extension."},"train.accelerators":{"value":"2,048 B300","source":"card","at":"README.md; Infrastructure","quote":"Hardware: 2,048 NVIDIA B300 GPUs"},"train.parallelism":{"value":"EP8 · HSDP (FSDP group 128)","source":"report","at":"p.11 §3.3 Infrastructure","quote":"For all models, we set the FSDP group size to 128. For Trinity Large, we set the EP group size to 8.","note":"Context extension added CP4."},"train.stages":{"value":[{"name":"phase 1","seq_len":8192,"lr":0.0008,"batch_sequences":12288,"note":"Report: 13T of the 20T mix, from which the 17T were sampled 13:4:3; blog: 10T. Warmup then constant LR; batch 16,384 after 4.9T.","quote":"The 17 trillion tokens that Trinity Large was trained on were sampled proportionally from the 20T mix (i.e. a 13:4:3 ratio across the three phases)."},{"name":"phase 2","seq_len":8192,"batch_sequences":16384,"note":"Report: 4T of the 20T mix; blog: 4T. Shifted toward code, math and STEM.","quote":"The 20 trillion mix was distributed across 13T tokens in phase 1, 4T tokens in phase 2, and 3T tokens in phase 3."},{"name":"phase 3","seq_len":8192,"batch_sequences":16384,"note":"Report: 3T of the 20T mix; blog: 3T. Random sequential document buffer enabled; LR ends at 1/10 of peak.","quote":"we introduce a document buffer mid-way through the Trinity Large training run, right before beginning phase 3."},{"name":"context extension","seq_len":262144,"lr":8e-05,"note":"~117B-token long-context dataset; cosine continued to 4.0e-5 (Muon).","quote":"Our long-context extension dataset comprises approximately 117 billion tokens across 35.6 million documents."}],"source":"report","at":"p.9 §3.1–3.2; p.13 §3.4.2–3.5"},"scaling.parametrization":{"value":"Muon LR × max(1, √(fan_out/fan_in)) for width transfer; embedding ×√d; init σ = 0.5/√d","source":"report","at":"p.12 §3.4.2 eq. 41; p.8 §2.5 eqs. 38–39","quote":"Empirically, we observe that this adjustment enables optimal learning rate transfer when scaling model width, for Muon.","note":"The report does not name a parametrization; config mup_enabled gates the √d embedding scale."},"scaling.ladder":{"value":"Trinity Nano (6B-A1B, 56 layers, d 1024) and Trinity Mini (26B-A3B, 32 layers, d 2048), 10T tokens each","source":"report","at":"p.2 §1; p.12 Table 2","quote":"serve as steps in our scaling ladder that validate the data pipeline, architecture, training recipe, and infrastructure required for Trinity Large."},"eval.aa_index":{"value":11,"source":"aa","at":"Artificial Analysis model page /models/trinity-large-thinking, Wayback snapshot 2026-10-19","quote":"Trinity Large Thinking scores 11 on the Artificial Analysis Intelligence Index","note":"Score of Trinity-Large-Thinking, post-trained from this base. Variant: Trinity Large Thinking. Index v4.3.2."}},"not_disclosed":["mtp.depth","train.weight_decay","train.betas","train.grad_clip","params.active_non_embedding"],"derived":{"params.counted_total":{"value":398635286016,"formula":"embedding + output head + all experts + other weights, arcee-ai/Trinity-Large-Base@b2a665b safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_embedding":{"value":1229979648,"formula":"input embedding + output head, arcee-ai/Trinity-Large-Base@b2a665b safetensors headers"},"params.counted_non_embedding":{"value":397405306368,"formula":"dense + routed backbone weights; excludes input embedding, output head, lookup tables, MTP and other modalities, arcee-ai/Trinity-Large-Base@b2a665b safetensors headers; reproduces stated total (backbone) and active (with output head)"},"params.counted_max_layer":{"value":7339782656,"formula":"max(sum of dense and routed weights sharing a full layers.<index> tensor path), all experts included, arcee-ai/Trinity-Large-Base@b2a665b safetensors headers"},"params.counted_active_non_embedding":{"value":12141706752.0,"formula":"non-expert weights + routed experts × 4/256, excluding embedding and output head, arcee-ai/Trinity-Large-Base@b2a665b safetensors headers; reproduces stated total (backbone) and active (with output head)"},"shape.aspect":{"value":51.2,"formula":"d_model / layers"},"attn.qk_width":{"value":6144,"formula":"heads × head_dim"},"attn.group_size":{"value":6.0,"formula":"heads / kv_heads"},"ffn.mult":{"value":4.0,"formula":"ffn.dim / d_model"},"moe.granularity":{"value":1.0,"formula":"moe.expert_dim / d_model"},"attn.swa_ratio":{"value":3.0,"formula":"attn.local_layers / attn.global_layers"},"moe.sparsity":{"value":64.0,"formula":"experts / top_k"},"moe.active_fraction":{"value":0.032663316582914576,"formula":"params.active / params.total"},"scaling.tpp":{"value":42.71356783919598,"formula":"train.tokens / params.total"},"scaling.tpp_active":{"value":1400.1326458654369,"formula":"train.tokens / params.counted_active_non_embedding"},"kv.growth_per_token":{"value":61440.0,"formula":"61440 B / 1 tokens (Global KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context":{"value":8808038400.0,"formula":"61440 B × 131072 (Global KV) + 184320 B × min(131072, 4096) (SWA KV); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token":{"value":67200.0,"formula":"kv.bytes_at_context / 131072 tokens; uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_per_token_per_active":{"value":5169.2307692307695,"formula":"kv.bytes_per_token (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"},"kv.bytes_at_context_per_active":{"value":677541415.3846154,"formula":"kv.bytes_at_context (131072 tokens) / (params.active / 1e9); uniform BF16, payload only; quantization overhead excluded. Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"}},"kv":{"per_token":61440.0,"windows":[[184320.0,4096]],"groups":[{"name":"Global KV","bytes_per_entry":61440.0,"tokens_per_entry":1},{"name":"SWA KV","bytes_per_entry":184320.0,"tokens_per_entry":1,"window":4096}],"formula":"Global KV: 15 × (2048 × 16/8) B/entry; SWA KV: 45 × (2048 × 16/8) B/entry","formula_version":"kv-2"}}]}