Qwen3.8-27B KV cache VRAM llama.cpp --cache-type-k / --cache-type-v Model 64 layers, hybrid 16 x (3 x Gated DeltaNet + 1 x Gated Attention) Growing KV only on 16 full-attention layers GQA: 24 Q heads, 4 KV heads, head dim 256 elem_k = elem_v = 16 * 4 * 256 = 16384 / token Formulas bytes/token = 16384 * (B_k + B_v) KiB/token = 16 * (B_k + B_v) GiB = ctx * KiB/token / 1048576 GiB = ctx * 16384 * (B_k + B_v) / 1073741824 Not included Gated DeltaNet recurrent state: fixed ~80-150 MiB, does not scale with ctx MTP draft KV (--spec-type draft-mtp): extra cache, same types via -ctkd/-ctvd compute / graph buffers, driver, desktop compositor llama.cpp alignment (log line 'KV self size = ... MiB' is authoritative) Notes Allowed types: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1 f16 and bf16 are the same size; q4_0 and iq4_nl are the same size Quantized V requires flash-attn = on Weight GGUF quant (Q4_K_S, IQ4_XS, ...) does not change this table Bytes per element type B/elem block f32 4.0 — f16 2.0 — bf16 2.0 — q8_0 1.0625 34 B / 32 q5_1 0.75 24 B / 32 q5_0 0.6875 22 B / 32 q4_1 0.625 20 B / 32 q4_0 0.5625 18 B / 32 iq4_nl 0.5625 18 B / 32 Memory layout k and v are separate ggml_tensor buffers, not one mixed struct ggml_tensor holds type + shape + strides; data is packed blocks, not float[] Attention dequants a block to f16/f32 in registers, then QK / softmax / PV 1 token, 1 head, dim 256 = 8 blocks (256 / 32) 1 token, all layers: K: 16 * 4 * 8 * sizeof(block_k) V: 16 * 4 * 8 * sizeof(block_v) Example pair q8_0 / q4_1, first 32-wide block of one head block_q8_0 34 B x = d * q ggml_half d int8_t qs[32] // -128..127 K source (f32, not stored) 0.42 -1.10 0.05 2.30 -0.80 0.15 1.70 -2.05 ... K bytes a5 25 | 13 ce 02 68 dc 07 4d a3 ... d = 0.022049 (fp16) qs = 19, -50, 2, 104, -36, 7, 77, -93, ... K dequant 0.4189 -1.1024 0.0441 2.2931 ... err ~ 0.01 block_q4_1 20 B x = d * q + m ggml_half d ggml_half m uint8_t qs[16] // two nibbles 0..15 per byte V source (f32, not stored) 0.10 0.85 1.40 0.30 2.10 0.05 0.70 1.90 ... V bytes 14 31 | 1f 25 | 51 29 0d c4 71 03 6b f2 ... d = 0.1587 m = 0.0200 qs = 1, 5, 9, 2, 13, 0, 4, 12, ... V dequant 0.179 0.814 1.448 0.337 2.083 ... err ~ 0.08 1 token totals at q8_0/q4_1 K 17408 B + V 10240 B = 27648 B = 27 KiB ======================================================================== Symmetric cache-type-k == cache-type-v ======================================================================== ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = f32 B/elem = 4.0 KiB/token = 128.00 GiB = ctx / 8192.00 ctx-size = 2048 # 0.25 GiB ctx-size = 4096 # 0.50 GiB ctx-size = 6144 # 0.75 GiB ctx-size = 8192 # 1.00 GiB ctx-size = 12288 # 1.50 GiB ctx-size = 16384 # 2.00 GiB ctx-size = 24576 # 3.00 GiB ctx-size = 32768 # 4.00 GiB ctx-size = 40960 # 5.00 GiB ctx-size = 49152 # 6.00 GiB ctx-size = 65536 # 8.00 GiB ctx-size = 81920 # 10.0 GiB ctx-size = 98304 # 12.0 GiB ctx-size = 114688 # 14.0 GiB ctx-size = 131072 # 16.0 GiB ctx-size = 147456 # 18.0 GiB ctx-size = 163840 # 20.0 GiB ctx-size = 180224 # 22.0 GiB ctx-size = 196608 # 24.0 GiB ctx-size = 229376 # 28.0 GiB ctx-size = 245760 # 30.0 GiB ctx-size = 262144 # 32.0 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = f16 B/elem = 2.0 KiB/token = 64.00 GiB = ctx / 16384.00 ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = bf16 KiB/token = 64.00 (same size as f16/f16) ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q8_0 B/elem = 1.0625 KiB/token = 34.00 GiB = ctx / 30840.47 ctx-size = 2048 # 0.066 GiB ctx-size = 4096 # 0.13 GiB ctx-size = 6144 # 0.20 GiB ctx-size = 8192 # 0.27 GiB ctx-size = 12288 # 0.40 GiB ctx-size = 16384 # 0.53 GiB ctx-size = 24576 # 0.80 GiB ctx-size = 32768 # 1.06 GiB ctx-size = 40960 # 1.33 GiB ctx-size = 49152 # 1.59 GiB ctx-size = 65536 # 2.12 GiB ctx-size = 81920 # 2.66 GiB ctx-size = 98304 # 3.19 GiB ctx-size = 114688 # 3.72 GiB ctx-size = 131072 # 4.25 GiB ctx-size = 147456 # 4.78 GiB ctx-size = 163840 # 5.31 GiB ctx-size = 180224 # 5.84 GiB ctx-size = 196608 # 6.38 GiB ctx-size = 229376 # 7.44 GiB ctx-size = 245760 # 7.97 GiB ctx-size = 262144 # 8.50 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q5_1 B/elem = 0.75 KiB/token = 24.00 GiB = ctx / 43690.67 ctx-size = 2048 # 0.047 GiB ctx-size = 4096 # 0.094 GiB ctx-size = 6144 # 0.14 GiB ctx-size = 8192 # 0.19 GiB ctx-size = 12288 # 0.28 GiB ctx-size = 16384 # 0.38 GiB ctx-size = 24576 # 0.56 GiB ctx-size = 32768 # 0.75 GiB ctx-size = 40960 # 0.94 GiB ctx-size = 49152 # 1.12 GiB ctx-size = 65536 # 1.50 GiB ctx-size = 81920 # 1.88 GiB ctx-size = 98304 # 2.25 GiB ctx-size = 114688 # 2.62 GiB ctx-size = 131072 # 3.00 GiB ctx-size = 147456 # 3.38 GiB ctx-size = 163840 # 3.75 GiB ctx-size = 180224 # 4.12 GiB ctx-size = 196608 # 4.50 GiB ctx-size = 229376 # 5.25 GiB ctx-size = 245760 # 5.62 GiB ctx-size = 262144 # 6.00 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q5_0 B/elem = 0.6875 KiB/token = 22.00 GiB = ctx / 47662.55 ctx-size = 2048 # 0.043 GiB ctx-size = 4096 # 0.086 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.17 GiB ctx-size = 12288 # 0.26 GiB ctx-size = 16384 # 0.34 GiB ctx-size = 24576 # 0.52 GiB ctx-size = 32768 # 0.69 GiB ctx-size = 40960 # 0.86 GiB ctx-size = 49152 # 1.03 GiB ctx-size = 65536 # 1.38 GiB ctx-size = 81920 # 1.72 GiB ctx-size = 98304 # 2.06 GiB ctx-size = 114688 # 2.41 GiB ctx-size = 131072 # 2.75 GiB ctx-size = 147456 # 3.09 GiB ctx-size = 163840 # 3.44 GiB ctx-size = 180224 # 3.78 GiB ctx-size = 196608 # 4.12 GiB ctx-size = 229376 # 4.81 GiB ctx-size = 245760 # 5.16 GiB ctx-size = 262144 # 5.50 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q4_1 B/elem = 0.625 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q4_0 B/elem = 0.5625 KiB/token = 18.00 GiB = ctx / 58254.22 ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = iq4_nl KiB/token = 18.00 (same size as q4_0/q4_0) ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB ======================================================================== All cache-type-k / cache-type-v pairs ======================================================================== ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = f32 B/elem = 4.0 + 4.0 = 8.0 KiB/token = 128.00 GiB = ctx / 8192.00 ctx-size = 2048 # 0.25 GiB ctx-size = 4096 # 0.50 GiB ctx-size = 6144 # 0.75 GiB ctx-size = 8192 # 1.00 GiB ctx-size = 12288 # 1.50 GiB ctx-size = 16384 # 2.00 GiB ctx-size = 24576 # 3.00 GiB ctx-size = 32768 # 4.00 GiB ctx-size = 40960 # 5.00 GiB ctx-size = 49152 # 6.00 GiB ctx-size = 65536 # 8.00 GiB ctx-size = 81920 # 10.0 GiB ctx-size = 98304 # 12.0 GiB ctx-size = 114688 # 14.0 GiB ctx-size = 131072 # 16.0 GiB ctx-size = 147456 # 18.0 GiB ctx-size = 163840 # 20.0 GiB ctx-size = 180224 # 22.0 GiB ctx-size = 196608 # 24.0 GiB ctx-size = 229376 # 28.0 GiB ctx-size = 245760 # 30.0 GiB ctx-size = 262144 # 32.0 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = f16 B/elem = 4.0 + 2.0 = 6.0 KiB/token = 96.00 GiB = ctx / 10922.67 ctx-size = 2048 # 0.19 GiB ctx-size = 4096 # 0.38 GiB ctx-size = 6144 # 0.56 GiB ctx-size = 8192 # 0.75 GiB ctx-size = 12288 # 1.12 GiB ctx-size = 16384 # 1.50 GiB ctx-size = 24576 # 2.25 GiB ctx-size = 32768 # 3.00 GiB ctx-size = 40960 # 3.75 GiB ctx-size = 49152 # 4.50 GiB ctx-size = 65536 # 6.00 GiB ctx-size = 81920 # 7.50 GiB ctx-size = 98304 # 9.00 GiB ctx-size = 114688 # 10.5 GiB ctx-size = 131072 # 12.0 GiB ctx-size = 147456 # 13.5 GiB ctx-size = 163840 # 15.0 GiB ctx-size = 180224 # 16.5 GiB ctx-size = 196608 # 18.0 GiB ctx-size = 229376 # 21.0 GiB ctx-size = 245760 # 22.5 GiB ctx-size = 262144 # 24.0 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = bf16 B/elem = 4.0 + 2.0 = 6.0 KiB/token = 96.00 GiB = ctx / 10922.67 ctx-size = 2048 # 0.19 GiB ctx-size = 4096 # 0.38 GiB ctx-size = 6144 # 0.56 GiB ctx-size = 8192 # 0.75 GiB ctx-size = 12288 # 1.12 GiB ctx-size = 16384 # 1.50 GiB ctx-size = 24576 # 2.25 GiB ctx-size = 32768 # 3.00 GiB ctx-size = 40960 # 3.75 GiB ctx-size = 49152 # 4.50 GiB ctx-size = 65536 # 6.00 GiB ctx-size = 81920 # 7.50 GiB ctx-size = 98304 # 9.00 GiB ctx-size = 114688 # 10.5 GiB ctx-size = 131072 # 12.0 GiB ctx-size = 147456 # 13.5 GiB ctx-size = 163840 # 15.0 GiB ctx-size = 180224 # 16.5 GiB ctx-size = 196608 # 18.0 GiB ctx-size = 229376 # 21.0 GiB ctx-size = 245760 # 22.5 GiB ctx-size = 262144 # 24.0 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = q8_0 B/elem = 4.0 + 1.0625 = 5.0625 KiB/token = 81.00 GiB = ctx / 12945.38 ctx-size = 2048 # 0.16 GiB ctx-size = 4096 # 0.32 GiB ctx-size = 6144 # 0.47 GiB ctx-size = 8192 # 0.63 GiB ctx-size = 12288 # 0.95 GiB ctx-size = 16384 # 1.27 GiB ctx-size = 24576 # 1.90 GiB ctx-size = 32768 # 2.53 GiB ctx-size = 40960 # 3.16 GiB ctx-size = 49152 # 3.80 GiB ctx-size = 65536 # 5.06 GiB ctx-size = 81920 # 6.33 GiB ctx-size = 98304 # 7.59 GiB ctx-size = 114688 # 8.86 GiB ctx-size = 131072 # 10.1 GiB ctx-size = 147456 # 11.4 GiB ctx-size = 163840 # 12.7 GiB ctx-size = 180224 # 13.9 GiB ctx-size = 196608 # 15.2 GiB ctx-size = 229376 # 17.7 GiB ctx-size = 245760 # 19.0 GiB ctx-size = 262144 # 20.2 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = q5_1 B/elem = 4.0 + 0.75 = 4.75 KiB/token = 76.00 GiB = ctx / 13797.05 ctx-size = 2048 # 0.15 GiB ctx-size = 4096 # 0.30 GiB ctx-size = 6144 # 0.45 GiB ctx-size = 8192 # 0.59 GiB ctx-size = 12288 # 0.89 GiB ctx-size = 16384 # 1.19 GiB ctx-size = 24576 # 1.78 GiB ctx-size = 32768 # 2.38 GiB ctx-size = 40960 # 2.97 GiB ctx-size = 49152 # 3.56 GiB ctx-size = 65536 # 4.75 GiB ctx-size = 81920 # 5.94 GiB ctx-size = 98304 # 7.12 GiB ctx-size = 114688 # 8.31 GiB ctx-size = 131072 # 9.50 GiB ctx-size = 147456 # 10.7 GiB ctx-size = 163840 # 11.9 GiB ctx-size = 180224 # 13.1 GiB ctx-size = 196608 # 14.2 GiB ctx-size = 229376 # 16.6 GiB ctx-size = 245760 # 17.8 GiB ctx-size = 262144 # 19.0 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = q5_0 B/elem = 4.0 + 0.6875 = 4.6875 KiB/token = 75.00 GiB = ctx / 13981.01 ctx-size = 2048 # 0.15 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.44 GiB ctx-size = 8192 # 0.59 GiB ctx-size = 12288 # 0.88 GiB ctx-size = 16384 # 1.17 GiB ctx-size = 24576 # 1.76 GiB ctx-size = 32768 # 2.34 GiB ctx-size = 40960 # 2.93 GiB ctx-size = 49152 # 3.52 GiB ctx-size = 65536 # 4.69 GiB ctx-size = 81920 # 5.86 GiB ctx-size = 98304 # 7.03 GiB ctx-size = 114688 # 8.20 GiB ctx-size = 131072 # 9.38 GiB ctx-size = 147456 # 10.5 GiB ctx-size = 163840 # 11.7 GiB ctx-size = 180224 # 12.9 GiB ctx-size = 196608 # 14.1 GiB ctx-size = 229376 # 16.4 GiB ctx-size = 245760 # 17.6 GiB ctx-size = 262144 # 18.8 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = q4_1 B/elem = 4.0 + 0.625 = 4.625 KiB/token = 74.00 GiB = ctx / 14169.95 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.58 GiB ctx-size = 12288 # 0.87 GiB ctx-size = 16384 # 1.16 GiB ctx-size = 24576 # 1.73 GiB ctx-size = 32768 # 2.31 GiB ctx-size = 40960 # 2.89 GiB ctx-size = 49152 # 3.47 GiB ctx-size = 65536 # 4.62 GiB ctx-size = 81920 # 5.78 GiB ctx-size = 98304 # 6.94 GiB ctx-size = 114688 # 8.09 GiB ctx-size = 131072 # 9.25 GiB ctx-size = 147456 # 10.4 GiB ctx-size = 163840 # 11.6 GiB ctx-size = 180224 # 12.7 GiB ctx-size = 196608 # 13.9 GiB ctx-size = 229376 # 16.2 GiB ctx-size = 245760 # 17.3 GiB ctx-size = 262144 # 18.5 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = q4_0 B/elem = 4.0 + 0.5625 = 4.5625 KiB/token = 73.00 GiB = ctx / 14364.05 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.57 GiB ctx-size = 12288 # 0.86 GiB ctx-size = 16384 # 1.14 GiB ctx-size = 24576 # 1.71 GiB ctx-size = 32768 # 2.28 GiB ctx-size = 40960 # 2.85 GiB ctx-size = 49152 # 3.42 GiB ctx-size = 65536 # 4.56 GiB ctx-size = 81920 # 5.70 GiB ctx-size = 98304 # 6.84 GiB ctx-size = 114688 # 7.98 GiB ctx-size = 131072 # 9.12 GiB ctx-size = 147456 # 10.3 GiB ctx-size = 163840 # 11.4 GiB ctx-size = 180224 # 12.5 GiB ctx-size = 196608 # 13.7 GiB ctx-size = 229376 # 16.0 GiB ctx-size = 245760 # 17.1 GiB ctx-size = 262144 # 18.2 GiB ------------------------------------------------------------------------ cache-type-k = f32 cache-type-v = iq4_nl B/elem = 4.0 + 0.5625 = 4.5625 KiB/token = 73.00 GiB = ctx / 14364.05 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.57 GiB ctx-size = 12288 # 0.86 GiB ctx-size = 16384 # 1.14 GiB ctx-size = 24576 # 1.71 GiB ctx-size = 32768 # 2.28 GiB ctx-size = 40960 # 2.85 GiB ctx-size = 49152 # 3.42 GiB ctx-size = 65536 # 4.56 GiB ctx-size = 81920 # 5.70 GiB ctx-size = 98304 # 6.84 GiB ctx-size = 114688 # 7.98 GiB ctx-size = 131072 # 9.12 GiB ctx-size = 147456 # 10.3 GiB ctx-size = 163840 # 11.4 GiB ctx-size = 180224 # 12.5 GiB ctx-size = 196608 # 13.7 GiB ctx-size = 229376 # 16.0 GiB ctx-size = 245760 # 17.1 GiB ctx-size = 262144 # 18.2 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = f32 B/elem = 2.0 + 4.0 = 6.0 KiB/token = 96.00 GiB = ctx / 10922.67 ctx-size = 2048 # 0.19 GiB ctx-size = 4096 # 0.38 GiB ctx-size = 6144 # 0.56 GiB ctx-size = 8192 # 0.75 GiB ctx-size = 12288 # 1.12 GiB ctx-size = 16384 # 1.50 GiB ctx-size = 24576 # 2.25 GiB ctx-size = 32768 # 3.00 GiB ctx-size = 40960 # 3.75 GiB ctx-size = 49152 # 4.50 GiB ctx-size = 65536 # 6.00 GiB ctx-size = 81920 # 7.50 GiB ctx-size = 98304 # 9.00 GiB ctx-size = 114688 # 10.5 GiB ctx-size = 131072 # 12.0 GiB ctx-size = 147456 # 13.5 GiB ctx-size = 163840 # 15.0 GiB ctx-size = 180224 # 16.5 GiB ctx-size = 196608 # 18.0 GiB ctx-size = 229376 # 21.0 GiB ctx-size = 245760 # 22.5 GiB ctx-size = 262144 # 24.0 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = f16 B/elem = 2.0 + 2.0 = 4.0 KiB/token = 64.00 GiB = ctx / 16384.00 ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = bf16 B/elem = 2.0 + 2.0 = 4.0 KiB/token = 64.00 GiB = ctx / 16384.00 ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = q8_0 B/elem = 2.0 + 1.0625 = 3.0625 KiB/token = 49.00 GiB = ctx / 21399.51 ctx-size = 2048 # 0.096 GiB ctx-size = 4096 # 0.19 GiB ctx-size = 6144 # 0.29 GiB ctx-size = 8192 # 0.38 GiB ctx-size = 12288 # 0.57 GiB ctx-size = 16384 # 0.77 GiB ctx-size = 24576 # 1.15 GiB ctx-size = 32768 # 1.53 GiB ctx-size = 40960 # 1.91 GiB ctx-size = 49152 # 2.30 GiB ctx-size = 65536 # 3.06 GiB ctx-size = 81920 # 3.83 GiB ctx-size = 98304 # 4.59 GiB ctx-size = 114688 # 5.36 GiB ctx-size = 131072 # 6.12 GiB ctx-size = 147456 # 6.89 GiB ctx-size = 163840 # 7.66 GiB ctx-size = 180224 # 8.42 GiB ctx-size = 196608 # 9.19 GiB ctx-size = 229376 # 10.7 GiB ctx-size = 245760 # 11.5 GiB ctx-size = 262144 # 12.2 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = q5_1 B/elem = 2.0 + 0.75 = 2.75 KiB/token = 44.00 GiB = ctx / 23831.27 ctx-size = 2048 # 0.086 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.26 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.52 GiB ctx-size = 16384 # 0.69 GiB ctx-size = 24576 # 1.03 GiB ctx-size = 32768 # 1.38 GiB ctx-size = 40960 # 1.72 GiB ctx-size = 49152 # 2.06 GiB ctx-size = 65536 # 2.75 GiB ctx-size = 81920 # 3.44 GiB ctx-size = 98304 # 4.12 GiB ctx-size = 114688 # 4.81 GiB ctx-size = 131072 # 5.50 GiB ctx-size = 147456 # 6.19 GiB ctx-size = 163840 # 6.88 GiB ctx-size = 180224 # 7.56 GiB ctx-size = 196608 # 8.25 GiB ctx-size = 229376 # 9.62 GiB ctx-size = 245760 # 10.3 GiB ctx-size = 262144 # 11.0 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = q5_0 B/elem = 2.0 + 0.6875 = 2.6875 KiB/token = 43.00 GiB = ctx / 24385.49 ctx-size = 2048 # 0.084 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.50 GiB ctx-size = 16384 # 0.67 GiB ctx-size = 24576 # 1.01 GiB ctx-size = 32768 # 1.34 GiB ctx-size = 40960 # 1.68 GiB ctx-size = 49152 # 2.02 GiB ctx-size = 65536 # 2.69 GiB ctx-size = 81920 # 3.36 GiB ctx-size = 98304 # 4.03 GiB ctx-size = 114688 # 4.70 GiB ctx-size = 131072 # 5.38 GiB ctx-size = 147456 # 6.05 GiB ctx-size = 163840 # 6.72 GiB ctx-size = 180224 # 7.39 GiB ctx-size = 196608 # 8.06 GiB ctx-size = 229376 # 9.41 GiB ctx-size = 245760 # 10.1 GiB ctx-size = 262144 # 10.8 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = q4_1 B/elem = 2.0 + 0.625 = 2.625 KiB/token = 42.00 GiB = ctx / 24966.10 ctx-size = 2048 # 0.082 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.33 GiB ctx-size = 12288 # 0.49 GiB ctx-size = 16384 # 0.66 GiB ctx-size = 24576 # 0.98 GiB ctx-size = 32768 # 1.31 GiB ctx-size = 40960 # 1.64 GiB ctx-size = 49152 # 1.97 GiB ctx-size = 65536 # 2.62 GiB ctx-size = 81920 # 3.28 GiB ctx-size = 98304 # 3.94 GiB ctx-size = 114688 # 4.59 GiB ctx-size = 131072 # 5.25 GiB ctx-size = 147456 # 5.91 GiB ctx-size = 163840 # 6.56 GiB ctx-size = 180224 # 7.22 GiB ctx-size = 196608 # 7.88 GiB ctx-size = 229376 # 9.19 GiB ctx-size = 245760 # 9.84 GiB ctx-size = 262144 # 10.5 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = q4_0 B/elem = 2.0 + 0.5625 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = f16 cache-type-v = iq4_nl B/elem = 2.0 + 0.5625 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = f32 B/elem = 2.0 + 4.0 = 6.0 KiB/token = 96.00 GiB = ctx / 10922.67 ctx-size = 2048 # 0.19 GiB ctx-size = 4096 # 0.38 GiB ctx-size = 6144 # 0.56 GiB ctx-size = 8192 # 0.75 GiB ctx-size = 12288 # 1.12 GiB ctx-size = 16384 # 1.50 GiB ctx-size = 24576 # 2.25 GiB ctx-size = 32768 # 3.00 GiB ctx-size = 40960 # 3.75 GiB ctx-size = 49152 # 4.50 GiB ctx-size = 65536 # 6.00 GiB ctx-size = 81920 # 7.50 GiB ctx-size = 98304 # 9.00 GiB ctx-size = 114688 # 10.5 GiB ctx-size = 131072 # 12.0 GiB ctx-size = 147456 # 13.5 GiB ctx-size = 163840 # 15.0 GiB ctx-size = 180224 # 16.5 GiB ctx-size = 196608 # 18.0 GiB ctx-size = 229376 # 21.0 GiB ctx-size = 245760 # 22.5 GiB ctx-size = 262144 # 24.0 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = f16 B/elem = 2.0 + 2.0 = 4.0 KiB/token = 64.00 GiB = ctx / 16384.00 ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = bf16 B/elem = 2.0 + 2.0 = 4.0 KiB/token = 64.00 GiB = ctx / 16384.00 ctx-size = 2048 # 0.12 GiB ctx-size = 4096 # 0.25 GiB ctx-size = 6144 # 0.38 GiB ctx-size = 8192 # 0.50 GiB ctx-size = 12288 # 0.75 GiB ctx-size = 16384 # 1.00 GiB ctx-size = 24576 # 1.50 GiB ctx-size = 32768 # 2.00 GiB ctx-size = 40960 # 2.50 GiB ctx-size = 49152 # 3.00 GiB ctx-size = 65536 # 4.00 GiB ctx-size = 81920 # 5.00 GiB ctx-size = 98304 # 6.00 GiB ctx-size = 114688 # 7.00 GiB ctx-size = 131072 # 8.00 GiB ctx-size = 147456 # 9.00 GiB ctx-size = 163840 # 10.0 GiB ctx-size = 180224 # 11.0 GiB ctx-size = 196608 # 12.0 GiB ctx-size = 229376 # 14.0 GiB ctx-size = 245760 # 15.0 GiB ctx-size = 262144 # 16.0 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = q8_0 B/elem = 2.0 + 1.0625 = 3.0625 KiB/token = 49.00 GiB = ctx / 21399.51 ctx-size = 2048 # 0.096 GiB ctx-size = 4096 # 0.19 GiB ctx-size = 6144 # 0.29 GiB ctx-size = 8192 # 0.38 GiB ctx-size = 12288 # 0.57 GiB ctx-size = 16384 # 0.77 GiB ctx-size = 24576 # 1.15 GiB ctx-size = 32768 # 1.53 GiB ctx-size = 40960 # 1.91 GiB ctx-size = 49152 # 2.30 GiB ctx-size = 65536 # 3.06 GiB ctx-size = 81920 # 3.83 GiB ctx-size = 98304 # 4.59 GiB ctx-size = 114688 # 5.36 GiB ctx-size = 131072 # 6.12 GiB ctx-size = 147456 # 6.89 GiB ctx-size = 163840 # 7.66 GiB ctx-size = 180224 # 8.42 GiB ctx-size = 196608 # 9.19 GiB ctx-size = 229376 # 10.7 GiB ctx-size = 245760 # 11.5 GiB ctx-size = 262144 # 12.2 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = q5_1 B/elem = 2.0 + 0.75 = 2.75 KiB/token = 44.00 GiB = ctx / 23831.27 ctx-size = 2048 # 0.086 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.26 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.52 GiB ctx-size = 16384 # 0.69 GiB ctx-size = 24576 # 1.03 GiB ctx-size = 32768 # 1.38 GiB ctx-size = 40960 # 1.72 GiB ctx-size = 49152 # 2.06 GiB ctx-size = 65536 # 2.75 GiB ctx-size = 81920 # 3.44 GiB ctx-size = 98304 # 4.12 GiB ctx-size = 114688 # 4.81 GiB ctx-size = 131072 # 5.50 GiB ctx-size = 147456 # 6.19 GiB ctx-size = 163840 # 6.88 GiB ctx-size = 180224 # 7.56 GiB ctx-size = 196608 # 8.25 GiB ctx-size = 229376 # 9.62 GiB ctx-size = 245760 # 10.3 GiB ctx-size = 262144 # 11.0 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = q5_0 B/elem = 2.0 + 0.6875 = 2.6875 KiB/token = 43.00 GiB = ctx / 24385.49 ctx-size = 2048 # 0.084 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.50 GiB ctx-size = 16384 # 0.67 GiB ctx-size = 24576 # 1.01 GiB ctx-size = 32768 # 1.34 GiB ctx-size = 40960 # 1.68 GiB ctx-size = 49152 # 2.02 GiB ctx-size = 65536 # 2.69 GiB ctx-size = 81920 # 3.36 GiB ctx-size = 98304 # 4.03 GiB ctx-size = 114688 # 4.70 GiB ctx-size = 131072 # 5.38 GiB ctx-size = 147456 # 6.05 GiB ctx-size = 163840 # 6.72 GiB ctx-size = 180224 # 7.39 GiB ctx-size = 196608 # 8.06 GiB ctx-size = 229376 # 9.41 GiB ctx-size = 245760 # 10.1 GiB ctx-size = 262144 # 10.8 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = q4_1 B/elem = 2.0 + 0.625 = 2.625 KiB/token = 42.00 GiB = ctx / 24966.10 ctx-size = 2048 # 0.082 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.33 GiB ctx-size = 12288 # 0.49 GiB ctx-size = 16384 # 0.66 GiB ctx-size = 24576 # 0.98 GiB ctx-size = 32768 # 1.31 GiB ctx-size = 40960 # 1.64 GiB ctx-size = 49152 # 1.97 GiB ctx-size = 65536 # 2.62 GiB ctx-size = 81920 # 3.28 GiB ctx-size = 98304 # 3.94 GiB ctx-size = 114688 # 4.59 GiB ctx-size = 131072 # 5.25 GiB ctx-size = 147456 # 5.91 GiB ctx-size = 163840 # 6.56 GiB ctx-size = 180224 # 7.22 GiB ctx-size = 196608 # 7.88 GiB ctx-size = 229376 # 9.19 GiB ctx-size = 245760 # 9.84 GiB ctx-size = 262144 # 10.5 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = q4_0 B/elem = 2.0 + 0.5625 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = bf16 cache-type-v = iq4_nl B/elem = 2.0 + 0.5625 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = f32 B/elem = 1.0625 + 4.0 = 5.0625 KiB/token = 81.00 GiB = ctx / 12945.38 ctx-size = 2048 # 0.16 GiB ctx-size = 4096 # 0.32 GiB ctx-size = 6144 # 0.47 GiB ctx-size = 8192 # 0.63 GiB ctx-size = 12288 # 0.95 GiB ctx-size = 16384 # 1.27 GiB ctx-size = 24576 # 1.90 GiB ctx-size = 32768 # 2.53 GiB ctx-size = 40960 # 3.16 GiB ctx-size = 49152 # 3.80 GiB ctx-size = 65536 # 5.06 GiB ctx-size = 81920 # 6.33 GiB ctx-size = 98304 # 7.59 GiB ctx-size = 114688 # 8.86 GiB ctx-size = 131072 # 10.1 GiB ctx-size = 147456 # 11.4 GiB ctx-size = 163840 # 12.7 GiB ctx-size = 180224 # 13.9 GiB ctx-size = 196608 # 15.2 GiB ctx-size = 229376 # 17.7 GiB ctx-size = 245760 # 19.0 GiB ctx-size = 262144 # 20.2 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = f16 B/elem = 1.0625 + 2.0 = 3.0625 KiB/token = 49.00 GiB = ctx / 21399.51 ctx-size = 2048 # 0.096 GiB ctx-size = 4096 # 0.19 GiB ctx-size = 6144 # 0.29 GiB ctx-size = 8192 # 0.38 GiB ctx-size = 12288 # 0.57 GiB ctx-size = 16384 # 0.77 GiB ctx-size = 24576 # 1.15 GiB ctx-size = 32768 # 1.53 GiB ctx-size = 40960 # 1.91 GiB ctx-size = 49152 # 2.30 GiB ctx-size = 65536 # 3.06 GiB ctx-size = 81920 # 3.83 GiB ctx-size = 98304 # 4.59 GiB ctx-size = 114688 # 5.36 GiB ctx-size = 131072 # 6.12 GiB ctx-size = 147456 # 6.89 GiB ctx-size = 163840 # 7.66 GiB ctx-size = 180224 # 8.42 GiB ctx-size = 196608 # 9.19 GiB ctx-size = 229376 # 10.7 GiB ctx-size = 245760 # 11.5 GiB ctx-size = 262144 # 12.2 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = bf16 B/elem = 1.0625 + 2.0 = 3.0625 KiB/token = 49.00 GiB = ctx / 21399.51 ctx-size = 2048 # 0.096 GiB ctx-size = 4096 # 0.19 GiB ctx-size = 6144 # 0.29 GiB ctx-size = 8192 # 0.38 GiB ctx-size = 12288 # 0.57 GiB ctx-size = 16384 # 0.77 GiB ctx-size = 24576 # 1.15 GiB ctx-size = 32768 # 1.53 GiB ctx-size = 40960 # 1.91 GiB ctx-size = 49152 # 2.30 GiB ctx-size = 65536 # 3.06 GiB ctx-size = 81920 # 3.83 GiB ctx-size = 98304 # 4.59 GiB ctx-size = 114688 # 5.36 GiB ctx-size = 131072 # 6.12 GiB ctx-size = 147456 # 6.89 GiB ctx-size = 163840 # 7.66 GiB ctx-size = 180224 # 8.42 GiB ctx-size = 196608 # 9.19 GiB ctx-size = 229376 # 10.7 GiB ctx-size = 245760 # 11.5 GiB ctx-size = 262144 # 12.2 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q8_0 B/elem = 1.0625 + 1.0625 = 2.125 KiB/token = 34.00 GiB = ctx / 30840.47 ctx-size = 2048 # 0.066 GiB ctx-size = 4096 # 0.13 GiB ctx-size = 6144 # 0.20 GiB ctx-size = 8192 # 0.27 GiB ctx-size = 12288 # 0.40 GiB ctx-size = 16384 # 0.53 GiB ctx-size = 24576 # 0.80 GiB ctx-size = 32768 # 1.06 GiB ctx-size = 40960 # 1.33 GiB ctx-size = 49152 # 1.59 GiB ctx-size = 65536 # 2.12 GiB ctx-size = 81920 # 2.66 GiB ctx-size = 98304 # 3.19 GiB ctx-size = 114688 # 3.72 GiB ctx-size = 131072 # 4.25 GiB ctx-size = 147456 # 4.78 GiB ctx-size = 163840 # 5.31 GiB ctx-size = 180224 # 5.84 GiB ctx-size = 196608 # 6.38 GiB ctx-size = 229376 # 7.44 GiB ctx-size = 245760 # 7.97 GiB ctx-size = 262144 # 8.50 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q5_1 B/elem = 1.0625 + 0.75 = 1.8125 KiB/token = 29.00 GiB = ctx / 36157.79 ctx-size = 2048 # 0.057 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.17 GiB ctx-size = 8192 # 0.23 GiB ctx-size = 12288 # 0.34 GiB ctx-size = 16384 # 0.45 GiB ctx-size = 24576 # 0.68 GiB ctx-size = 32768 # 0.91 GiB ctx-size = 40960 # 1.13 GiB ctx-size = 49152 # 1.36 GiB ctx-size = 65536 # 1.81 GiB ctx-size = 81920 # 2.27 GiB ctx-size = 98304 # 2.72 GiB ctx-size = 114688 # 3.17 GiB ctx-size = 131072 # 3.62 GiB ctx-size = 147456 # 4.08 GiB ctx-size = 163840 # 4.53 GiB ctx-size = 180224 # 4.98 GiB ctx-size = 196608 # 5.44 GiB ctx-size = 229376 # 6.34 GiB ctx-size = 245760 # 6.80 GiB ctx-size = 262144 # 7.25 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q5_0 B/elem = 1.0625 + 0.6875 = 1.75 KiB/token = 28.00 GiB = ctx / 37449.14 ctx-size = 2048 # 0.055 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.16 GiB ctx-size = 8192 # 0.22 GiB ctx-size = 12288 # 0.33 GiB ctx-size = 16384 # 0.44 GiB ctx-size = 24576 # 0.66 GiB ctx-size = 32768 # 0.88 GiB ctx-size = 40960 # 1.09 GiB ctx-size = 49152 # 1.31 GiB ctx-size = 65536 # 1.75 GiB ctx-size = 81920 # 2.19 GiB ctx-size = 98304 # 2.62 GiB ctx-size = 114688 # 3.06 GiB ctx-size = 131072 # 3.50 GiB ctx-size = 147456 # 3.94 GiB ctx-size = 163840 # 4.38 GiB ctx-size = 180224 # 4.81 GiB ctx-size = 196608 # 5.25 GiB ctx-size = 229376 # 6.12 GiB ctx-size = 245760 # 6.56 GiB ctx-size = 262144 # 7.00 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q4_1 B/elem = 1.0625 + 0.625 = 1.6875 KiB/token = 27.00 GiB = ctx / 38836.15 ctx-size = 2048 # 0.053 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.16 GiB ctx-size = 8192 # 0.21 GiB ctx-size = 12288 # 0.32 GiB ctx-size = 16384 # 0.42 GiB ctx-size = 24576 # 0.63 GiB ctx-size = 32768 # 0.84 GiB ctx-size = 40960 # 1.05 GiB ctx-size = 49152 # 1.27 GiB ctx-size = 65536 # 1.69 GiB ctx-size = 81920 # 2.11 GiB ctx-size = 98304 # 2.53 GiB ctx-size = 114688 # 2.95 GiB ctx-size = 131072 # 3.38 GiB ctx-size = 147456 # 3.80 GiB ctx-size = 163840 # 4.22 GiB ctx-size = 180224 # 4.64 GiB ctx-size = 196608 # 5.06 GiB ctx-size = 229376 # 5.91 GiB ctx-size = 245760 # 6.33 GiB ctx-size = 262144 # 6.75 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = q4_0 B/elem = 1.0625 + 0.5625 = 1.625 KiB/token = 26.00 GiB = ctx / 40329.85 ctx-size = 2048 # 0.051 GiB ctx-size = 4096 # 0.10 GiB ctx-size = 6144 # 0.15 GiB ctx-size = 8192 # 0.20 GiB ctx-size = 12288 # 0.30 GiB ctx-size = 16384 # 0.41 GiB ctx-size = 24576 # 0.61 GiB ctx-size = 32768 # 0.81 GiB ctx-size = 40960 # 1.02 GiB ctx-size = 49152 # 1.22 GiB ctx-size = 65536 # 1.62 GiB ctx-size = 81920 # 2.03 GiB ctx-size = 98304 # 2.44 GiB ctx-size = 114688 # 2.84 GiB ctx-size = 131072 # 3.25 GiB ctx-size = 147456 # 3.66 GiB ctx-size = 163840 # 4.06 GiB ctx-size = 180224 # 4.47 GiB ctx-size = 196608 # 4.88 GiB ctx-size = 229376 # 5.69 GiB ctx-size = 245760 # 6.09 GiB ctx-size = 262144 # 6.50 GiB ------------------------------------------------------------------------ cache-type-k = q8_0 cache-type-v = iq4_nl B/elem = 1.0625 + 0.5625 = 1.625 KiB/token = 26.00 GiB = ctx / 40329.85 ctx-size = 2048 # 0.051 GiB ctx-size = 4096 # 0.10 GiB ctx-size = 6144 # 0.15 GiB ctx-size = 8192 # 0.20 GiB ctx-size = 12288 # 0.30 GiB ctx-size = 16384 # 0.41 GiB ctx-size = 24576 # 0.61 GiB ctx-size = 32768 # 0.81 GiB ctx-size = 40960 # 1.02 GiB ctx-size = 49152 # 1.22 GiB ctx-size = 65536 # 1.62 GiB ctx-size = 81920 # 2.03 GiB ctx-size = 98304 # 2.44 GiB ctx-size = 114688 # 2.84 GiB ctx-size = 131072 # 3.25 GiB ctx-size = 147456 # 3.66 GiB ctx-size = 163840 # 4.06 GiB ctx-size = 180224 # 4.47 GiB ctx-size = 196608 # 4.88 GiB ctx-size = 229376 # 5.69 GiB ctx-size = 245760 # 6.09 GiB ctx-size = 262144 # 6.50 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = f32 B/elem = 0.75 + 4.0 = 4.75 KiB/token = 76.00 GiB = ctx / 13797.05 ctx-size = 2048 # 0.15 GiB ctx-size = 4096 # 0.30 GiB ctx-size = 6144 # 0.45 GiB ctx-size = 8192 # 0.59 GiB ctx-size = 12288 # 0.89 GiB ctx-size = 16384 # 1.19 GiB ctx-size = 24576 # 1.78 GiB ctx-size = 32768 # 2.38 GiB ctx-size = 40960 # 2.97 GiB ctx-size = 49152 # 3.56 GiB ctx-size = 65536 # 4.75 GiB ctx-size = 81920 # 5.94 GiB ctx-size = 98304 # 7.12 GiB ctx-size = 114688 # 8.31 GiB ctx-size = 131072 # 9.50 GiB ctx-size = 147456 # 10.7 GiB ctx-size = 163840 # 11.9 GiB ctx-size = 180224 # 13.1 GiB ctx-size = 196608 # 14.2 GiB ctx-size = 229376 # 16.6 GiB ctx-size = 245760 # 17.8 GiB ctx-size = 262144 # 19.0 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = f16 B/elem = 0.75 + 2.0 = 2.75 KiB/token = 44.00 GiB = ctx / 23831.27 ctx-size = 2048 # 0.086 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.26 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.52 GiB ctx-size = 16384 # 0.69 GiB ctx-size = 24576 # 1.03 GiB ctx-size = 32768 # 1.38 GiB ctx-size = 40960 # 1.72 GiB ctx-size = 49152 # 2.06 GiB ctx-size = 65536 # 2.75 GiB ctx-size = 81920 # 3.44 GiB ctx-size = 98304 # 4.12 GiB ctx-size = 114688 # 4.81 GiB ctx-size = 131072 # 5.50 GiB ctx-size = 147456 # 6.19 GiB ctx-size = 163840 # 6.88 GiB ctx-size = 180224 # 7.56 GiB ctx-size = 196608 # 8.25 GiB ctx-size = 229376 # 9.62 GiB ctx-size = 245760 # 10.3 GiB ctx-size = 262144 # 11.0 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = bf16 B/elem = 0.75 + 2.0 = 2.75 KiB/token = 44.00 GiB = ctx / 23831.27 ctx-size = 2048 # 0.086 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.26 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.52 GiB ctx-size = 16384 # 0.69 GiB ctx-size = 24576 # 1.03 GiB ctx-size = 32768 # 1.38 GiB ctx-size = 40960 # 1.72 GiB ctx-size = 49152 # 2.06 GiB ctx-size = 65536 # 2.75 GiB ctx-size = 81920 # 3.44 GiB ctx-size = 98304 # 4.12 GiB ctx-size = 114688 # 4.81 GiB ctx-size = 131072 # 5.50 GiB ctx-size = 147456 # 6.19 GiB ctx-size = 163840 # 6.88 GiB ctx-size = 180224 # 7.56 GiB ctx-size = 196608 # 8.25 GiB ctx-size = 229376 # 9.62 GiB ctx-size = 245760 # 10.3 GiB ctx-size = 262144 # 11.0 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q8_0 B/elem = 0.75 + 1.0625 = 1.8125 KiB/token = 29.00 GiB = ctx / 36157.79 ctx-size = 2048 # 0.057 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.17 GiB ctx-size = 8192 # 0.23 GiB ctx-size = 12288 # 0.34 GiB ctx-size = 16384 # 0.45 GiB ctx-size = 24576 # 0.68 GiB ctx-size = 32768 # 0.91 GiB ctx-size = 40960 # 1.13 GiB ctx-size = 49152 # 1.36 GiB ctx-size = 65536 # 1.81 GiB ctx-size = 81920 # 2.27 GiB ctx-size = 98304 # 2.72 GiB ctx-size = 114688 # 3.17 GiB ctx-size = 131072 # 3.62 GiB ctx-size = 147456 # 4.08 GiB ctx-size = 163840 # 4.53 GiB ctx-size = 180224 # 4.98 GiB ctx-size = 196608 # 5.44 GiB ctx-size = 229376 # 6.34 GiB ctx-size = 245760 # 6.80 GiB ctx-size = 262144 # 7.25 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q5_1 B/elem = 0.75 + 0.75 = 1.5 KiB/token = 24.00 GiB = ctx / 43690.67 ctx-size = 2048 # 0.047 GiB ctx-size = 4096 # 0.094 GiB ctx-size = 6144 # 0.14 GiB ctx-size = 8192 # 0.19 GiB ctx-size = 12288 # 0.28 GiB ctx-size = 16384 # 0.38 GiB ctx-size = 24576 # 0.56 GiB ctx-size = 32768 # 0.75 GiB ctx-size = 40960 # 0.94 GiB ctx-size = 49152 # 1.12 GiB ctx-size = 65536 # 1.50 GiB ctx-size = 81920 # 1.88 GiB ctx-size = 98304 # 2.25 GiB ctx-size = 114688 # 2.62 GiB ctx-size = 131072 # 3.00 GiB ctx-size = 147456 # 3.38 GiB ctx-size = 163840 # 3.75 GiB ctx-size = 180224 # 4.12 GiB ctx-size = 196608 # 4.50 GiB ctx-size = 229376 # 5.25 GiB ctx-size = 245760 # 5.62 GiB ctx-size = 262144 # 6.00 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q5_0 B/elem = 0.75 + 0.6875 = 1.4375 KiB/token = 23.00 GiB = ctx / 45590.26 ctx-size = 2048 # 0.045 GiB ctx-size = 4096 # 0.090 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.18 GiB ctx-size = 12288 # 0.27 GiB ctx-size = 16384 # 0.36 GiB ctx-size = 24576 # 0.54 GiB ctx-size = 32768 # 0.72 GiB ctx-size = 40960 # 0.90 GiB ctx-size = 49152 # 1.08 GiB ctx-size = 65536 # 1.44 GiB ctx-size = 81920 # 1.80 GiB ctx-size = 98304 # 2.16 GiB ctx-size = 114688 # 2.52 GiB ctx-size = 131072 # 2.88 GiB ctx-size = 147456 # 3.23 GiB ctx-size = 163840 # 3.59 GiB ctx-size = 180224 # 3.95 GiB ctx-size = 196608 # 4.31 GiB ctx-size = 229376 # 5.03 GiB ctx-size = 245760 # 5.39 GiB ctx-size = 262144 # 5.75 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q4_1 B/elem = 0.75 + 0.625 = 1.375 KiB/token = 22.00 GiB = ctx / 47662.55 ctx-size = 2048 # 0.043 GiB ctx-size = 4096 # 0.086 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.17 GiB ctx-size = 12288 # 0.26 GiB ctx-size = 16384 # 0.34 GiB ctx-size = 24576 # 0.52 GiB ctx-size = 32768 # 0.69 GiB ctx-size = 40960 # 0.86 GiB ctx-size = 49152 # 1.03 GiB ctx-size = 65536 # 1.38 GiB ctx-size = 81920 # 1.72 GiB ctx-size = 98304 # 2.06 GiB ctx-size = 114688 # 2.41 GiB ctx-size = 131072 # 2.75 GiB ctx-size = 147456 # 3.09 GiB ctx-size = 163840 # 3.44 GiB ctx-size = 180224 # 3.78 GiB ctx-size = 196608 # 4.12 GiB ctx-size = 229376 # 4.81 GiB ctx-size = 245760 # 5.16 GiB ctx-size = 262144 # 5.50 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = q4_0 B/elem = 0.75 + 0.5625 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = q5_1 cache-type-v = iq4_nl B/elem = 0.75 + 0.5625 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = f32 B/elem = 0.6875 + 4.0 = 4.6875 KiB/token = 75.00 GiB = ctx / 13981.01 ctx-size = 2048 # 0.15 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.44 GiB ctx-size = 8192 # 0.59 GiB ctx-size = 12288 # 0.88 GiB ctx-size = 16384 # 1.17 GiB ctx-size = 24576 # 1.76 GiB ctx-size = 32768 # 2.34 GiB ctx-size = 40960 # 2.93 GiB ctx-size = 49152 # 3.52 GiB ctx-size = 65536 # 4.69 GiB ctx-size = 81920 # 5.86 GiB ctx-size = 98304 # 7.03 GiB ctx-size = 114688 # 8.20 GiB ctx-size = 131072 # 9.38 GiB ctx-size = 147456 # 10.5 GiB ctx-size = 163840 # 11.7 GiB ctx-size = 180224 # 12.9 GiB ctx-size = 196608 # 14.1 GiB ctx-size = 229376 # 16.4 GiB ctx-size = 245760 # 17.6 GiB ctx-size = 262144 # 18.8 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = f16 B/elem = 0.6875 + 2.0 = 2.6875 KiB/token = 43.00 GiB = ctx / 24385.49 ctx-size = 2048 # 0.084 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.50 GiB ctx-size = 16384 # 0.67 GiB ctx-size = 24576 # 1.01 GiB ctx-size = 32768 # 1.34 GiB ctx-size = 40960 # 1.68 GiB ctx-size = 49152 # 2.02 GiB ctx-size = 65536 # 2.69 GiB ctx-size = 81920 # 3.36 GiB ctx-size = 98304 # 4.03 GiB ctx-size = 114688 # 4.70 GiB ctx-size = 131072 # 5.38 GiB ctx-size = 147456 # 6.05 GiB ctx-size = 163840 # 6.72 GiB ctx-size = 180224 # 7.39 GiB ctx-size = 196608 # 8.06 GiB ctx-size = 229376 # 9.41 GiB ctx-size = 245760 # 10.1 GiB ctx-size = 262144 # 10.8 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = bf16 B/elem = 0.6875 + 2.0 = 2.6875 KiB/token = 43.00 GiB = ctx / 24385.49 ctx-size = 2048 # 0.084 GiB ctx-size = 4096 # 0.17 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.34 GiB ctx-size = 12288 # 0.50 GiB ctx-size = 16384 # 0.67 GiB ctx-size = 24576 # 1.01 GiB ctx-size = 32768 # 1.34 GiB ctx-size = 40960 # 1.68 GiB ctx-size = 49152 # 2.02 GiB ctx-size = 65536 # 2.69 GiB ctx-size = 81920 # 3.36 GiB ctx-size = 98304 # 4.03 GiB ctx-size = 114688 # 4.70 GiB ctx-size = 131072 # 5.38 GiB ctx-size = 147456 # 6.05 GiB ctx-size = 163840 # 6.72 GiB ctx-size = 180224 # 7.39 GiB ctx-size = 196608 # 8.06 GiB ctx-size = 229376 # 9.41 GiB ctx-size = 245760 # 10.1 GiB ctx-size = 262144 # 10.8 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q8_0 B/elem = 0.6875 + 1.0625 = 1.75 KiB/token = 28.00 GiB = ctx / 37449.14 ctx-size = 2048 # 0.055 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.16 GiB ctx-size = 8192 # 0.22 GiB ctx-size = 12288 # 0.33 GiB ctx-size = 16384 # 0.44 GiB ctx-size = 24576 # 0.66 GiB ctx-size = 32768 # 0.88 GiB ctx-size = 40960 # 1.09 GiB ctx-size = 49152 # 1.31 GiB ctx-size = 65536 # 1.75 GiB ctx-size = 81920 # 2.19 GiB ctx-size = 98304 # 2.62 GiB ctx-size = 114688 # 3.06 GiB ctx-size = 131072 # 3.50 GiB ctx-size = 147456 # 3.94 GiB ctx-size = 163840 # 4.38 GiB ctx-size = 180224 # 4.81 GiB ctx-size = 196608 # 5.25 GiB ctx-size = 229376 # 6.12 GiB ctx-size = 245760 # 6.56 GiB ctx-size = 262144 # 7.00 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q5_1 B/elem = 0.6875 + 0.75 = 1.4375 KiB/token = 23.00 GiB = ctx / 45590.26 ctx-size = 2048 # 0.045 GiB ctx-size = 4096 # 0.090 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.18 GiB ctx-size = 12288 # 0.27 GiB ctx-size = 16384 # 0.36 GiB ctx-size = 24576 # 0.54 GiB ctx-size = 32768 # 0.72 GiB ctx-size = 40960 # 0.90 GiB ctx-size = 49152 # 1.08 GiB ctx-size = 65536 # 1.44 GiB ctx-size = 81920 # 1.80 GiB ctx-size = 98304 # 2.16 GiB ctx-size = 114688 # 2.52 GiB ctx-size = 131072 # 2.88 GiB ctx-size = 147456 # 3.23 GiB ctx-size = 163840 # 3.59 GiB ctx-size = 180224 # 3.95 GiB ctx-size = 196608 # 4.31 GiB ctx-size = 229376 # 5.03 GiB ctx-size = 245760 # 5.39 GiB ctx-size = 262144 # 5.75 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q5_0 B/elem = 0.6875 + 0.6875 = 1.375 KiB/token = 22.00 GiB = ctx / 47662.55 ctx-size = 2048 # 0.043 GiB ctx-size = 4096 # 0.086 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.17 GiB ctx-size = 12288 # 0.26 GiB ctx-size = 16384 # 0.34 GiB ctx-size = 24576 # 0.52 GiB ctx-size = 32768 # 0.69 GiB ctx-size = 40960 # 0.86 GiB ctx-size = 49152 # 1.03 GiB ctx-size = 65536 # 1.38 GiB ctx-size = 81920 # 1.72 GiB ctx-size = 98304 # 2.06 GiB ctx-size = 114688 # 2.41 GiB ctx-size = 131072 # 2.75 GiB ctx-size = 147456 # 3.09 GiB ctx-size = 163840 # 3.44 GiB ctx-size = 180224 # 3.78 GiB ctx-size = 196608 # 4.12 GiB ctx-size = 229376 # 4.81 GiB ctx-size = 245760 # 5.16 GiB ctx-size = 262144 # 5.50 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q4_1 B/elem = 0.6875 + 0.625 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = q4_0 B/elem = 0.6875 + 0.5625 = 1.25 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = q5_0 cache-type-v = iq4_nl B/elem = 0.6875 + 0.5625 = 1.25 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = f32 B/elem = 0.625 + 4.0 = 4.625 KiB/token = 74.00 GiB = ctx / 14169.95 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.58 GiB ctx-size = 12288 # 0.87 GiB ctx-size = 16384 # 1.16 GiB ctx-size = 24576 # 1.73 GiB ctx-size = 32768 # 2.31 GiB ctx-size = 40960 # 2.89 GiB ctx-size = 49152 # 3.47 GiB ctx-size = 65536 # 4.62 GiB ctx-size = 81920 # 5.78 GiB ctx-size = 98304 # 6.94 GiB ctx-size = 114688 # 8.09 GiB ctx-size = 131072 # 9.25 GiB ctx-size = 147456 # 10.4 GiB ctx-size = 163840 # 11.6 GiB ctx-size = 180224 # 12.7 GiB ctx-size = 196608 # 13.9 GiB ctx-size = 229376 # 16.2 GiB ctx-size = 245760 # 17.3 GiB ctx-size = 262144 # 18.5 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = f16 B/elem = 0.625 + 2.0 = 2.625 KiB/token = 42.00 GiB = ctx / 24966.10 ctx-size = 2048 # 0.082 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.33 GiB ctx-size = 12288 # 0.49 GiB ctx-size = 16384 # 0.66 GiB ctx-size = 24576 # 0.98 GiB ctx-size = 32768 # 1.31 GiB ctx-size = 40960 # 1.64 GiB ctx-size = 49152 # 1.97 GiB ctx-size = 65536 # 2.62 GiB ctx-size = 81920 # 3.28 GiB ctx-size = 98304 # 3.94 GiB ctx-size = 114688 # 4.59 GiB ctx-size = 131072 # 5.25 GiB ctx-size = 147456 # 5.91 GiB ctx-size = 163840 # 6.56 GiB ctx-size = 180224 # 7.22 GiB ctx-size = 196608 # 7.88 GiB ctx-size = 229376 # 9.19 GiB ctx-size = 245760 # 9.84 GiB ctx-size = 262144 # 10.5 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = bf16 B/elem = 0.625 + 2.0 = 2.625 KiB/token = 42.00 GiB = ctx / 24966.10 ctx-size = 2048 # 0.082 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.25 GiB ctx-size = 8192 # 0.33 GiB ctx-size = 12288 # 0.49 GiB ctx-size = 16384 # 0.66 GiB ctx-size = 24576 # 0.98 GiB ctx-size = 32768 # 1.31 GiB ctx-size = 40960 # 1.64 GiB ctx-size = 49152 # 1.97 GiB ctx-size = 65536 # 2.62 GiB ctx-size = 81920 # 3.28 GiB ctx-size = 98304 # 3.94 GiB ctx-size = 114688 # 4.59 GiB ctx-size = 131072 # 5.25 GiB ctx-size = 147456 # 5.91 GiB ctx-size = 163840 # 6.56 GiB ctx-size = 180224 # 7.22 GiB ctx-size = 196608 # 7.88 GiB ctx-size = 229376 # 9.19 GiB ctx-size = 245760 # 9.84 GiB ctx-size = 262144 # 10.5 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q8_0 B/elem = 0.625 + 1.0625 = 1.6875 KiB/token = 27.00 GiB = ctx / 38836.15 ctx-size = 2048 # 0.053 GiB ctx-size = 4096 # 0.11 GiB ctx-size = 6144 # 0.16 GiB ctx-size = 8192 # 0.21 GiB ctx-size = 12288 # 0.32 GiB ctx-size = 16384 # 0.42 GiB ctx-size = 24576 # 0.63 GiB ctx-size = 32768 # 0.84 GiB ctx-size = 40960 # 1.05 GiB ctx-size = 49152 # 1.27 GiB ctx-size = 65536 # 1.69 GiB ctx-size = 81920 # 2.11 GiB ctx-size = 98304 # 2.53 GiB ctx-size = 114688 # 2.95 GiB ctx-size = 131072 # 3.38 GiB ctx-size = 147456 # 3.80 GiB ctx-size = 163840 # 4.22 GiB ctx-size = 180224 # 4.64 GiB ctx-size = 196608 # 5.06 GiB ctx-size = 229376 # 5.91 GiB ctx-size = 245760 # 6.33 GiB ctx-size = 262144 # 6.75 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q5_1 B/elem = 0.625 + 0.75 = 1.375 KiB/token = 22.00 GiB = ctx / 47662.55 ctx-size = 2048 # 0.043 GiB ctx-size = 4096 # 0.086 GiB ctx-size = 6144 # 0.13 GiB ctx-size = 8192 # 0.17 GiB ctx-size = 12288 # 0.26 GiB ctx-size = 16384 # 0.34 GiB ctx-size = 24576 # 0.52 GiB ctx-size = 32768 # 0.69 GiB ctx-size = 40960 # 0.86 GiB ctx-size = 49152 # 1.03 GiB ctx-size = 65536 # 1.38 GiB ctx-size = 81920 # 1.72 GiB ctx-size = 98304 # 2.06 GiB ctx-size = 114688 # 2.41 GiB ctx-size = 131072 # 2.75 GiB ctx-size = 147456 # 3.09 GiB ctx-size = 163840 # 3.44 GiB ctx-size = 180224 # 3.78 GiB ctx-size = 196608 # 4.12 GiB ctx-size = 229376 # 4.81 GiB ctx-size = 245760 # 5.16 GiB ctx-size = 262144 # 5.50 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q5_0 B/elem = 0.625 + 0.6875 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q4_1 B/elem = 0.625 + 0.625 = 1.25 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = q4_0 B/elem = 0.625 + 0.5625 = 1.1875 KiB/token = 19.00 GiB = ctx / 55188.21 ctx-size = 2048 # 0.037 GiB ctx-size = 4096 # 0.074 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.15 GiB ctx-size = 12288 # 0.22 GiB ctx-size = 16384 # 0.30 GiB ctx-size = 24576 # 0.45 GiB ctx-size = 32768 # 0.59 GiB ctx-size = 40960 # 0.74 GiB ctx-size = 49152 # 0.89 GiB ctx-size = 65536 # 1.19 GiB ctx-size = 81920 # 1.48 GiB ctx-size = 98304 # 1.78 GiB ctx-size = 114688 # 2.08 GiB ctx-size = 131072 # 2.38 GiB ctx-size = 147456 # 2.67 GiB ctx-size = 163840 # 2.97 GiB ctx-size = 180224 # 3.27 GiB ctx-size = 196608 # 3.56 GiB ctx-size = 229376 # 4.16 GiB ctx-size = 245760 # 4.45 GiB ctx-size = 262144 # 4.75 GiB ------------------------------------------------------------------------ cache-type-k = q4_1 cache-type-v = iq4_nl B/elem = 0.625 + 0.5625 = 1.1875 KiB/token = 19.00 GiB = ctx / 55188.21 ctx-size = 2048 # 0.037 GiB ctx-size = 4096 # 0.074 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.15 GiB ctx-size = 12288 # 0.22 GiB ctx-size = 16384 # 0.30 GiB ctx-size = 24576 # 0.45 GiB ctx-size = 32768 # 0.59 GiB ctx-size = 40960 # 0.74 GiB ctx-size = 49152 # 0.89 GiB ctx-size = 65536 # 1.19 GiB ctx-size = 81920 # 1.48 GiB ctx-size = 98304 # 1.78 GiB ctx-size = 114688 # 2.08 GiB ctx-size = 131072 # 2.38 GiB ctx-size = 147456 # 2.67 GiB ctx-size = 163840 # 2.97 GiB ctx-size = 180224 # 3.27 GiB ctx-size = 196608 # 3.56 GiB ctx-size = 229376 # 4.16 GiB ctx-size = 245760 # 4.45 GiB ctx-size = 262144 # 4.75 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = f32 B/elem = 0.5625 + 4.0 = 4.5625 KiB/token = 73.00 GiB = ctx / 14364.05 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.57 GiB ctx-size = 12288 # 0.86 GiB ctx-size = 16384 # 1.14 GiB ctx-size = 24576 # 1.71 GiB ctx-size = 32768 # 2.28 GiB ctx-size = 40960 # 2.85 GiB ctx-size = 49152 # 3.42 GiB ctx-size = 65536 # 4.56 GiB ctx-size = 81920 # 5.70 GiB ctx-size = 98304 # 6.84 GiB ctx-size = 114688 # 7.98 GiB ctx-size = 131072 # 9.12 GiB ctx-size = 147456 # 10.3 GiB ctx-size = 163840 # 11.4 GiB ctx-size = 180224 # 12.5 GiB ctx-size = 196608 # 13.7 GiB ctx-size = 229376 # 16.0 GiB ctx-size = 245760 # 17.1 GiB ctx-size = 262144 # 18.2 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = f16 B/elem = 0.5625 + 2.0 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = bf16 B/elem = 0.5625 + 2.0 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q8_0 B/elem = 0.5625 + 1.0625 = 1.625 KiB/token = 26.00 GiB = ctx / 40329.85 ctx-size = 2048 # 0.051 GiB ctx-size = 4096 # 0.10 GiB ctx-size = 6144 # 0.15 GiB ctx-size = 8192 # 0.20 GiB ctx-size = 12288 # 0.30 GiB ctx-size = 16384 # 0.41 GiB ctx-size = 24576 # 0.61 GiB ctx-size = 32768 # 0.81 GiB ctx-size = 40960 # 1.02 GiB ctx-size = 49152 # 1.22 GiB ctx-size = 65536 # 1.62 GiB ctx-size = 81920 # 2.03 GiB ctx-size = 98304 # 2.44 GiB ctx-size = 114688 # 2.84 GiB ctx-size = 131072 # 3.25 GiB ctx-size = 147456 # 3.66 GiB ctx-size = 163840 # 4.06 GiB ctx-size = 180224 # 4.47 GiB ctx-size = 196608 # 4.88 GiB ctx-size = 229376 # 5.69 GiB ctx-size = 245760 # 6.09 GiB ctx-size = 262144 # 6.50 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q5_1 B/elem = 0.5625 + 0.75 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q5_0 B/elem = 0.5625 + 0.6875 = 1.25 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q4_1 B/elem = 0.5625 + 0.625 = 1.1875 KiB/token = 19.00 GiB = ctx / 55188.21 ctx-size = 2048 # 0.037 GiB ctx-size = 4096 # 0.074 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.15 GiB ctx-size = 12288 # 0.22 GiB ctx-size = 16384 # 0.30 GiB ctx-size = 24576 # 0.45 GiB ctx-size = 32768 # 0.59 GiB ctx-size = 40960 # 0.74 GiB ctx-size = 49152 # 0.89 GiB ctx-size = 65536 # 1.19 GiB ctx-size = 81920 # 1.48 GiB ctx-size = 98304 # 1.78 GiB ctx-size = 114688 # 2.08 GiB ctx-size = 131072 # 2.38 GiB ctx-size = 147456 # 2.67 GiB ctx-size = 163840 # 2.97 GiB ctx-size = 180224 # 3.27 GiB ctx-size = 196608 # 3.56 GiB ctx-size = 229376 # 4.16 GiB ctx-size = 245760 # 4.45 GiB ctx-size = 262144 # 4.75 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = q4_0 B/elem = 0.5625 + 0.5625 = 1.125 KiB/token = 18.00 GiB = ctx / 58254.22 ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB ------------------------------------------------------------------------ cache-type-k = q4_0 cache-type-v = iq4_nl B/elem = 0.5625 + 0.5625 = 1.125 KiB/token = 18.00 GiB = ctx / 58254.22 ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = f32 B/elem = 0.5625 + 4.0 = 4.5625 KiB/token = 73.00 GiB = ctx / 14364.05 ctx-size = 2048 # 0.14 GiB ctx-size = 4096 # 0.29 GiB ctx-size = 6144 # 0.43 GiB ctx-size = 8192 # 0.57 GiB ctx-size = 12288 # 0.86 GiB ctx-size = 16384 # 1.14 GiB ctx-size = 24576 # 1.71 GiB ctx-size = 32768 # 2.28 GiB ctx-size = 40960 # 2.85 GiB ctx-size = 49152 # 3.42 GiB ctx-size = 65536 # 4.56 GiB ctx-size = 81920 # 5.70 GiB ctx-size = 98304 # 6.84 GiB ctx-size = 114688 # 7.98 GiB ctx-size = 131072 # 9.12 GiB ctx-size = 147456 # 10.3 GiB ctx-size = 163840 # 11.4 GiB ctx-size = 180224 # 12.5 GiB ctx-size = 196608 # 13.7 GiB ctx-size = 229376 # 16.0 GiB ctx-size = 245760 # 17.1 GiB ctx-size = 262144 # 18.2 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = f16 B/elem = 0.5625 + 2.0 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = bf16 B/elem = 0.5625 + 2.0 = 2.5625 KiB/token = 41.00 GiB = ctx / 25575.02 ctx-size = 2048 # 0.080 GiB ctx-size = 4096 # 0.16 GiB ctx-size = 6144 # 0.24 GiB ctx-size = 8192 # 0.32 GiB ctx-size = 12288 # 0.48 GiB ctx-size = 16384 # 0.64 GiB ctx-size = 24576 # 0.96 GiB ctx-size = 32768 # 1.28 GiB ctx-size = 40960 # 1.60 GiB ctx-size = 49152 # 1.92 GiB ctx-size = 65536 # 2.56 GiB ctx-size = 81920 # 3.20 GiB ctx-size = 98304 # 3.84 GiB ctx-size = 114688 # 4.48 GiB ctx-size = 131072 # 5.12 GiB ctx-size = 147456 # 5.77 GiB ctx-size = 163840 # 6.41 GiB ctx-size = 180224 # 7.05 GiB ctx-size = 196608 # 7.69 GiB ctx-size = 229376 # 8.97 GiB ctx-size = 245760 # 9.61 GiB ctx-size = 262144 # 10.2 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = q8_0 B/elem = 0.5625 + 1.0625 = 1.625 KiB/token = 26.00 GiB = ctx / 40329.85 ctx-size = 2048 # 0.051 GiB ctx-size = 4096 # 0.10 GiB ctx-size = 6144 # 0.15 GiB ctx-size = 8192 # 0.20 GiB ctx-size = 12288 # 0.30 GiB ctx-size = 16384 # 0.41 GiB ctx-size = 24576 # 0.61 GiB ctx-size = 32768 # 0.81 GiB ctx-size = 40960 # 1.02 GiB ctx-size = 49152 # 1.22 GiB ctx-size = 65536 # 1.62 GiB ctx-size = 81920 # 2.03 GiB ctx-size = 98304 # 2.44 GiB ctx-size = 114688 # 2.84 GiB ctx-size = 131072 # 3.25 GiB ctx-size = 147456 # 3.66 GiB ctx-size = 163840 # 4.06 GiB ctx-size = 180224 # 4.47 GiB ctx-size = 196608 # 4.88 GiB ctx-size = 229376 # 5.69 GiB ctx-size = 245760 # 6.09 GiB ctx-size = 262144 # 6.50 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = q5_1 B/elem = 0.5625 + 0.75 = 1.3125 KiB/token = 21.00 GiB = ctx / 49932.19 ctx-size = 2048 # 0.041 GiB ctx-size = 4096 # 0.082 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.25 GiB ctx-size = 16384 # 0.33 GiB ctx-size = 24576 # 0.49 GiB ctx-size = 32768 # 0.66 GiB ctx-size = 40960 # 0.82 GiB ctx-size = 49152 # 0.98 GiB ctx-size = 65536 # 1.31 GiB ctx-size = 81920 # 1.64 GiB ctx-size = 98304 # 1.97 GiB ctx-size = 114688 # 2.30 GiB ctx-size = 131072 # 2.62 GiB ctx-size = 147456 # 2.95 GiB ctx-size = 163840 # 3.28 GiB ctx-size = 180224 # 3.61 GiB ctx-size = 196608 # 3.94 GiB ctx-size = 229376 # 4.59 GiB ctx-size = 245760 # 4.92 GiB ctx-size = 262144 # 5.25 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = q5_0 B/elem = 0.5625 + 0.6875 = 1.25 KiB/token = 20.00 GiB = ctx / 52428.80 ctx-size = 2048 # 0.039 GiB ctx-size = 4096 # 0.078 GiB ctx-size = 6144 # 0.12 GiB ctx-size = 8192 # 0.16 GiB ctx-size = 12288 # 0.23 GiB ctx-size = 16384 # 0.31 GiB ctx-size = 24576 # 0.47 GiB ctx-size = 32768 # 0.62 GiB ctx-size = 40960 # 0.78 GiB ctx-size = 49152 # 0.94 GiB ctx-size = 65536 # 1.25 GiB ctx-size = 81920 # 1.56 GiB ctx-size = 98304 # 1.88 GiB ctx-size = 114688 # 2.19 GiB ctx-size = 131072 # 2.50 GiB ctx-size = 147456 # 2.81 GiB ctx-size = 163840 # 3.12 GiB ctx-size = 180224 # 3.44 GiB ctx-size = 196608 # 3.75 GiB ctx-size = 229376 # 4.38 GiB ctx-size = 245760 # 4.69 GiB ctx-size = 262144 # 5.00 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = q4_1 B/elem = 0.5625 + 0.625 = 1.1875 KiB/token = 19.00 GiB = ctx / 55188.21 ctx-size = 2048 # 0.037 GiB ctx-size = 4096 # 0.074 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.15 GiB ctx-size = 12288 # 0.22 GiB ctx-size = 16384 # 0.30 GiB ctx-size = 24576 # 0.45 GiB ctx-size = 32768 # 0.59 GiB ctx-size = 40960 # 0.74 GiB ctx-size = 49152 # 0.89 GiB ctx-size = 65536 # 1.19 GiB ctx-size = 81920 # 1.48 GiB ctx-size = 98304 # 1.78 GiB ctx-size = 114688 # 2.08 GiB ctx-size = 131072 # 2.38 GiB ctx-size = 147456 # 2.67 GiB ctx-size = 163840 # 2.97 GiB ctx-size = 180224 # 3.27 GiB ctx-size = 196608 # 3.56 GiB ctx-size = 229376 # 4.16 GiB ctx-size = 245760 # 4.45 GiB ctx-size = 262144 # 4.75 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = q4_0 B/elem = 0.5625 + 0.5625 = 1.125 KiB/token = 18.00 GiB = ctx / 58254.22 ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB ------------------------------------------------------------------------ cache-type-k = iq4_nl cache-type-v = iq4_nl B/elem = 0.5625 + 0.5625 = 1.125 KiB/token = 18.00 GiB = ctx / 58254.22 ctx-size = 2048 # 0.035 GiB ctx-size = 4096 # 0.070 GiB ctx-size = 6144 # 0.11 GiB ctx-size = 8192 # 0.14 GiB ctx-size = 12288 # 0.21 GiB ctx-size = 16384 # 0.28 GiB ctx-size = 24576 # 0.42 GiB ctx-size = 32768 # 0.56 GiB ctx-size = 40960 # 0.70 GiB ctx-size = 49152 # 0.84 GiB ctx-size = 65536 # 1.12 GiB ctx-size = 81920 # 1.41 GiB ctx-size = 98304 # 1.69 GiB ctx-size = 114688 # 1.97 GiB ctx-size = 131072 # 2.25 GiB ctx-size = 147456 # 2.53 GiB ctx-size = 163840 # 2.81 GiB ctx-size = 180224 # 3.09 GiB ctx-size = 196608 # 3.38 GiB ctx-size = 229376 # 3.94 GiB ctx-size = 245760 # 4.22 GiB ctx-size = 262144 # 4.50 GiB