# B70 on oikeasti kakkoskortti hitaammassa PCIe-väylässä, mutta softat näyttää välillä näin päin
[hans@pcmr ~]$ inxi -GSD -mm
System:
  Host: pcmr Kernel: 7.2.0-1-cachyos arch: x86_64 bits: 64
  Desktop: KDE Plasma v: 6.7.4 Distro: CachyOS
Memory:
  System RAM: total: 64 GiB available: 62.71 GiB used: 8.7 GiB (13.9%)
  Array-1: capacity: 128 GiB slots: 4 modules: 4 EC: None
  Device-1: Channel-A DIMM 0 type: DDR4 size: 16 GiB speed: 3200 MT/s
  Device-2: Channel-A DIMM 1 type: DDR4 size: 16 GiB speed: 3200 MT/s
  Device-3: Channel-B DIMM 0 type: DDR4 size: 16 GiB speed: 3200 MT/s
  Device-4: Channel-B DIMM 1 type: DDR4 size: 16 GiB speed: 3200 MT/s
Graphics:
  Device-1: Intel Battlemage G31 [Arc Pro B70] driver: xe v: kernel
  Device-2: Advanced Micro Devices [AMD/ATI] Navi 48 [Radeon RX 9070/9070
    XT/9070 GRE] driver: amdgpu v: kernel
  Display: wayland server: X.org v: 1.21.1.24 with: Xwayland v: 24.1.13
    compositor: kwin_wayland driver: gpu: amdgpu resolution: 3840x2160~120Hz
  API: EGL v: 1.5 drivers: iris,radeonsi,swrast
    platforms: gbm,wayland,x11,surfaceless,device
  API: OpenGL v: 4.6 vendor: amd mesa v: 26.3.0-devel renderer: AMD Radeon
    RX 9070 XT (radeonsi gfx1201 ACO DRM 3.64 7.2.0-1-cachyos)
  API: Vulkan v: 1.4.357 drivers: radv,intel,llvmpipe surfaces: N/A
  Info: Tools: api: clinfo, eglinfo, glxinfo, vulkaninfo
    de: kscreen-console,kscreen-doctor gpu: amd-smi, amdgpu_top, gputop,
    intel_gpu_top, lsgpu, lact, nvidia-smi wl: wayland-info
    x11: xdpyinfo, xprop, xrandr
Drives:
  Local Storage: total: 8.19 TiB used: 4.77 TiB (58.2%)
  ID-1: /dev/nvme0n1 vendor: Samsung model: SSD 990 PRO with Heatsink 4TB
    size: 3.64 TiB
  ID-2: /dev/sda vendor: Western Digital model: WD50NDZW-11BCSS1
    size: 4.55 TiB type: USB
[hans@pcmr ~]$ cat /proc/cmdline 
nvme_core.default_ps_max_latency_us=0 nvme.poll_queues=8 hpet=disable amdgpu.freesync_video=1 amdgpu.wbrf=0 amdgpu.seamless=1 xe.force_probe=e223 pcie_aspm=off pci=noaer iommu=pt usbcore.autosuspend=-1 amdgpu.ppfeaturemask=0xffffffff mitigations=off nokaslr norandmaps split_lock_detect=off split_lock_mitigate=0 quiet rw rootflags=subvol=/@ root=UUID=REDACTED

~/ai/cache

~/ai/input/lähdekuva.png

~/ai/models/audio_encoders
~/ai/models/checkpoints
~/ai/models/clip_vision
~/ai/models/configs
~/ai/models/controlnet
~/ai/models/diffusers
~/ai/models/diffusion_models
~/ai/models/inpaint
~/ai/models/ipadapter
~/ai/models/latent_upscale_models
~/ai/models/llm
~/ai/models/llm_aux
~/ai/models/llm_presets
~/ai/models/loras
~/ai/models/model_patches
~/ai/models/sams
~/ai/models/style_models
~/ai/models/text_encoders
~/ai/models/ultralytics
~/ai/models/upscale_models
~/ai/models/vae

~/ai/onetrainer

~/ai/output/generoitu_kuva.png
~/ai/output/video/generoitu_video.mp4

~/ai/rocm

~/ai/src

~/ai/training/datasets/onetrainer_lähdedata

~/ai/user/default/workflows/comfy_workflow.json

~/ai/xpu

#!/usr/bin/env bash
# ~/.local/bin/updateai
# updateai — CachyOS dual ComfyUI + llama.cpp + OneTrainer
#   rocm       : ComfyUI ROCm + llama.cpp Vulkan  → ${HOME}/ai/rocm
#   xpu        : ComfyUI XPU  + llama.cpp SYCL    → ${HOME}/ai/xpu
#   onetrainer : OneTrainer XPU (PR 1413)         → ${HOME}/ai/onetrainer
# Runtime: CachyOS, uv, no containers. Requires Bash 5+, uv, git, curl,
#          makepkg+sudo (ROCm), cmake, vulkan-headers, shaderc, spirv-headers,
#          Intel oneAPI Base Toolkit (SYCL). Optional: ccache, ninja.
set -euo pipefail

readonly BASEDIR="${HOME}/ai"
readonly CACHE_DIR="${BASEDIR}/cache"
readonly PYTHON_VER="3.13"
readonly UV_INSTALL_URL="https://astral.sh/uv/install.sh"
readonly ROCM_PATH="/opt/rocm"
readonly ROCM_NIGHTLY_INDEX="https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/"
readonly ROCM_TARBALL_BASE="https://rocm.nightlies.amd.com/tarball-multi-arch"
readonly PYTORCH_XPU_INDEX="https://download.pytorch.org/whl/nightly/xpu"
readonly PYTORCH_XPU_STABLE_INDEX="https://download.pytorch.org/whl/xpu"
readonly ONETRAINER_REPO="https://github.com/Nerogar/OneTrainer.git"
readonly ONETRAINER_DIR="${BASEDIR}/onetrainer"
readonly ONETRAINER_PR="1413"
readonly ONETRAINER_DIFFUSERS_REF="1ffa423"
readonly ONETRAINER_MGDS_REF="3a6994a"
readonly ONETRAINER_MUON_REF="f90a42b"
readonly LLAMA_CPP_REPO="https://github.com/ggml-org/llama.cpp.git"
readonly LLAMA_SRC_DIR="${BASEDIR}/src/llama.cpp"
readonly ONEAPI_SETVARS="/opt/intel/oneapi/setvars.sh"
readonly UNIT_DIR="${HOME}/.config/systemd/user"
readonly APP_DIR="${HOME}/.local/share/applications"
readonly LLM_MODELS_DIR="${BASEDIR}/models/llm"
readonly LLM_AUX_DIR="${BASEDIR}/models/llm_aux"
readonly LLM_PRESETS_DIR="${BASEDIR}/models/llm_presets"
readonly LLM_PRESET_FILE_16GB="${LLM_PRESETS_DIR}/16gb.ini"
readonly LLM_PRESET_FILE_32GB="${LLM_PRESETS_DIR}/32gb.ini"
readonly LLM_PRESET_FILE_SYSTEM_PROMPT_EN="${LLM_PRESETS_DIR}/system_prompt_en.txt"
readonly LLM_PRESET_FILE_SYSTEM_PROMPT_FI="${LLM_PRESETS_DIR}/system_prompt_fi.txt"
readonly LLM_PRESET_FILE_ONETRAINER_APP="${APP_DIR}/OneTrainer.desktop"
readonly RADEON_ICD="/usr/share/vulkan/icd.d/radeon_icd.x86_64.json"

readonly REPOS=(
    "https://github.com/Acly/comfyui-inpaint-nodes comfyui_inpaint_nodes"
    "https://github.com/Acly/comfyui-tooling-nodes comfyui_tooling_nodes"
    "https://github.com/AIToldMeTo/comfyui-cache-cleaner comfyui_cache_cleaner"
    "https://github.com/alexopus/ComfyUI-Image-Saver comfyui_image_saver"
    "https://github.com/Azornes/Comfyui-Resolution-Master comfyui_resolution_master"
    "https://github.com/BadCafeCode/masquerade-nodes-comfyui masquerade_nodes_comfyui"
    "https://github.com/Billy024/comfyui-aspect-ratio-crop-node comfyui_aspect_ratio_crop_node"
    "https://github.com/chflame163/ComfyUI_LayerStyle comfyui_layerstyle"
    "https://github.com/chrisgoringe/cg-use-everywhere cg_use_everywhere"
    "https://github.com/city96/ComfyUI-GGUF comfyui_gguf"
    "https://github.com/ClownsharkBatwing/RES4LYF res4lyf"
    "https://github.com/crystian/ComfyUI-Crystools comfyui_crystools"
    "https://github.com/cubiq/ComfyUI_essentials comfyui_essentials"
    "https://github.com/cubiq/ComfyUI_IPAdapter_plus comfyui_ipadapter_plus"
    "https://github.com/DoctorDiffusion/ComfyUI-MediaMixer comfyui_mediamixer"
    "https://github.com/Fannovel16/comfyui_controlnet_aux comfyui_controlnet_aux"
    "https://github.com/Fannovel16/ComfyUI-Frame-Interpolation comfyui_frame_interpolation"
    "https://github.com/JPS-GER/ComfyUI_JPS-Nodes comfyui_jps_nodes"
    "https://github.com/kijai/ComfyUI-KJNodes comfyui_kjnodes"
    "https://github.com/kijai/ComfyUI-MelBandRoFormer comfyui_melbandroformer"
    "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite comfyui_videohelpersuite"
    "https://github.com/LAOGOU-666/Comfyui-Memory_Cleanup comfyui_memory_cleanup"
    "https://github.com/ltdrdata/ComfyUI-Impact-Pack comfyui_impact_pack"
    "https://github.com/ltdrdata/ComfyUI-Impact-Subpack comfyui_impact_subpack"
    "https://github.com/ltdrdata/ComfyUI-Manager comfyui_manager"
    "https://github.com/orssorbit/ComfyUI-wanBlockswap comfyui_wanblockswap"
    "https://github.com/pythongosssss/ComfyUI-Custom-Scripts comfyui_custom_scripts"
    "https://github.com/rgthree/rgthree-comfy rgthree_comfy"
    "https://github.com/SeanScripts/ComfyUI-Unload-Model comfyui_unload_model"
    "https://github.com/Smirnov75/ComfyUI-mxToolkit comfyui_mxtoolkit"
    "https://github.com/tritant/ComfyUI_Custom_Switch comfyui_custom_switch"
    "https://github.com/WASasquatch/was-node-suite-comfyui was_node_suite_comfyui"
    "https://github.com/yolain/ComfyUI-Easy-Use comfyui_easy_use"
)

start_time=${SECONDS}
CMAKE_GENERATOR_ARGS=()
NOTES=()

log() { printf '\n=== %s ===\n\n' "$*"; }
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
note() { NOTES+=("$1"); }
print_notes() {
    local n
    for n in "${NOTES[@]}"; do
        printf '\n%s\n' "$n"
    done
}

usage() {
    cat <<EOF
Usage: $(basename "$0") [rocm|xpu|onetrainer|all]
  rocm        ComfyUI ROCm + llama.cpp Vulkan   \${BASEDIR}/rocm
  xpu         ComfyUI XPU  + llama.cpp SYCL     \${BASEDIR}/xpu
  onetrainer  OneTrainer XPU (PR ${ONETRAINER_PR})
  all         rocm + xpu + onetrainer

Services:
  comfy-rocm.service     :8189
  llama-vulkan.service   :9932
  comfy-xpu.service      :8188
  llama-sycl.service     :9931
EOF
    exit 1
}

ensure_uv() {
    export PATH="${HOME}/.local/bin:${PATH}"
    if command -v uv >/dev/null 2>&1; then
        log "uv already installed ($(uv --version 2>/dev/null || echo unknown))"
        return 0
    fi
    log "Installing uv"
    curl -LsSf "${UV_INSTALL_URL}" | sh
    export PATH="${HOME}/.local/bin:${PATH}"
}

stop_service() {
    systemctl --user daemon-reload >/dev/null 2>&1 || true
    systemctl --user stop "$1" >/dev/null 2>&1 || true
}

enable_units() {
    mkdir -p "${UNIT_DIR}"
    systemctl --user daemon-reload
}

clean_base() {
    log "Completely reinstalling $1"
    rm -rf "$1"
    mkdir -p "$1"
}

ensure_shared_dirs() {
    local -a dirs=(
        "${BASEDIR}/input" "${BASEDIR}/output" "${BASEDIR}/user"
        "${BASEDIR}/models" "${LLM_MODELS_DIR}" "${LLM_AUX_DIR}" "${LLM_PRESETS_DIR}"
        "${CACHE_DIR}/tunableop" "${CACHE_DIR}/miopen" "${CACHE_DIR}/triton"
        "${CACHE_DIR}/sycl" "${CACHE_DIR}/torchinductor" "${UNIT_DIR}" "${APP_DIR}"
    )
    local d
    for d in "${dirs[@]}"; do
        mkdir -p "${d}"
    done
}

git_rev() {
    git -C "$1" rev-parse --short HEAD 2>/dev/null || echo unknown
}

write_manifest() {
    local dest=$1
    shift
    {
        printf 'updated=%s\n' "$(date -Iseconds)"
        printf 'llama_src=%s\n' "$(git_rev "${LLAMA_SRC_DIR}")"
        printf '%s\n' "$@"
    } > "${dest}"
}

# ---------------------------------------------------------------------------
# ComfyUI + venv
# ---------------------------------------------------------------------------
setup_venv_and_comfy() {
    local basedir=$1
    cd "${basedir}"
    log "Creating Python ${PYTHON_VER} venv and cloning ComfyUI"
    uv venv --python "${PYTHON_VER}" venv
    # shellcheck source=/dev/null
    source venv/bin/activate
    git clone --quiet --no-tags --depth 1 https://github.com/comfyanonymous/ComfyUI comfy
    cd comfy
}

install_custom_nodes() {
    log "Installing ComfyUI custom nodes (parallel)"
    mkdir -p custom_nodes
    cd custom_nodes
    printf '%s\n' "${REPOS[@]}" | xargs -P "$(nproc)" -L 1 sh -c '
        git clone --quiet --no-tags --depth 1 "$0" "$1" || { echo "FAIL $0" >&2; exit 1; }
    '
    cd ..
}

update_pip_and_requirements() {
    log "Upgrading pip and installing requirements"
    uv pip install --upgrade pip wheel wheel_stub
    local -a req_args=("-r" "requirements.txt")
    local req_file
    while IFS= read -r -d '' req_file; do
        [[ -s "${req_file}" ]] && req_args+=("-r" "${req_file}")
    done < <(find custom_nodes -name "requirements.txt" -type f -print0)
    uv pip install --upgrade "${req_args[@]}"
}

setup_extra_model_paths() {
    log "Configuring extra_model_paths.yaml"
    local -a modeldirs=(
        audio_encoders checkpoints clip_vision configs controlnet
        diffusion_models inpaint ipadapter latent_upscale_models
        loras model_patches sams style_models text_encoders
        llm llm_aux llm_presets ultralytics upscale_models vae
    )
    {
        printf 'my_models:\n  base_path: %s\n' "${BASEDIR}/models"
        local dir
        for dir in "${modeldirs[@]}"; do
            mkdir -p "${BASEDIR}/models/${dir}"
            printf '  %s: %s\n' "${dir}" "${dir}"
        done
    } > extra_model_paths.yaml
}

# ---------------------------------------------------------------------------
# llama.cpp
# ---------------------------------------------------------------------------
ensure_llm_preset() {
    log "Writing LLM preset: ${LLM_PRESET_FILE_16GB}"
    cat > "${LLM_PRESET_FILE_16GB}" <<PRESET_EOF
# 16 GiB VRAM (primary desktop GPU, single user)
# ${LLM_PRESET_FILE_16GB}
[*]
flash-attn = on
gpu-layers = all
parallel = 1
cache-type-k = q4_0
cache-type-v = q4_0
batch-size = 384
ubatch-size = 384
load-mode = mlock
spec-type = draft-mtp,ngram-mod
spec-draft-n-max = 2
spec-ngram-mod-n-match = 24
spec-ngram-mod-n-min = 24
spec-ngram-mod-n-max = 32
image-min-tokens = 1024
image-max-tokens = 2048

# KV cache VRAM
# Qwen3.8-27B: 16 attn layers × 4 KV heads × 256 dim × 2 (K+V)
# q4_0/q4_0: 0.5 B/elem → 16 KiB/token → GiB ≈ ctx / 65536
# https://datakeskus.io/files/qwen38-27b-kv-cache.txt
#ctx-size = 2048    # 0.03 GiB
#ctx-size = 4096    # 0.06 GiB
#ctx-size = 6144    # 0.09 GiB
#ctx-size = 8192    # 0.13 GiB
#ctx-size = 16384   # 0.25 GiB
#ctx-size = 32768   # 0.50 GiB
#ctx-size = 49152   # 0.75 GiB
#ctx-size = 65536   # 1.00 GiB
#ctx-size = 81920   # 1.25 GiB
#ctx-size = 98304   # 1.50 GiB
#ctx-size = 114688  # 1.75 GiB
#ctx-size = 131072  # 2.00 GiB
#ctx-size = 147456  # 2.25 GiB
#ctx-size = 163840  # 2.50 GiB
#ctx-size = 180224  # 2.75 GiB
#ctx-size = 196608  # 3.00 GiB
#ctx-size = 229376  # 3.50 GiB
#ctx-size = 245760  # 3.75 GiB
#ctx-size = 262144  # 4.00 GiB

# Model VRAM
# Qwen3.8-27B-ZB4.00-MIN-v3-IQ4_XS.gguf     12.7 GiB
# mmproj-Qwen3.8-27B-BF16.gguf              0.89 GiB
# https://huggingface.co/tooltd/Qwen3.8-27B-IQ4-XS-16GB-VRAM-GGUF
# https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF

[Instruct]
ctx-size = 32768
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision CPU]
ctx-size = 32768
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-BF16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision GPU]
ctx-size = 32768
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-BF16.gguf
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Thinking]
ctx-size = 32768
m = ${LLM_MODELS_DIR}/Qwen3.8-27B-ZB4.00-MIN-v3-IQ4_XS.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision CPU]
ctx-size = 32768
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-BF16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision GPU]
ctx-size = 32768
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-BF16.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}
PRESET_EOF

    log "Writing LLM preset: ${LLM_PRESET_FILE_32GB}"
    cat > "${LLM_PRESET_FILE_32GB}" <<PRESET_EOF
# 32 GiB VRAM (dedicated AI GPU, single user)
# ${LLM_PRESET_FILE_32GB}
[*]
flash-attn = on
gpu-layers = all
parallel = 1
cache-type-k = q8_0
cache-type-v = q4_1
batch-size = 4096
ubatch-size = 2048
load-mode = none
spec-type = draft-mtp,ngram-mod
spec-draft-n-max = 3
spec-ngram-mod-n-match = 24
spec-ngram-mod-n-min = 24
spec-ngram-mod-n-max = 32
image-min-tokens = 1024
image-max-tokens = 2048

# KV cache VRAM
# Qwen3.8-27B: 16 attn layers × 4 KV heads × 256 dim × 2 (K+V)
# q8_0/q4_1 : ~0.75 B/elem → 24 KiB/token → GiB ≈ ctx / 43690
# https://datakeskus.io/files/qwen38-27b-kv-cache.txt
#ctx-size = 2048     # 0.05 GiB
#ctx-size = 4096     # 0.09 GiB
#ctx-size = 8192     # 0.19 GiB
#ctx-size = 16384    # 0.38 GiB
#ctx-size = 32768    # 0.75 GiB
#ctx-size = 49152    # 1.13 GiB
#ctx-size = 65536    # 1.50 GiB
#ctx-size = 87040    # 2.00 GiB
#ctx-size = 98304    # 2.25 GiB
#ctx-size = 131072   # 3.00 GiB
#ctx-size = 163840   # 3.75 GiB
#ctx-size = 196608   # 4.50 GiB
#ctx-size = 229376   # 5.25 GiB
#ctx-size = 262144   # 6.00 GiB

# Model VRAM
# Qwen3.8-27B-Uncensored-Q6_K.gguf          20.9 GiB
# mmproj-Qwen3.8-27B-Uncensored-f16.gguf    0.89 GiB
# https://huggingface.co/orcarouter/Qwen3.8-27B-Uncensored-GGUF
m = ${LLM_MODELS_DIR}/Qwen3.8-27B-Uncensored-Q6_K.gguf

[Instruct]
ctx-size = 98304
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision CPU]
ctx-size = 98304
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision GPU]
ctx-size = 98304
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Thinking]
ctx-size = 98304
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision CPU]
ctx-size = 98304
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision GPU]
ctx-size = 98304
mmproj = ${LLM_AUX_DIR}/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}
PRESET_EOF

    log "Writing system prompt: ${LLM_PRESET_FILE_SYSTEM_PROMPT_EN}"
    cat > "${LLM_PRESET_FILE_SYSTEM_PROMPT_EN}" <<'PRESET_EOF'
Role: Software engineer. Treat the user as a peer. No tutoring.

Language: Everything in English, unless requested otherwise.

Style: Direct, technically precise, concise. No pleasantries, flattery, or filler.

Constraints: No emojis. Do not end responses with a question. Stay on task; no unsolicited expansions.

Code and comments: Prefer simple, fast, compact, readable code. Do not overcomplicate. Comment only non-obvious logic.

Python: Google Python Style Guide, Black, reST function docstrings.

Bash 5+ on Linux only. Google Shell Style Guide, ShellCheck, shfmt. Use set -euo pipefail. No POSIX or cross-platform compatibility.
PRESET_EOF

    log "Writing system prompt: ${LLM_PRESET_FILE_SYSTEM_PROMPT_FI}"
    cat > "${LLM_PRESET_FILE_SYSTEM_PROMPT_FI}" <<'PRESET_EOF'
Role: Software engineer. Treat the user as a peer. No tutoring.

Language: Base text in Finnish. No colloquial speak (sä, sun, sulla, mä, mun, mulla), do not address formally, use regular Finnish (sinä, sinun, sinulla, minä, minun, minulla).

Style: Direct, technically precise, concise. No pleasantries, flattery, or filler.

Constraints: No emojis. Do not end responses with a question. Stay on task; no unsolicited expansions.

Code and comments in English. Prefer simple, fast, compact, readable code. Do not overcomplicate. Comment only non-obvious logic.

Python: Google Python Style Guide, Black, reST function docstrings.

Bash 5+ on Linux only. Google Shell Style Guide, ShellCheck, shfmt. Use set -euo pipefail. No POSIX or cross-platform compatibility.
PRESET_EOF

    log "Writing system prompt: ${LLM_PRESET_FILE_ONETRAINER_APP}"
    cat > "${LLM_PRESET_FILE_ONETRAINER_APP}" <<PRESET_EOF
# ${LLM_PRESET_FILE_ONETRAINER_APP}
[Desktop Entry]
Comment=
Exec=/usr/bin/bash -c 'cd ${ONETRAINER_DIR}; source venv/bin/activate; python scripts/train_ui_qt.py &>log.txt'
Icon=${ONETRAINER_DIR}/resources/icons/icon.png
Name=OneTrainer
NoDisplay=false
Path=
PrefersNonDefaultGPU=false
StartupNotify=true
Terminal=false
TerminalOptions=
Type=Application
X-KDE-SubstituteUID=false
X-KDE-Username=
PRESET_EOF
}

ensure_llama_src() {
    mkdir -p "$(dirname "${LLAMA_SRC_DIR}")"
    if [[ -d "${LLAMA_SRC_DIR}/.git" ]]; then
        log "Updating llama.cpp source"
        git -C "${LLAMA_SRC_DIR}" fetch --depth 1 origin
        git -C "${LLAMA_SRC_DIR}" reset --hard origin/HEAD
    else
        log "Cloning llama.cpp → ${LLAMA_SRC_DIR}"
        rm -rf "${LLAMA_SRC_DIR}"
        git clone --quiet --no-tags --depth 1 "${LLAMA_CPP_REPO}" "${LLAMA_SRC_DIR}"
    fi
}

setup_build_env() {
    if command -v ccache >/dev/null 2>&1; then
        export CMAKE_C_COMPILER_LAUNCHER=ccache
        export CMAKE_CXX_COMPILER_LAUNCHER=ccache
        export CCACHE_COMPILERCHECK=content
        log "ccache enabled"
    fi
    if command -v ninja >/dev/null 2>&1; then
        CMAKE_GENERATOR_ARGS=(-G Ninja)
        log "Ninja generator enabled"
    else
        CMAKE_GENERATOR_ARGS=()
    fi
}

install_llama_artifacts() {
    local build_bin=$1 dest=$2
    mkdir -p "${dest}"
    cp -a "${build_bin}"/llama-* "${dest}/"
    if compgen -G "${build_bin}/*.so*" >/dev/null; then
        cp -a "${build_bin}"/*.so* "${dest}/"
    fi
    chmod +x "${dest}"/llama-*
}

build_llama_vulkan() {
    local basedir=$1
    local build_dir="${LLAMA_SRC_DIR}/build-vulkan"
    ensure_llama_src
    setup_build_env
    log "Building llama.cpp (Vulkan)"
    cd "${LLAMA_SRC_DIR}"
    cmake -B "${build_dir}" \
        "${CMAKE_GENERATOR_ARGS[@]}" \
        -DGGML_VULKAN=ON \
        -DCMAKE_BUILD_TYPE=Release
    cmake --build "${build_dir}" --config Release -j"$(nproc)"
    install_llama_artifacts "${build_dir}/bin" "${basedir}/venv/bin"
}

source_oneapi() {
    [[ -f "${ONEAPI_SETVARS}" ]] || die "oneAPI not found at ${ONEAPI_SETVARS}"
    if [[ -n "${ONEAPI_ROOT:-}" ]]; then
        return 0
    fi
    set +u
    set --
    # shellcheck source=/dev/null
    source "${ONEAPI_SETVARS}"
    set -u
}

build_llama_sycl() {
    local basedir=$1
    local build_dir="${LLAMA_SRC_DIR}/build-sycl"
    log "Sourcing oneAPI environment"
    source_oneapi
    ensure_llama_src
    setup_build_env
    log "Building llama.cpp (SYCL + F16 + oneDNN graphs)"
    cd "${LLAMA_SRC_DIR}"
    cmake -B "${build_dir}" \
        "${CMAKE_GENERATOR_ARGS[@]}" \
        -DGGML_SYCL=ON \
        -DGGML_SYCL_F16=ON \
        -DGGML_SYCL_DNN=ON \
        -DGGML_SYCL_TARGET=INTEL \
        -DGGML_SYCL_DEVICE_ARCH=bmg_g21 \
        -DCMAKE_C_COMPILER=icx \
        -DCMAKE_CXX_COMPILER=icpx \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_C_FLAGS_RELEASE="-O3 -DNDEBUG" \
        -DCMAKE_CXX_FLAGS_RELEASE="-O3 -DNDEBUG"
    cmake --build "${build_dir}" --config Release -j"$(nproc)"
    install_llama_artifacts "${build_dir}/bin" "${basedir}/venv/bin"
}

# ---------------------------------------------------------------------------
# ROCm + Vulkan
# ---------------------------------------------------------------------------
create_comfy_rocm_service() {
    local basedir_rocm=$1
    log "Writing comfy-rocm.service"
    cat > "${UNIT_DIR}/comfy-rocm.service" <<EOF
# ${UNIT_DIR}/comfy-rocm.service
[Unit]
Description=ComfyUI ROCm
After=network.target

[Service]
WorkingDirectory=${basedir_rocm}/comfy
Type=simple
Restart=always
RestartSec=10
TimeoutStopSec=10
OOMScoreAdjust=-500

Environment=ROCM_PATH=${ROCM_PATH}
Environment=ROCM_HOME=${ROCM_PATH}
Environment=HIP_PATH=${ROCM_PATH}
Environment=HIP_PLATFORM=amd
Environment=PYTORCH_TUNABLEOP_ENABLED=1
Environment=PYTORCH_TUNABLEOP_TUNING=0
Environment=PYTORCH_TUNABLEOP_FILENAME=${CACHE_DIR}/tunableop/results.csv
Environment=COMFYUI_ENABLE_MIOPEN=1
Environment=MIOPEN_FIND_MODE=FAST
Environment=PYTORCH_MIOPEN_SUGGEST_NHWC=0
Environment=PYTORCH_HIP_ALLOC_CONF=expandable_segments:True
Environment=TORCH_BLAS_PREFER_HIPBLASLT=1
Environment=HSA_ENABLE_SDMA=0
Environment=FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE
Environment=MIOPEN_USER_DB_PATH=${CACHE_DIR}/miopen
Environment=MIOPEN_CUSTOM_CACHE_DIR=${CACHE_DIR}/miopen
Environment=TRITON_CACHE_DIR=${CACHE_DIR}/triton
Environment=TORCHINDUCTOR_CACHE_DIR=${CACHE_DIR}/torchinductor

ExecStart=/bin/bash -c 'source ${basedir_rocm}/venv/bin/activate && \\
    export PATH=${ROCM_PATH}/bin:\$PATH && \\
    export LD_LIBRARY_PATH=${ROCM_PATH}/lib:${basedir_rocm}/venv/lib:\${LD_LIBRARY_PATH:-} && \\
    python main.py \\
    --input-directory ${BASEDIR}/input \\
    --output-directory ${BASEDIR}/output \\
    --user-directory ${BASEDIR}/user \\
    --port 8189 \\
    --disable-cuda-malloc \\
    --enable-dynamic-vram \\
    --use-pytorch-cross-attention \\
    --reserve-vram 2 \\
    --fast'

[Install]
WantedBy=default.target
EOF
}

create_vulkan_service() {
    local basedir_rocm=$1
    log "Writing llama-vulkan.service"
    cat > "${UNIT_DIR}/llama-vulkan.service" <<EOF
# ${UNIT_DIR}/llama-vulkan.service
[Unit]
Description=llama.cpp Vulkan
After=network.target

[Service]
WorkingDirectory=${basedir_rocm}
Type=simple
Restart=on-failure
RestartSec=5
TimeoutStopSec=15
OOMScoreAdjust=-500

Environment=VK_ICD_FILENAMES=${RADEON_ICD}
Environment=RADV_PERFTEST=transfer_queue

ExecStart=/bin/bash -c 'export LD_LIBRARY_PATH=${basedir_rocm}/venv/bin:\${LD_LIBRARY_PATH:-} && \\
    exec ${basedir_rocm}/venv/bin/llama-server \\
    --host 127.0.0.1 \\
    --port 9932 \\
    --models-max 1 \\
    --tools all \\
    --models-preset ${LLM_PRESET_FILE_16GB}'

[Install]
WantedBy=default.target
EOF
}

latest_rocm_tarball() {
    curl -sL "${ROCM_TARBALL_BASE}/" \
        | grep -oE 'therock-dist-linux-gfx120X-all-[0-9.]+a[0-9]{8}\.tar\.gz' \
        | sort -V | tail -1
}

install_rocm_nightly() {
    local basedir_rocm=$1
    local latest_tarball latest_pkgver installed
    latest_tarball=$(latest_rocm_tarball)
    [[ -n "${latest_tarball}" ]] || die "Could not determine latest ROCm tarball"
    latest_pkgver="${latest_tarball#therock-dist-linux-gfx120X-all-}"
    latest_pkgver="${latest_pkgver%.tar.gz}"
    installed=$(pacman -Q rocm-nightly 2>/dev/null | awk '{print $2}' || true)
    if [[ "${installed}" == "${latest_pkgver}-"* ]]; then
        log "ROCm nightly ${latest_pkgver} already installed, skipping makepkg"
        return 0
    fi

    log "Building ROCm nightly ${latest_pkgver}"
    local rocm_dir="${basedir_rocm}/rocm-nightly"
    mkdir -p "${rocm_dir}"
    cd "${rocm_dir}"

    cat > PKGBUILD <<PKGBUILD_EOF
pkgname="rocm-nightly"
pkgver=${latest_pkgver}
pkgrel=1
pkgdesc="ROCm nightly (RDNA4 gfx120X) — monolithic"
arch=('x86_64')
url="https://rocm.nightlies.amd.com"
license=('MIT' 'custom:LicenseRef-ROCm-EULA')
depends=('glibc' 'gcc-libs' 'python-pyelftools' 'python')
_rocm_packages=(
    hip-runtime-amd amdsmi composable-kernel
    rocm-core rocm-cmake rocm-llvm rocm-device-libs
    hsa-rocr hsakmt-roct hsa-amd-aqlprofile
    comgr rocminfo rocwmma rocprofiler-register
    rocm-smi-lib rocm-gdb rocm-dbgapi rocprofiler roctracer
    rocm-opencl-runtime rocm-opencl-sdk
    hipblas hipblas-common hipblaslt rocblas
    hipfft rocfft hipdnn hiprand rocrand hipsolver rocsolver
    hipsparse hipsparselt rocsparse
    rccl rocalution rocprim rocthrust hipcub
    hipify-clang miopen-hip
    rocm-hip-sdk rocm-hip-libraries rocm-hip-runtime rocm-ml-sdk
)
provides=("\${_rocm_packages[@]}" opencl-driver "rocm=${latest_pkgver}")
conflicts=("\${_rocm_packages[@]}" rocm)
options=('!strip' '!debug')
source=("${ROCM_TARBALL_BASE}/${latest_tarball}")
sha256sums=('SKIP')
noextract=("${latest_tarball}")

package() {
    install -d "\${pkgdir}${ROCM_PATH}"
    tar xzf "${latest_tarball}" -C "\${pkgdir}${ROCM_PATH}"
    install -Dm644 /dev/stdin "\${pkgdir}/etc/ld.so.conf.d/rocm-nightly-gfx120X-all.conf" <<CONF
${ROCM_PATH}/lib
${ROCM_PATH}/lib64
CONF
    install -Dm644 /dev/stdin "\${pkgdir}/etc/OpenCL/vendors/amdocl64.icd" <<ICD
${ROCM_PATH}/lib/opencl/libamdocl64.so
ICD
    install -d "\${pkgdir}/usr/share/licenses/\${pkgname}"
    if [[ -f "\${pkgdir}${ROCM_PATH}/share/doc/NOTICES.txt" ]]; then
        ln -s "${ROCM_PATH}/share/doc/NOTICES.txt" \\
            "\${pkgdir}/usr/share/licenses/\${pkgname}/NOTICES.txt"
    fi
}
PKGBUILD_EOF

    if [[ ! -f "${latest_tarball}" ]]; then
        curl -LO "${ROCM_TARBALL_BASE}/${latest_tarball}"
    fi
    makepkg --skipchecksums -si --noconfirm
    rm -f "${latest_tarball}"
    sudo ldconfig
    cd "${basedir_rocm}"
    rm -rf "${rocm_dir}"
}

purge_cuda_pypi() {
    local -a pkgs
    mapfile -t pkgs < <(uv pip freeze | awk -F== 'BEGIN{IGNORECASE=1}
        /^(nvidia-|cuda-|pynvml|onnxruntime-gpu)/ {print $1}')
    if ((${#pkgs[@]})); then
        log "Removing CUDA PyPI leftovers"
        uv pip uninstall -y "${pkgs[@]}" >/dev/null
    fi
}

install_torch_rocm() {
    log "Installing ROCm PyTorch stack"
    uv pip install --upgrade --force-reinstall \
        --extra-index-url "${ROCM_NIGHTLY_INDEX}" \
        torch torchaudio torchvision triton \
        filelock fsspec jinja2 markupsafe mpmath networkx numpy pillow \
        setuptools sympy typing-extensions apex cxxfilt sageattention soundfile
    uv pip install "kornia>=0.7.0,<0.8.0" --no-deps --force-reinstall
    purge_cuda_pypi
}

smoke_rocm() {
    local basedir_rocm=$1
    log "ROCm smoke test"
    # shellcheck source=/dev/null
    source "${basedir_rocm}/venv/bin/activate"
    export ROCM_PATH
    export HIP_PATH="${ROCM_PATH}"
    export HIP_PLATFORM=amd
    export PATH="${ROCM_PATH}/bin:${PATH}"
    export LD_LIBRARY_PATH="${ROCM_PATH}/lib:${basedir_rocm}/venv/lib:${LD_LIBRARY_PATH:-}"
    python - <<'PY'
import torch
print("torch", torch.__version__, "hip", getattr(torch.version, "hip", None))
if not torch.cuda.is_available():
    raise SystemExit("ROCm HIP device not available")
print("device", torch.cuda.get_device_name(0))
PY
    [[ -x "${basedir_rocm}/venv/bin/llama-server" ]] || die "llama-server missing"
    deactivate
}

setup_rocm() {
    local basedir_rocm="${BASEDIR}/rocm"
    stop_service comfy-rocm
    stop_service llama-vulkan
    ensure_shared_dirs
    ensure_llm_preset
    clean_base "${basedir_rocm}"
    setup_venv_and_comfy "${basedir_rocm}"
    create_comfy_rocm_service "${basedir_rocm}"
    create_vulkan_service "${basedir_rocm}"
    install_custom_nodes
    update_pip_and_requirements
    install_rocm_nightly "${basedir_rocm}"
    cd "${basedir_rocm}/comfy"
    install_torch_rocm
    setup_extra_model_paths
    build_llama_vulkan "${basedir_rocm}"
    write_manifest "${basedir_rocm}/.updateai-manifest" \
        "comfy=$(git_rev "${basedir_rocm}/comfy")" \
        "torch=$(python -c 'import torch; print(torch.__version__)')" \
        "rocm=$(pacman -Q rocm-nightly 2>/dev/null || echo missing)"
    deactivate
    smoke_rocm "${basedir_rocm}"
    enable_units comfy-rocm.service llama-vulkan.service
    note "ROCm stack: ${basedir_rocm}
  comfy-rocm.service     http://127.0.0.1:8189/
  llama-vulkan.service   http://127.0.0.1:9932/
Restart: systemctl --user restart comfy-rocm llama-vulkan"
}

# ---------------------------------------------------------------------------
# XPU + SYCL
# ---------------------------------------------------------------------------
create_comfy_xpu_service() {
    local basedir_xpu=$1
    log "Writing comfy-xpu.service"
    cat > "${UNIT_DIR}/comfy-xpu.service" <<EOF
# ${UNIT_DIR}/comfy-xpu.service
[Unit]
Description=ComfyUI XPU
After=network.target

[Service]
WorkingDirectory=${basedir_xpu}/comfy
Type=simple
Restart=always
RestartSec=10
TimeoutStopSec=10
OOMScoreAdjust=-500

Environment=ONEAPI_DEVICE_SELECTOR=level_zero:0
#Environment=SYCL_CACHE_PERSISTENT=1
#Environment=SYCL_CACHE_DIR=${CACHE_DIR}/sycl
Environment=TRITON_CACHE_DIR=${CACHE_DIR}/triton
Environment=TORCHINDUCTOR_CACHE_DIR=${CACHE_DIR}/torchinductor

ExecStart=/bin/bash -c 'source ${basedir_xpu}/venv/bin/activate && \\
    export LD_LIBRARY_PATH=${basedir_xpu}/venv/lib:\${LD_LIBRARY_PATH:-} && \\
    set +u && source ${ONEAPI_SETVARS} && set -u && \\
    python main.py \\
    --input-directory ${BASEDIR}/input \\
    --output-directory ${BASEDIR}/output \\
    --user-directory ${BASEDIR}/user \\
    --port 8188 \\
    --oneapi-device-selector level_zero:0'

[Install]
WantedBy=default.target
EOF
}

create_sycl_service() {
    local basedir_xpu=$1
    log "Writing llama-sycl.service"
    cat > "${UNIT_DIR}/llama-sycl.service" <<EOF
# ${UNIT_DIR}/llama-sycl.service
[Unit]
Description=llama.cpp SYCL
After=network.target

[Service]
WorkingDirectory=${basedir_xpu}
Type=simple
Restart=on-failure
RestartSec=5
TimeoutStopSec=15
OOMScoreAdjust=-500

Environment=ONEAPI_DEVICE_SELECTOR=level_zero:0
#Environment=SYCL_CACHE_PERSISTENT=1
#Environment=SYCL_CACHE_DIR=${CACHE_DIR}/sycl

ExecStart=/bin/bash -c 'set +u && source ${ONEAPI_SETVARS} && set -u && \\
    export LD_LIBRARY_PATH=${basedir_xpu}/venv/bin:\${LD_LIBRARY_PATH:-} && \\
    exec ${basedir_xpu}/venv/bin/llama-server \\
    --host 127.0.0.1 \\
    --port 9931 \\
    --models-max 1 \\
    --tools all \\
    --models-preset ${LLM_PRESET_FILE_32GB}'

[Install]
WantedBy=default.target
EOF
}

install_torch_xpu() {
    log "Installing PyTorch XPU nightly"
    uv pip install --pre --upgrade --force-reinstall \
        --index-url "${PYTORCH_XPU_INDEX}" \
        torch torchvision torchaudio
    uv pip install --force-reinstall 'numpy<2.5'
    purge_cuda_pypi
}

smoke_xpu() {
    local basedir_xpu=$1
    log "XPU smoke test"
    # shellcheck source=/dev/null
    source "${basedir_xpu}/venv/bin/activate"
    source_oneapi
    export ONEAPI_DEVICE_SELECTOR=level_zero:0
    python - <<'PY'
import torch
print("torch", torch.__version__)
print("xpu", torch.xpu.is_available(), "count", torch.xpu.device_count())
if not torch.xpu.is_available():
    raise SystemExit("XPU not available")
print("device", torch.xpu.get_device_name(0))
PY
    [[ -x "${basedir_xpu}/venv/bin/llama-server" ]] || die "llama-server missing"
    deactivate
}

setup_xpu() {
    local basedir_xpu="${BASEDIR}/xpu"
    stop_service comfy-xpu
    stop_service llama-sycl
    ensure_shared_dirs
    ensure_llm_preset
    clean_base "${basedir_xpu}"
    setup_venv_and_comfy "${basedir_xpu}"
    create_comfy_xpu_service "${basedir_xpu}"
    create_sycl_service "${basedir_xpu}"
    install_custom_nodes
    update_pip_and_requirements
    install_torch_xpu
    setup_extra_model_paths
    build_llama_sycl "${basedir_xpu}"
    write_manifest "${basedir_xpu}/.updateai-manifest" \
        "comfy=$(git_rev "${basedir_xpu}/comfy")" \
        "torch=$(python -c 'import torch; print(torch.__version__)')"
    deactivate
    smoke_xpu "${basedir_xpu}"
    enable_units comfy-xpu.service llama-sycl.service
    note "XPU stack: ${basedir_xpu}
  comfy-xpu.service      http://127.0.0.1:8188/
  llama-sycl.service     http://127.0.0.1:9931/
Restart: systemctl --user restart comfy-xpu llama-sycl"
}

# ---------------------------------------------------------------------------
# OneTrainer XPU
# ---------------------------------------------------------------------------
onetrainer_uv() { UV_NO_PROJECT=1 uv "$@"; }

sync_onetrainer_src() {
    local ot=$1
    mkdir -p "$(dirname "${ot}")"
    if [[ -d "${ot}/.git" ]]; then
        log "Updating OneTrainer source"
        git -C "${ot}" fetch --prune origin
        git -C "${ot}" checkout --force master
        git -C "${ot}" reset --hard origin/master
        git -C "${ot}" clean -fd \
            --exclude=venv \
            --exclude=training_concepts \
            --exclude=training_configs \
            --exclude=training_samples \
            --exclude=workspace \
            --exclude=workspace-cache
    else
        log "Cloning OneTrainer → ${ot}"
        rm -rf "${ot}"
        git clone --quiet --no-tags --depth 1 "${ONETRAINER_REPO}" "${ot}"
    fi
    log "Merging PR ${ONETRAINER_PR}"
    git -C "${ot}" fetch origin "pull/${ONETRAINER_PR}/head:pr-${ONETRAINER_PR}"
    git -C "${ot}" merge "pr-${ONETRAINER_PR}" --no-edit || true
    if [[ -f "${ot}/.git/MERGE_HEAD" ]]; then
        if git -C "${ot}" diff --name-only --diff-filter=U | grep -qx 'scripts/train_ui.py'; then
            git -C "${ot}" checkout --ours scripts/train_ui.py
            git -C "${ot}" add scripts/train_ui.py
        fi
        if git -C "${ot}" diff --name-only --diff-filter=U | grep -q .; then
            die "Unresolved OneTrainer merge conflicts remain"
        fi
        git -C "${ot}" add -u
        git -C "${ot}" commit --no-edit
    fi
}

patch_onetrainer_xpu() {
    local ot=$1
    log "Applying XPU patches"
    python - <<PY
from pathlib import Path
import re

root = Path("${ot}")

p = root / "scripts/util/import_util.py"
text = p.read_text()
if "ipex_init" not in text:
    p.write_text(text.rstrip() + """

    import torch

    if hasattr(torch, "xpu") and torch.xpu.is_available():
        from ipex_to_cuda import ipex_init

        ipex_init()
        print("CUDA -> XPU hijacking active")
""")
    print("import_util.py: ipex hook added")
else:
    print("import_util.py: ipex hook already present")

tu = root / "modules/util/torch_util.py"
kept = []
removed = 0
for line in tu.read_text().splitlines(keepends=True):
    if 'print ("true")' in line or 'print("true")' in line:
        removed += 1
        continue
    kept.append(line)
if removed:
    tu.write_text("".join(kept))
    print(f"torch_util.py: removed {removed} debug print(s)")
else:
    print("torch_util.py: no debug print")

zluda = root / "modules/zluda/ZLUDA.py"
ztext = zluda.read_text()
new_fn = '''def is_zluda(device):
    try:
        if isinstance(device, str):
            if device.lower() == "cpu":
                return False
            if device.lower().startswith("cuda"):
                parts = device.split(":")
                device = int(parts[1]) if len(parts) > 1 else 0
        return str(torch.cuda.get_device_name(device)).endswith("[ZLUDA]")
    except Exception:
        return False
'''
ztext2, n = re.subn(
    r"def is_zluda\([\s\S]*?(?=\ndef |\Z)",
    new_fn + "\n",
    ztext,
    count=1,
)
if n != 1:
    raise SystemExit(f"ZLUDA is_zluda replace count={n}")
zluda.write_text(ztext2)
print("ZLUDA.py: is_zluda patched")
PY
    python -m py_compile \
        "${ot}/scripts/util/import_util.py" \
        "${ot}/modules/util/torch_util.py" \
        "${ot}/modules/zluda/ZLUDA.py"
}

install_onetrainer_python() {
    local ot=$1
    log "Creating OneTrainer Python ${PYTHON_VER} venv"
    rm -rf "${ot}/venv"
    onetrainer_uv venv "${ot}/venv" --python "${PYTHON_VER}"
    # shellcheck source=/dev/null
    source "${ot}/venv/bin/activate"
    log "Installing torch XPU stable + ipex_to_cuda"
    onetrainer_uv pip install torch torchvision torchaudio \
        --index-url "${PYTORCH_XPU_STABLE_INDEX}"
    onetrainer_uv pip install "git+https://github.com/Disty0/ipex_to_cuda.git"
    onetrainer_uv pip install \
        "git+https://github.com/huggingface/diffusers.git@${ONETRAINER_DIFFUSERS_REF}" \
        "git+https://github.com/Nerogar/mgds.git@${ONETRAINER_MGDS_REF}" \
        "git+https://github.com/KellerJordan/Muon.git@${ONETRAINER_MUON_REF}"
    grep -vE '^\-e |^numpy==' "${ot}/requirements-global.txt" \
        | onetrainer_uv pip install -r -
    onetrainer_uv pip install onnxruntime
    python - <<'PY'
import torch
print(torch.__version__)
print("xpu", torch.xpu.is_available(), torch.xpu.device_count())
if not torch.xpu.is_available():
    raise SystemExit("XPU not available in OneTrainer venv")
PY
    deactivate
}

setup_onetrainer() {
    local ot="${ONETRAINER_DIR}"
    log "Installing OneTrainer XPU stack → ${ot}"
    ensure_shared_dirs
    sync_onetrainer_src "${ot}"
    patch_onetrainer_xpu "${ot}"
    install_onetrainer_python "${ot}"
    write_manifest "${ot}/.updateai-manifest" \
        "onetrainer=$(git_rev "${ot}")"
    note "OneTrainer XPU: ${ot}
  Launch: source ${ot}/venv/bin/activate && python ${ot}/scripts/train_ui_qt.py
  Device: xpu
  Kept:   ${ot}/training_concepts
          ${ot}/training_configs
          ${ot}/training_samples
          ${ot}/workspace
          ${ot}/workspace-cache"
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
    [[ $# -eq 1 ]] || usage
    local target=$1
    case "${target}" in
        rocm|xpu|onetrainer|all) ;;
        *) usage ;;
    esac

    log "Starting updateai (${target})"
    ensure_uv
    ensure_shared_dirs

    case "${target}" in
        rocm) setup_rocm ;;
        xpu) setup_xpu ;;
        onetrainer) setup_onetrainer ;;
        all)
            setup_rocm
            setup_xpu
            setup_onetrainer
            ;;
    esac

    print_notes
    local elapsed=$((SECONDS - start_time))
    if (( elapsed >= 60 )); then
        printf '\n=== READY (%d minutes %d seconds) ===\n\n' \
            "$((elapsed / 60))" "$((elapsed % 60))"
    else
        printf '\n=== READY (%d seconds) ===\n\n' "${elapsed}"
    fi
}

main "$@"

# 32 GiB VRAM (dedicated AI GPU, single user)
# /home/hans/ai/models/llm_presets/32gb.ini
[*]
flash-attn = on
gpu-layers = all
parallel = 1
cache-type-k = q8_0
cache-type-v = q4_1
batch-size = 4096
ubatch-size = 2048
load-mode = none
spec-type = draft-mtp,ngram-mod
spec-draft-n-max = 3
spec-ngram-mod-n-match = 24
spec-ngram-mod-n-min = 24
spec-ngram-mod-n-max = 32
image-min-tokens = 1024
image-max-tokens = 2048

# KV cache VRAM
# Qwen3.8-27B: 16 attn layers × 4 KV heads × 256 dim × 2 (K+V)
# q8_0/q4_1 : ~0.75 B/elem → 24 KiB/token → GiB ≈ ctx / 43690
# https://datakeskus.io/files/qwen38-27b-kv-cache.txt
#ctx-size = 2048     # 0.05 GiB
#ctx-size = 4096     # 0.09 GiB
#ctx-size = 8192     # 0.19 GiB
#ctx-size = 16384    # 0.38 GiB
#ctx-size = 32768    # 0.75 GiB
#ctx-size = 49152    # 1.13 GiB
#ctx-size = 65536    # 1.50 GiB
#ctx-size = 87040    # 2.00 GiB
#ctx-size = 98304    # 2.25 GiB
#ctx-size = 131072   # 3.00 GiB
#ctx-size = 163840   # 3.75 GiB
#ctx-size = 196608   # 4.50 GiB
#ctx-size = 229376   # 5.25 GiB
#ctx-size = 262144   # 6.00 GiB

# Model VRAM
# Qwen3.8-27B-Uncensored-Q6_K.gguf          20.9 GiB
# mmproj-Qwen3.8-27B-Uncensored-f16.gguf    0.89 GiB
# https://huggingface.co/orcarouter/Qwen3.8-27B-Uncensored-GGUF
m = /home/hans/ai/models/llm/Qwen3.8-27B-Uncensored-Q6_K.gguf

[Instruct]
ctx-size = 65536
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision CPU]
ctx-size = 65536
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision GPU]
ctx-size = 65536
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Thinking]
ctx-size = 65536
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision CPU]
ctx-size = 65536
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision GPU]
ctx-size = 32768
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-Uncensored-f16.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

# 16 GiB VRAM (primary desktop GPU, single user)
# /home/hans/ai/models/llm_presets/16gb.ini
[*]
flash-attn = on
gpu-layers = all
parallel = 1
cache-type-k = q4_0
cache-type-v = q4_0
batch-size = 384
ubatch-size = 384
load-mode = mlock
spec-type = draft-mtp,ngram-mod
spec-draft-n-max = 2
spec-ngram-mod-n-match = 24
spec-ngram-mod-n-min = 24
spec-ngram-mod-n-max = 32
image-min-tokens = 1024
image-max-tokens = 2048

# KV cache VRAM
# Qwen3.8-27B: 16 attn layers × 4 KV heads × 256 dim × 2 (K+V)
# q4_0/q4_0: 0.5 B/elem → 16 KiB/token → GiB ≈ ctx / 65536
# https://datakeskus.io/files/qwen38-27b-kv-cache.txt
#ctx-size = 2048    # 0.03 GiB
#ctx-size = 4096    # 0.06 GiB
#ctx-size = 6144    # 0.09 GiB
#ctx-size = 8192    # 0.13 GiB
#ctx-size = 16384   # 0.25 GiB
#ctx-size = 32768   # 0.50 GiB
#ctx-size = 49152   # 0.75 GiB
#ctx-size = 65536   # 1.00 GiB
#ctx-size = 81920   # 1.25 GiB
#ctx-size = 98304   # 1.50 GiB
#ctx-size = 114688  # 1.75 GiB
#ctx-size = 131072  # 2.00 GiB
#ctx-size = 147456  # 2.25 GiB
#ctx-size = 163840  # 2.50 GiB
#ctx-size = 180224  # 2.75 GiB
#ctx-size = 196608  # 3.00 GiB
#ctx-size = 229376  # 3.50 GiB
#ctx-size = 245760  # 3.75 GiB
#ctx-size = 262144  # 4.00 GiB

# Model VRAM
# Qwen3.8-27B-ZB4.00-MIN-v3-IQ4_XS.gguf     12.7 GiB
# mmproj-Qwen3.8-27B-BF16.gguf              0.89 GiB
# https://huggingface.co/tooltd/Qwen3.8-27B-IQ4-XS-16GB-VRAM-GGUF
# https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF

[Instruct]
ctx-size = 32768
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision CPU]
ctx-size = 32768
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-BF16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Instruct Vision GPU]
ctx-size = 32768
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-BF16.gguf
temperature = 0.7
top-p = 0.8
top-k = 20
min-p = 0.0
presence-penalty = 1.5
repeat-penalty = 1.0
reasoning = off
chat-template-kwargs = {"enable_thinking": false}

[Thinking]
ctx-size = 32768
m = /home/hans/ai/models/llm/Qwen3.8-27B-ZB4.00-MIN-v3-IQ4_XS.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision CPU]
ctx-size = 32768
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-BF16.gguf
# saves VRAM; slower image/PDF/video analysis
no-mmproj-offload = on
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

[Thinking Vision GPU]
ctx-size = 32768
mmproj = /home/hans/ai/models/llm_aux/mmproj-Qwen3.8-27B-BF16.gguf
temperature = 1.0
top-p = 0.95
top-k = 20
min-p = 0.0
presence-penalty = 0.0
repeat-penalty = 1.0
reasoning-preserve = on
reasoning-effort = medium
chat-template-kwargs = {"reasoning_effort": "medium"}

# /home/hans/.config/systemd/user/comfy-rocm.service
[Unit]
Description=ComfyUI ROCm
After=network.target

[Service]
WorkingDirectory=/home/hans/ai/rocm/comfy
Type=simple
Restart=always
RestartSec=10
TimeoutStopSec=10
OOMScoreAdjust=-500

Environment=ROCM_PATH=/opt/rocm
Environment=ROCM_HOME=/opt/rocm
Environment=HIP_PATH=/opt/rocm
Environment=HIP_PLATFORM=amd
Environment=PYTORCH_TUNABLEOP_ENABLED=1
Environment=PYTORCH_TUNABLEOP_TUNING=0
Environment=PYTORCH_TUNABLEOP_FILENAME=/home/hans/ai/cache/tunableop/results.csv
Environment=COMFYUI_ENABLE_MIOPEN=1
Environment=MIOPEN_FIND_MODE=FAST
Environment=PYTORCH_MIOPEN_SUGGEST_NHWC=0
Environment=PYTORCH_HIP_ALLOC_CONF=expandable_segments:True
Environment=TORCH_BLAS_PREFER_HIPBLASLT=1
Environment=HSA_ENABLE_SDMA=0
Environment=FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE
Environment=MIOPEN_USER_DB_PATH=/home/hans/ai/cache/miopen
Environment=MIOPEN_CUSTOM_CACHE_DIR=/home/hans/ai/cache/miopen
Environment=TRITON_CACHE_DIR=/home/hans/ai/cache/triton
Environment=TORCHINDUCTOR_CACHE_DIR=/home/hans/ai/cache/torchinductor

ExecStart=/bin/bash -c 'source /home/hans/ai/rocm/venv/bin/activate && \
    export PATH=/opt/rocm/bin:$PATH && \
    export LD_LIBRARY_PATH=/opt/rocm/lib:/home/hans/ai/rocm/venv/lib:${LD_LIBRARY_PATH:-} && \
    python main.py \
    --input-directory /home/hans/ai/input \
    --output-directory /home/hans/ai/output \
    --user-directory /home/hans/ai/user \
    --port 8189 \
    --disable-cuda-malloc \
    --enable-dynamic-vram \
    --use-pytorch-cross-attention \
    --reserve-vram 2 \
    --fast'

[Install]
WantedBy=default.target

# /home/hans/.config/systemd/user/comfy-xpu.service
[Unit]
Description=ComfyUI XPU
After=network.target

[Service]
WorkingDirectory=/home/hans/ai/xpu/comfy
Type=simple
Restart=always
RestartSec=10
TimeoutStopSec=10
OOMScoreAdjust=-500

Environment=ONEAPI_DEVICE_SELECTOR=level_zero:0
#Environment=SYCL_CACHE_PERSISTENT=1
#Environment=SYCL_CACHE_DIR=/home/hans/ai/cache/sycl
Environment=TRITON_CACHE_DIR=/home/hans/ai/cache/triton
Environment=TORCHINDUCTOR_CACHE_DIR=/home/hans/ai/cache/torchinductor

ExecStart=/bin/bash -c 'source /home/hans/ai/xpu/venv/bin/activate && \
    export LD_LIBRARY_PATH=/home/hans/ai/xpu/venv/lib:${LD_LIBRARY_PATH:-} && \
    set +u && source /opt/intel/oneapi/setvars.sh && set -u && \
    python main.py \
    --input-directory /home/hans/ai/input \
    --output-directory /home/hans/ai/output \
    --user-directory /home/hans/ai/user \
    --port 8188 \
    --oneapi-device-selector level_zero:0'

[Install]
WantedBy=default.target

# /home/hans/.config/systemd/user/llama-sycl.service
[Unit]
Description=llama.cpp SYCL
After=network.target

[Service]
WorkingDirectory=/home/hans/ai/xpu
Type=simple
Restart=on-failure
RestartSec=5
TimeoutStopSec=15
OOMScoreAdjust=-500

Environment=ONEAPI_DEVICE_SELECTOR=level_zero:0
#Environment=SYCL_CACHE_PERSISTENT=1
#Environment=SYCL_CACHE_DIR=/home/hans/ai/cache/sycl

ExecStart=/bin/bash -c 'set +u && source /opt/intel/oneapi/setvars.sh && set -u && \
    export LD_LIBRARY_PATH=/home/hans/ai/xpu/venv/bin:${LD_LIBRARY_PATH:-} && \
    exec /home/hans/ai/xpu/venv/bin/llama-server \
    --host 127.0.0.1 \
    --port 9931 \
    --models-max 1 \
    --tools all \
    --models-preset /home/hans/ai/models/llm_presets/32gb.ini'

[Install]
WantedBy=default.target

# /home/hans/.config/systemd/user/llama-vulkan.service
[Unit]
Description=llama.cpp Vulkan
After=network.target

[Service]
WorkingDirectory=/home/hans/ai/rocm
Type=simple
Restart=on-failure
RestartSec=5
TimeoutStopSec=15
OOMScoreAdjust=-500

Environment=VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/radeon_icd.x86_64.json
Environment=RADV_PERFTEST=transfer_queue

ExecStart=/bin/bash -c 'export LD_LIBRARY_PATH=/home/hans/ai/rocm/venv/bin:${LD_LIBRARY_PATH:-} && \
    exec /home/hans/ai/rocm/venv/bin/llama-server \
    --host 127.0.0.1 \
    --port 9932 \
    --models-max 1 \
    --tools all \
    --models-preset /home/hans/ai/models/llm_presets/16gb.ini'

[Install]
WantedBy=default.target

# /home/hans/.local/share/applications/OneTrainer.desktop
[Desktop Entry]
Comment=
Exec=/usr/bin/bash -c 'cd /home/hans/ai/onetrainer; source venv/bin/activate; python scripts/train_ui_qt.py &>log.txt'
Icon=/home/hans/ai/onetrainer/resources/icons/icon.png
Name=OneTrainer
NoDisplay=false
Path=
PrefersNonDefaultGPU=false
StartupNotify=true
Terminal=false
TerminalOptions=
Type=Application
X-KDE-SubstituteUID=false
X-KDE-Username=