sudo pacman -Syu cmake cuda nodejs npm
paru -Sy python-modelscope

npm config set registry https://mirrors.cloud.tencent.com/npm/
npm config set strict-ssl false

cd
git clone https://github.com/TheTom/llama-cpp-turboquant.git
cd llama-cpp-turboquant

# Updated 2026-06-17: The latest version of llama-cpp-turboquant (9901 35ac80d)
# fixed the performance issue on decoding with a slightly larger GPU memory footprint
# (see https://github.com/TheTom/llama-cpp-turboquant/issues/177).
# The context length may need to be adjusted.
#
# Updated 2026-06-11: The latest version of llama-cpp-turboquant (9450 73eb521)
# has some performance issues on decoding.
# It is recommended to use version 9438 ab11a71, which has slightly improved
# prefilling performance compared to version 9418 2cbfdc6 in the performance test.
# git checkout ab11a71

# Following three lines for compiling Web UI frontend
cd tools/ui
npm i
npm run build

cd ../..
export CUDACXX="/opt/cuda/bin/nvcc"
cmake -B build -DGGML_CUDA=ON -DLLAMA_BUILD_WEBUI=ON && cmake --build build --config Release -j --target llama-server llama-cli

modelscope download --model Tariel/Qwen3.6-27B-4bpw-MTP.gguf --local_dir ~/Qwen3.6

cd ~/llama-cpp-turboquant/build/bin
./llama-server -m ~/Qwen3.6/Qwen3.6-27B-4bpw-MTP.gguf \
    -mm ~/Qwen3.6/mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf \
    -ngl all -fa on --spec-type draft-mtp --spec-draft-n-max 3 -np 1 -kvu \
    --ctx-checkpoints 64 --threads-http 2 \
    -ctk q8_0 -ctv turbo3 \
    -c 190208 \
    --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0 \
    --host 0.0.0.0 --port 22345 \
    --api-key "sk-jintianshifengkuangxingqisivivo50" \
    -a qwen3.6-27b \
    --jinja --chat-template-file ~/Qwen3.6/chat_template.jinja \
    --reasoning on --reasoning-format deepseek

sudo ufw allow 22345

paru -Sy opencode-bin llama-swap-bin rtk
rtk init -g --opencode	# Initialize rtk for opencode

captureBuffer: 20
performance:
  every: 10s
startPort: 60001
sendLoadingState: true

macros:
  "llama-turboquant": >
    /home/user/llama-cpp-turboquant-bin/llama-server --port ${PORT}

  "qwen_dir": "/home/user/Qwen3.6"

apiKeys:
  - "sk-jintianshifengkuangxingqisivivo50"

models:
  "qwen3.6-27b":
    cmd: |
      ${llama-turboquant}
      -m ${qwen_dir}/Qwen3.6-27B-4bpw-MTP.gguf
      -mm ${qwen_dir}/mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf
      -ngl all -fa on --spec-type draft-mtp --spec-draft-n-max 3 -np 1 -kvu
      --ctx-checkpoints 64 --threads-http 2
      -ctk q8_0 -ctv turbo3
      -c 190208
      --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0
      --jinja --chat-template-file ${qwen_dir}/chat_template.jinja
      --reasoning on --reasoning-format deepseek

    name: "Qwen 3.6 27B"
    ttl: 0
    filters:
      setParamsByID:
        "${MODEL_ID}:thinking":
          chat_template_kwargs:
            enable_thinking: true
        "${MODEL_ID}:instruct":
          chat_template_kwargs:
            enable_thinking: false
    concurrencyLimit: 2
    timeouts:
      connect: 30
      keepalive: 30
      responseHeader: 60
      tlsHandshake: 10
      idleConn: 90

hooks:
  on_startup:
    preload:
      - "qwen3.6-27b"

{
  "$schema": "https://opencode.ai/config.json",
  "model": "llama.cpp/qwen3.6-27b:thinking",
  "small_model": "llama.cpp/qwen3.6-27b:instruct",
  "provider": {
    "llama.cpp": {
      "npm": "@ai-sdk/openai-compatible",
      "name": "llama.cpp (Local)",
      "options": {
        "baseURL": "http://localhost:22345/v1",
        "apiKey": "sk-jintianshifengkuangxingqisivivo50"
      },
      "models": {
        "qwen3.6-27b:thinking": {
          "name": "Qwen3.6 27B (thinking mode)",
          "modalities": {
            "input": ["text", "image"],
            "output": ["text"]
          },
          "limit": {
            "context": 190208,
            "output": 65536
          },
          "options": {
            "max_tokens": 65536
          }
        },
        "qwen3.6-27b:instruct": {
          "name": "Qwen3.6 27B (non-thinking mode)",
          "modalities": {
            "input": ["text", "image"],
            "output": ["text"]
          },
          "limit": {
            "context": 190208,
            "output": 65536
          },
          "options": {
            "max_tokens": 65536
          }
        }
      }
    }
  }
}

llama-swap --config llama-swap-config.yaml --listen 0.0.0.0:22345