Original Summary

参考kvmem官方发布的[Bonsai 2 prism.3 — Experimental]版本及其说明。 本机配置为7945HX + 4060 mobile，8G显存，32G内存。 具体模型效果不做详评，但在我的应用场景中，已经可用。 实际运行效果，提示词是为了长输出乱写的，然后输出限制10240，被截断了。 5.49.621.948 I slot prompt eval time = 6097.37 ms / 1543 tokens ( 253.06 tokens per second), cache = 0 5.49.621.952 I slot eval time = 252377.37 ms / 10240 tokens ( 40.57 tokens per second) 5.49.621.953 I slot total time = 258474.73 ms / 11783 tokens 参考官方给出的启动脚本、启动参数以及相关说明做了部分修改，加了个chat_template和调整了目录结构。分bat和ps1 bat @echo off setlocal cd /d "%~dp0" rem Launch the PowerShell startup script with fixed parameters powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0start-bonsai-cli.ps1" ^ -Model "Ternary-Bonsai-2-27B-PTQ1_0-MTP-r3-Q4_0.gguf" ^ -Mmproj "mmproj-Qwen3.8-27B-Q5_K-MIX.gguf" ^ -ChatTemplateFile "chat_template.jinja" ^ -Gpu 0 ^ -Context 131072 ^ -Budget 24576 ^ -Reserve 10240 ^ -KvType q8_0 ^ -ReasoningBudget 4096 ^ -Mtp ^ -DraftTokens 1 ^ -DraftKvType f16 endlocal ps1 #Requires -Version 5.1 [CmdletBinding()] param( [Parameter(Mandatory = $true)][string]$Model, [string]$Mmproj, [string]$ChatTemplateFile, [string]$Gpu = '0', [ValidateRange(1, 65535)][int]$Port = 18202, [ValidateRange(128, 262144)][int]$Context = 131072, [ValidateRange(128, 262144)][int]$Budget = 24576, [ValidateRange(128, 262144)][int]$Reserve = 10240, [ValidateRange(1, 4096)][int]$Batch = 128, [ValidateSet('q8_0', 'q5_0', 'q4_0')][string]$KvType = 'q8_0', [ValidateRange(-1, 262144)][int]$ReasoningBudget = 4096, [switch]$Mtp, [ValidateRange(1, 2)][int]$DraftTokens = 1, [ValidateSet('f16', 'q8_0', 'q5_0', 'q4_0')][string]$DraftKvType = 'f16', [switch]$MmprojGpu, [switch]$DryRun ) $ErrorActionPreference = 'Stop' # Script root directory (i.e. ..\) $root = (Resolve-Path -LiteralPath $PSScriptRoot).Path # kvmem and models are relative to the script root $kvmemRoot = Join-Path $root 'kvmem' $modelsRoot = Join-Path $root 'models' $server = Join-Path $kvmemRoot 'bin\llama-kvmem-server.exe' $ui = Join-Path $kvmemRoot 'share\kvmem\ui' # Resolve a relative path: try models dir first, then script root, then CWD function Resolve-AssetPath { param( [string]$Path, [string]$Label = 'Path' ) if ([string]::IsNullOrWhiteSpace($Path)) { throw "$Label is empty." } if ([System.IO.Path]::IsPathRooted($Path)) { return (Resolve-Path -LiteralPath $Path).Path } # Try relative to models directory $candidate = Join-Path $modelsRoot $Path if (Test-Path -LiteralPath $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } # Try relative to script root $candidate = Join-Path $root $Path if (Test-Path -LiteralPath $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } # Fall back to the current working directory return (Resolve-Path -LiteralPath $Path).Path } # Validate fixed paths foreach ($path in @($server, $ui)) { if (-not (Test-Path -LiteralPath $path)) { throw "Missing required path: $path" } } # Resolve model path $Model = Resolve-AssetPath -Path $Model -Label 'Model' if (-not (Test-Path -LiteralPath $Model -PathType Leaf)) { throw "Model not found: $Model" } # Resolve mmproj path (if provided) if ($Mmproj) { $Mmproj = Resolve-AssetPath -Path $Mmproj -Label 'Mmproj' if (-not (Test-Path -LiteralPath $Mmproj -PathType Leaf)) { throw "Vision projector not found: $Mmproj" } } # Resolve chat template path (if provided) if ($ChatTemplateFile) { $ChatTemplateFile = Resolve-AssetPath -Path $ChatTemplateFile -Label 'ChatTemplateFile' if (-not (Test-Path -LiteralPath $ChatTemplateFile -PathType Leaf)) { throw "Chat template file not found: $ChatTemplateFile" } } if (($Budget % 128) -ne 0 -or ($Reserve % 128) -ne 0) { throw 'Budget and Reserve must be multiples of 128.' } if (($Budget + $Reserve) -gt $Context) { throw 'Budget + Reserve must not exceed Context.' } $serverArgs = @( '-m', $Model, '-ngl', '99', '--host', '0.0.0.0', '--port', "$Port", '-c', "$Context", '-b', "$Batch", '--ubatch-size', "$Batch", '-n', "$Reserve", '--kvmem-budget', "$Budget", '--kvmem-gen-reserve', "$Reserve", '--kvmem-block-tokens', '128', '--kv-dtype', $KvType, '--kvmem-query-policy', 'user', '--kvmem-query-replay', 'auto', '--flash-attn', 'on', '--enable-thinking', '--reasoning-budget', "$ReasoningBudget", '--ui-dir', $ui ) # Append chat template if provided if ($ChatTemplateFile) { $serverArgs += @('--chat-template-file', $ChatTemplateFile) } if ($Mtp) { $serverArgs += @('--spec-type', 'draft-mtp') } else { $serverArgs += @('--spec-type', 'none') } if ($Mmproj) { $serverArgs += @('--mmproj', $Mmproj) if ($MmprojGpu) { $serverArgs += '--mmproj-offload' } else { $serverArgs += '--no-mmproj-offload' } } if ($Mtp) { $serverArgs += @( '--spec-draft-n-max', "$DraftTokens", '--spec-kv-dtype', $DraftKvType, '--kvmem-mtp-state', 'snapshots' ) } $env:CUDA_VISIBLE_DEVICES = $Gpu $env:CUDA_DEVICE_ORDER = 'PCI_BUS_ID' $visionPlacement = if ($Mmproj) { if ($MmprojGpu) { 'GPU' } else { 'CPU' } } else { 'off' } $speculation = if ($Mtp) { "MTP$DraftTokens" } else { 'off' } if ($DryRun) { @{ argv = @($server) + $serverArgs environment = @{ CUDA_VISIBLE_DEVICES = $Gpu CUDA_DEVICE_ORDER = 'PCI_BUS_ID' } paths = @{ root = $root kvmemRoot = $kvmemRoot modelsRoot = $modelsRoot server = $server ui = $ui model = $Model mmproj = $Mmproj chatTemplateFile = $ChatTemplateFile } } | ConvertTo-Json -Depth 6 return } Write-Host "Starting Bonsai ($speculation, vision: $visionPlacement) at http://0.0.0.0:$Port/" & $server @serverArgs exit $LASTEXITCODE 1 个帖子 - 1 位参与者 阅读完整话题

  • 情报分类：硬件与数码
  • 分类依据：内容涉及硬件、数码产品或通信卡
  • 信息来源：服务器 / LINUX DO - 最新话题
  • 发布时间：2026/10/2 15:58:41