# Run llama-server (llama.cpp) with GGUF. # # From repo root: # powershell -ExecutionPolicy Bypass -File install\assistant\run_llama_server.ps1 # # Binary: llama-server.exe in PATH, or $env:LLAMA_SERVER, or tools\llama-server.exe. # WESP: WESP_LLM_BASE_URL=http://127.0.0.1:8080 and WESP_LLM_MODEL = .gguf filename $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path if (-not $env:WESP_ASSISTANT_DIR -or [string]::IsNullOrWhiteSpace($env:WESP_ASSISTANT_DIR)) { $root = Join-Path $repoRoot "data\assistant" } else { $root = $env:WESP_ASSISTANT_DIR } $file = if ($env:WESP_LLM_GGUF_FILE) { $env:WESP_LLM_GGUF_FILE } else { "qwen2.5-1.5b-instruct-q5_k_m.gguf" } $host_ = if ($env:WESP_LLM_HOST) { $env:WESP_LLM_HOST } else { "127.0.0.1" } $port = if ($env:WESP_LLM_PORT) { $env:WESP_LLM_PORT } else { "8080" } $gguf = Join-Path (Join-Path $root "models") $file if (-not (Test-Path -LiteralPath $gguf)) { Write-Error "Model file missing: $gguf - run install/assistant/download_model.ps1 or copy a .gguf into models/" } $bin = $null if ($env:LLAMA_SERVER -and (Test-Path -LiteralPath $env:LLAMA_SERVER)) { $bin = $env:LLAMA_SERVER } if (-not $bin) { $candidates = @( (Join-Path $repoRoot "tools\llama-server.exe"), (Join-Path $repoRoot "tools\llama-server\llama-server.exe") ) foreach ($p in $candidates) { if (Test-Path -LiteralPath $p) { $bin = $p; break } } } if (-not $bin) { $cmd = Get-Command "llama-server" -ErrorAction SilentlyContinue if ($cmd) { $bin = $cmd.Source } } if (-not $bin) { Write-Host "" Write-Host "llama-server.exe not found. Download llama.cpp Windows x64 build:" -ForegroundColor Yellow Write-Host " https://github.com/ggml-org/llama.cpp/releases" -ForegroundColor Cyan Write-Host "Place llama-server.exe in:" -ForegroundColor Yellow Write-Host " $repoRoot\tools\llama-server.exe" -ForegroundColor Cyan Write-Host "Or set: `$env:LLAMA_SERVER = 'C:\path\to\llama-server.exe'" -ForegroundColor Cyan Write-Host "" exit 1 } $serverArgs = @( "-m", $gguf, "--host", $host_, "--port", $port, "-c", "8192" ) $argLine = $serverArgs -join ' ' Write-Host ('Starting llama-server: {0} {1}' -f $bin, $argLine) -ForegroundColor Green & $bin @serverArgs