62 lines
2.3 KiB
PowerShell
62 lines
2.3 KiB
PowerShell
# Run llama-server (llama.cpp) with GGUF.
|
|
#
|
|
# From repo root:
|
|
# powershell -ExecutionPolicy Bypass -File install\assistant\run_llama_server.ps1
|
|
#
|
|
# Binary: llama-server.exe in PATH, or $env:LLAMA_SERVER, or tools\llama-server.exe.
|
|
# WESP: WESP_LLM_BASE_URL=http://127.0.0.1:8080 and WESP_LLM_MODEL = .gguf filename
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
|
|
if (-not $env:WESP_ASSISTANT_DIR -or [string]::IsNullOrWhiteSpace($env:WESP_ASSISTANT_DIR)) {
|
|
$root = Join-Path $repoRoot "data\assistant"
|
|
} else {
|
|
$root = $env:WESP_ASSISTANT_DIR
|
|
}
|
|
$file = if ($env:WESP_LLM_GGUF_FILE) { $env:WESP_LLM_GGUF_FILE } else { "qwen2.5-1.5b-instruct-q5_k_m.gguf" }
|
|
$host_ = if ($env:WESP_LLM_HOST) { $env:WESP_LLM_HOST } else { "127.0.0.1" }
|
|
$port = if ($env:WESP_LLM_PORT) { $env:WESP_LLM_PORT } else { "8080" }
|
|
$gguf = Join-Path (Join-Path $root "models") $file
|
|
|
|
if (-not (Test-Path -LiteralPath $gguf)) {
|
|
Write-Error "Model file missing: $gguf - run install/assistant/download_model.ps1 or copy a .gguf into models/"
|
|
}
|
|
|
|
$bin = $null
|
|
if ($env:LLAMA_SERVER -and (Test-Path -LiteralPath $env:LLAMA_SERVER)) {
|
|
$bin = $env:LLAMA_SERVER
|
|
}
|
|
if (-not $bin) {
|
|
$candidates = @(
|
|
(Join-Path $repoRoot "tools\llama-server.exe"),
|
|
(Join-Path $repoRoot "tools\llama-server\llama-server.exe")
|
|
)
|
|
foreach ($p in $candidates) {
|
|
if (Test-Path -LiteralPath $p) { $bin = $p; break }
|
|
}
|
|
}
|
|
if (-not $bin) {
|
|
$cmd = Get-Command "llama-server" -ErrorAction SilentlyContinue
|
|
if ($cmd) { $bin = $cmd.Source }
|
|
}
|
|
if (-not $bin) {
|
|
Write-Host ""
|
|
Write-Host "llama-server.exe not found. Download llama.cpp Windows x64 build:" -ForegroundColor Yellow
|
|
Write-Host " https://github.com/ggml-org/llama.cpp/releases" -ForegroundColor Cyan
|
|
Write-Host "Place llama-server.exe in:" -ForegroundColor Yellow
|
|
Write-Host " $repoRoot\tools\llama-server.exe" -ForegroundColor Cyan
|
|
Write-Host "Or set: `$env:LLAMA_SERVER = 'C:\path\to\llama-server.exe'" -ForegroundColor Cyan
|
|
Write-Host ""
|
|
exit 1
|
|
}
|
|
|
|
$serverArgs = @(
|
|
"-m", $gguf,
|
|
"--host", $host_,
|
|
"--port", $port,
|
|
"-c", "8192"
|
|
)
|
|
$argLine = $serverArgs -join ' '
|
|
Write-Host ('Starting llama-server: {0} {1}' -f $bin, $argLine) -ForegroundColor Green
|
|
& $bin @serverArgs
|