Skip to content
Released Model Routing Compatibility Smoke

Model Routing End-to-End Smoke #21

Sign in to view logs

Model Routing End-to-End Smoke

Model Routing End-to-End Smoke #21

Workflow file for this run

.github/workflows/test-model-routing.yml at beeea9b
name: Released Model Routing Compatibility Smoke
on:
push:
branches: [main]
paths:
- '.github/workflows/test-model-routing.yml'
- 'containers/api-proxy/routing-*'
- 'containers/api-proxy/fixtures/routing-contract/**'
- 'containers/api-proxy/server.js'
- 'containers/api-proxy/server-factory.js'
- 'containers/api-proxy/startup.js'
- 'containers/api-proxy/providers/copilot.js'
- 'src/routing/**'
- 'src/cli-workflow.ts'
- 'src/compose-generator.ts'
- 'src/commands/resolve-credentials.ts'
- 'src/config-file.ts'
- 'src/config-mapper.ts'
- 'src/schema-validator.ts'
- 'src/services/api-proxy-env-config.ts'
- 'src/services/api-proxy-service-config.ts'
- 'src/services/agent-environment/tool-specific-environment.ts'
- 'src/services/credentials/copilot-credential-env.ts'
- 'src/services/router-service.ts'
- 'tests/integration/model-routing.test.ts'
schedule:
- cron: '0 9 * * 1'
workflow_dispatch:
permissions:
contents: read
copilot-requests: write
env:
AGENT_IMAGE: ghcr.io/github/gh-aw-firewall/agent:0.28.29@sha256:edcf17ae63dd74366bc911a74678b9e264d66ac51c48ec156c80e2619892ebbb
API_PROXY_IMAGE: ghcr.io/github/gh-aw-firewall/api-proxy:0.28.29@sha256:5cc683af8156b39c15bd2370615a85775a8b179bed9f49c490a3068d667dfa2b
SQUID_IMAGE: ghcr.io/github/gh-aw-firewall/squid:0.28.29@sha256:1d5e169c4df14e87fc826261b94cf4ddaf2f08aca2a5b88701100ec193968193
ROUTER_IMAGE: ghcr.io/githubnext/gh-aw-router:latest@sha256:d1612d0eaec3fa8f14c38bbd0a6a0682732fc9f83b7fec94219d3e757a048270
jobs:
model-routing:
name: Exercise live routing and Copilot inference
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci
- name: Build AWF
run: npm run build
- name: Show Docker versions
run: |
docker version
docker compose version
- name: Run router/API-proxy planning smoke
run: npm run test:integration -- --runInBand model-routing.test.ts
- name: Run a routed Copilot inference through AWF
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
response="$(curl --fail --silent --show-error --max-time 120 \
-H "Content-Type: application/json" \
--data "{\"model\":\"claude-haiku-4.5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly ROUTED_OK.\"}],\"max_tokens\":16,\"stream\":false}" \
"${COPILOT_API_URL}/chat/completions")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content;
assert.equal(typeof text, "string", "Copilot response contained no message text");
assert.match(text, /ROUTED_OK/, "Copilot did not return the expected smoke response");
console.log("ROUTED_COPILOT_INFERENCE=passed model=claude-haiku-4.5");
' "$response"
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Run an effort-routed Copilot inference through AWF
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-effort"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/gpt-5.4-mini
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The agent learns the routed model, effort, and endpoint from /reflect
# and must send exactly that request; anything else is rejected with 403.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
request="$(node -e '
const assert = require("node:assert/strict");
const routing = JSON.parse(process.argv[1]).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
assert.equal(s.provider, "copilot");
assert.equal(s.model, "github-copilot/gpt-5.4-mini");
assert.equal(s.wire_model, "gpt-5.4-mini");
assert.equal(typeof s.effort, "string", "the selection carries no reasoning effort");
assert.equal(s.endpoint, "/responses");
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
console.log(JSON.stringify({
model: s.wire_model,
input: "Reply with exactly ROUTED_OK.",
reasoning: { effort: s.effort },
max_output_tokens: 2048,
stream: false,
}));
' "$reflect")"
response="$(curl --fail --silent --show-error --max-time 180 \
-H "Content-Type: application/json" \
--data "$request" \
"${COPILOT_API_URL}/responses")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = (result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(text, /ROUTED_OK/, "Copilot did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_COPILOT_EFFORT_INFERENCE=passed model=gpt-5.4-mini endpoint=/responses");
' "$response"
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Route between two Copilot candidates and enforce the choice
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-multi"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
- github-copilot/gpt-5.4-mini
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# ROUTING_TARGET=selected sends the routed choice; ROUTING_TARGET=other
# sends the candidate the router did not pick, which must be rejected.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
plan="$(node -e '
const assert = require("node:assert/strict");
const routing = JSON.parse(process.argv[1]).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
const candidates = {
"github-copilot/claude-haiku-4.5": {
path: "/chat/completions",
body: { model: "claude-haiku-4.5", messages: [{ role: "user", content: "Reply with exactly ROUTED_OK." }], max_tokens: 16, stream: false },
},
"github-copilot/gpt-5.4-mini": {
path: "/responses",
body: { model: "gpt-5.4-mini", input: "Reply with exactly ROUTED_OK.", reasoning: { effort: s.effort ?? "low" }, max_output_tokens: 2048, stream: false },
},
};
assert.ok(Object.hasOwn(candidates, s.model), "router selected a model outside the allow-list: " + s.model);
assert.equal(s.endpoint, candidates[s.model].path, "unexpected endpoint for " + s.model);
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
const model = process.env.ROUTING_TARGET === "other"
? Object.keys(candidates).find(model => model !== s.model)
: s.model;
console.log(JSON.stringify({ model, ...candidates[model] }));
' "$reflect")"
path="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).path)' "$plan")"
body="$(node -e 'process.stdout.write(JSON.stringify(JSON.parse(process.argv[1]).body))' "$plan")"
target="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).model)' "$plan")"
if [ "${ROUTING_TARGET}" = other ]; then
rejected="$(mktemp)"
status="$(curl --silent --show-error --max-time 60 -o "$rejected" -w '%{http_code}' \
-H "Content-Type: application/json" --data "$body" "${COPILOT_API_URL}${path}")"
node -e '
const assert = require("node:assert/strict");
assert.equal(process.argv[1], "403", "non-selected candidate was not rejected");
const error = JSON.parse(require("node:fs").readFileSync(process.argv[3], "utf8")).error;
assert.equal(error?.code, "model_routing_mismatch");
console.log("ROUTED_ENFORCEMENT=rejected model=" + process.argv[2]);
' "$status" "$target" "$rejected"
rm -f "$rejected"
exit 0
fi
response="$(curl --fail --silent --show-error --max-time 180 \
-H "Content-Type: application/json" --data "$body" "${COPILOT_API_URL}${path}")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content ?? (result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(String(text), /ROUTED_OK/, "Copilot did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_MULTI_CANDIDATE_INFERENCE=passed model=" + process.argv[2]);
' "$response" "$target"
SCRIPT
)"
echo "::group::Routed choice is admitted"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" --env ROUTING_TARGET=selected -- bash -euc "$ROUTED_COMMAND"
echo "::endgroup::"
echo "::group::Non-selected candidate is rejected"
set +e
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" --env ROUTING_TARGET=other -- bash -euc "$ROUTED_COMMAND"
status=$?
set -e
echo "::endgroup::"
# A mismatched request is a terminal routing failure, reported as exit 78.
test "$status" -eq 78 || { echo "expected AWF exit 78 after a routing mismatch, got ${status}" >&2; exit 1; }
echo "ROUTED_MISMATCH_EXIT=78"
- name: Install the Copilot CLI used by gh-aw
env:
# Keep in step with gh-aw's DefaultCopilotVersion.
COPILOT_CLI_VERSION: 1.0.89
run: |
set -euo pipefail
npm install --global --no-audit --no-fund "@github/copilot@${COPILOT_CLI_VERSION}"
copilot --version
- name: Run the Copilot CLI agent on the routed selection
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-cli"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
- github-copilot/gpt-5.4-mini
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The shape a gh-aw engine step would take: read the routed selection
# from /reflect and hand it to the Copilot CLI. Any request the CLI
# makes with another model or effort is rejected and AWF exits 78.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
selection="$(node -e '
const assert = require("node:assert/strict");
const routing = JSON.parse(process.argv[1]).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
console.log(s.wire_model + " " + (s.effort ?? ""));
' "$reflect")"
read -r model effort <<<"$selection"
args=(--model "$model")
if [ -n "${effort:-}" ]; then args+=(--reasoning-effort "$effort"); fi
export HOME="$(mktemp -d)"
output="$(copilot "${args[@]}" \
--prompt "Reply with exactly ROUTED_OK and nothing else." \
--silent --no-color --allow-all-tools --disable-builtin-mcps \
--no-custom-instructions --no-auto-update --no-ask-user)"
printf '%s\n' "$output"
grep -q ROUTED_OK <<<"$output" || { echo "Copilot CLI did not return ROUTED_OK" >&2; exit 1; }
echo "ROUTED_COPILOT_CLI=passed model=${model} effort=${effort:-none}"
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Show Docker state on failure
if: failure()
run: |
docker ps -a
docker network ls
sudo sh -c 'grep -h model_routing /tmp/api-proxy-logs-*/* 2>/dev/null | tail -n 40' || true
native-provider-routing:
name: Exercise live ${{ matrix.provider }} routing
# Paid provider keys: run on the weekly schedule and on demand, not on every push.
if: false
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- provider: openai
domain: api.openai.com
model: openai/gpt-5.4
wire_model: gpt-5.4
- provider: anthropic
domain: api.anthropic.com
model: anthropic/claude-sonnet-5
wire_model: claude-sonnet-5
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci
- name: Build AWF
run: npm run build
- name: Run a routed ${{ matrix.provider }} inference through AWF
env:
OPENAI_API_KEY: ${{ matrix.provider == 'openai' && secrets.OPENAI_API_KEY || '' }}
ANTHROPIC_API_KEY: ${{ matrix.provider == 'anthropic' && secrets.ANTHROPIC_API_KEY || '' }}
ROUTING_PROVIDER: ${{ matrix.provider }}
ROUTING_DOMAIN: ${{ matrix.domain }}
ROUTING_MODEL: ${{ matrix.model }}
ROUTING_WIRE_MODEL: ${{ matrix.wire_model }}
run: |
set -euo pipefail
case "${ROUTING_PROVIDER}" in
openai) test -n "${OPENAI_API_KEY}" || { echo "OPENAI_API_KEY secret is not available" >&2; exit 1; } ;;
anthropic) test -n "${ANTHROPIC_API_KEY}" || { echo "ANTHROPIC_API_KEY secret is not available" >&2; exit 1; } ;;
esac
ROUTING_DIR="${RUNNER_TEMP}/model-routing-${ROUTING_PROVIDER}"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- ${ROUTING_DOMAIN}
apiProxy:
enabled: true
allowedModels:
- ${ROUTING_MODEL}
routing:
provider: ${ROUTING_PROVIDER}
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The agent reads the routed selection from /reflect (management port
# 10000) and sends exactly that model, effort, and native endpoint.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
reflect="$(curl --fail --silent --show-error --max-time 30 "http://172.30.0.30:10000/reflect")"
plan="$(node -e '
const assert = require("node:assert/strict");
const [reflectJson, provider, model, wireModel] = process.argv.slice(1);
const routing = JSON.parse(reflectJson).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
assert.equal(s.provider, provider);
assert.equal(s.model, model);
assert.equal(s.wire_model, wireModel);
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
const prompt = "Reply with exactly ROUTED_OK.";
let base, body;
if (provider === "anthropic") {
assert.equal(s.endpoint, "/v1/messages");
base = process.env.ANTHROPIC_BASE_URL;
body = { model: s.wire_model, max_tokens: 4096, messages: [{ role: "user", content: prompt }] };
if (s.effort !== null) body.output_config = { effort: s.effort };
} else {
base = process.env.OPENAI_BASE_URL;
if (s.effort === null) {
assert.equal(s.endpoint, "/chat/completions");
body = { model: s.wire_model, messages: [{ role: "user", content: prompt }], stream: false };
} else {
assert.equal(s.endpoint, "/responses");
body = { model: s.wire_model, input: prompt, reasoning: { effort: s.effort }, max_output_tokens: 4096, stream: false };
}
}
assert.ok(base, "provider base URL is missing from the agent environment");
const url = new URL(base).origin + (s.endpoint.startsWith("/v1/") ? s.endpoint : "/v1" + s.endpoint);
console.log(JSON.stringify({ url, body }));
' "$reflect" "$ROUTING_PROVIDER" "$ROUTING_MODEL" "$ROUTING_WIRE_MODEL")"
url="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).url)' "$plan")"
body="$(node -e 'process.stdout.write(JSON.stringify(JSON.parse(process.argv[1]).body))' "$plan")"
response="$(curl --fail-with-body --silent --show-error --max-time 180 \
-H "Content-Type: application/json" -H "anthropic-version: 2023-06-01" \
--data "$body" "$url")" || { echo "routed request failed: ${response:-}" >&2; exit 1; }
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content ??
(Array.isArray(result.content) ? result.content.filter(p => p.type === "text").map(p => p.text).join("") : null) ??
(result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(String(text), /ROUTED_OK/, "provider did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_NATIVE_INFERENCE=passed provider=" + process.argv[2] + " model=" + process.argv[3]);
' "$response" "$ROUTING_PROVIDER" "$ROUTING_MODEL"
SCRIPT
)"
sudo --preserve-env=OPENAI_API_KEY,ANTHROPIC_API_KEY,ROUTING_PROVIDER,ROUTING_MODEL,ROUTING_WIRE_MODEL,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" \
--env ROUTING_PROVIDER="${ROUTING_PROVIDER}" --env ROUTING_MODEL="${ROUTING_MODEL}" --env ROUTING_WIRE_MODEL="${ROUTING_WIRE_MODEL}" \
-- bash -euc "$ROUTED_COMMAND"
- name: Show Docker state on failure
if: failure()
run: |
docker ps -a
docker network ls
sudo sh -c 'grep -h model_routing /tmp/api-proxy-logs-*/* 2>/dev/null | tail -n 40' || true