Skip to content
Model Routing End-to-End Smoke

test: refresh model routing end-to-end smoke (#9344) #30

Sign in to view logs

test: refresh model routing end-to-end smoke (#9344)

test: refresh model routing end-to-end smoke (#9344) #30

Workflow file for this run

.github/workflows/test-model-routing.yml at 9bc799a
name: Model Routing End-to-End Smoke
on:
push:
branches: [main]
paths:
- '.github/workflows/test-model-routing.yml'
- 'containers/api-proxy/Dockerfile'
- 'containers/api-proxy/routing-*'
- 'containers/api-proxy/fixtures/routing-contract/**'
- 'containers/api-proxy/management.js'
- 'containers/api-proxy/guards/model-policy-guard.js'
- 'containers/api-proxy/proxy-guards.js'
- 'containers/api-proxy/proxy-request.js'
- 'containers/api-proxy/server.js'
- 'containers/api-proxy/server-factory.js'
- 'containers/api-proxy/startup.js'
- 'containers/api-proxy/providers/copilot.js'
- 'src/routing/**'
- 'src/cli-workflow.ts'
- 'src/compose-generator.ts'
- 'src/commands/resolve-credentials.ts'
- 'src/config-file.ts'
- 'src/config-mapper.ts'
- 'src/schema-validator.ts'
- 'src/services/api-proxy-env-config.ts'
- 'src/services/api-proxy-service-config.ts'
- 'src/services/agent-environment/tool-specific-environment.ts'
- 'src/services/credentials/copilot-credential-env.ts'
- 'src/services/router-service.ts'
- 'tests/integration/model-routing.test.ts'
schedule:
- cron: '0 9 * * 1'
workflow_dispatch:
permissions:
contents: read
copilot-requests: write
env:
AGENT_IMAGE: ghcr.io/github/gh-aw-firewall/agent:0.28.29@sha256:edcf17ae63dd74366bc911a74678b9e264d66ac51c48ec156c80e2619892ebbb
SQUID_IMAGE: ghcr.io/github/gh-aw-firewall/squid:0.28.29@sha256:1d5e169c4df14e87fc826261b94cf4ddaf2f08aca2a5b88701100ec193968193
# Model-routing runs require digest-pinned images; the proxy is built from source and pinned in the job.
API_PROXY_LOCAL_TAG: localhost:5000/awf/api-proxy:routing-smoke
ROUTER_IMAGE: ghcr.io/githubnext/gh-aw-router:latest@sha256:d1612d0eaec3fa8f14c38bbd0a6a0682732fc9f83b7fec94219d3e757a048270
jobs:
model-routing:
name: Exercise live routing, policy, and Copilot inference
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci
- name: Build AWF
run: npm run build
- name: Show Docker versions
run: |
docker version
docker compose version
- name: Prepare smoke images
run: |
set -euo pipefail
docker pull "$AGENT_IMAGE"
docker pull "$SQUID_IMAGE"
docker pull "$ROUTER_IMAGE"
docker run --detach --name awf-routing-registry \
--publish 127.0.0.1:5000:5000 registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for attempt in $(seq 1 30); do
if curl --fail --silent http://127.0.0.1:5000/v2/ >/dev/null; then
break
fi
sleep 1
done
curl --fail --silent http://127.0.0.1:5000/v2/ >/dev/null
# Exercise the API-proxy source under test, not the last released image.
docker build --tag "$API_PROXY_LOCAL_TAG" containers/api-proxy
docker push "$API_PROXY_LOCAL_TAG"
api_proxy_digest="$(docker image inspect --format='{{index .RepoDigests 0}}' "$API_PROXY_LOCAL_TAG" | sed 's/.*@//')"
[[ "$api_proxy_digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "Could not resolve a sha256 digest for the locally built API proxy image" >&2
exit 1
}
printf 'API_PROXY_IMAGE=localhost:5000/awf/api-proxy:routing-smoke@%s\n' "$api_proxy_digest" >> "$GITHUB_ENV"
- name: Run router/API-proxy planning smoke
run: npm run test:integration -- --runInBand model-routing.test.ts
- name: Run a routed Copilot inference through AWF
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
response="$(curl --fail --silent --show-error --max-time 120 \
-H "Content-Type: application/json" \
--data "{\"model\":\"claude-haiku-4.5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly ROUTED_OK.\"}],\"max_tokens\":16,\"stream\":false}" \
"${COPILOT_API_URL}/chat/completions")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content;
assert.equal(typeof text, "string", "Copilot response contained no message text");
assert.match(text, /ROUTED_OK/, "Copilot did not return the expected smoke response");
console.log("ROUTED_COPILOT_INFERENCE=passed model=claude-haiku-4.5");
' "$response"
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Run an effort-routed Copilot inference through AWF
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-effort"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/gpt-5.4-mini
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The agent learns the routed model, effort, and endpoint from /reflect
# and seeds its request from it. The selection is advisory, not enforced.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
request="$(node -e '
const assert = require("node:assert/strict");
const routing = JSON.parse(process.argv[1]).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
assert.equal(s.provider, "copilot");
assert.equal(s.model, "github-copilot/gpt-5.4-mini");
assert.equal(s.wire_model, "gpt-5.4-mini");
assert.equal(typeof s.effort, "string", "the selection carries no reasoning effort");
assert.equal(s.endpoint, "/responses");
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
console.log(JSON.stringify({
model: s.wire_model,
input: "Reply with exactly ROUTED_OK.",
reasoning: { effort: s.effort },
max_output_tokens: 2048,
stream: false,
}));
' "$reflect")"
response="$(curl --fail --silent --show-error --max-time 180 \
-H "Content-Type: application/json" \
--data "$request" \
"${COPILOT_API_URL}/responses")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = (result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(text, /ROUTED_OK/, "Copilot did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_COPILOT_EFFORT_INFERENCE=passed model=gpt-5.4-mini endpoint=/responses");
' "$response"
for effort in none max; do
request="$(node -e '
console.log(JSON.stringify({
model: "gpt-5.4-mini",
input: "Reply with exactly ROUTED_OK.",
reasoning: { effort: process.argv[1] },
max_output_tokens: 2048,
stream: false,
}));
' "$effort")"
response="$(curl --fail --silent --show-error --max-time 180 \
-H "Content-Type: application/json" \
--data "$request" \
"${COPILOT_API_URL}/responses")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = (result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(text, /ROUTED_OK/, "Copilot failed an explicit effort request: " + JSON.stringify(result).slice(0, 500));
console.log("COPILOT_EFFORT_INFERENCE=passed effort=" + process.argv[2]);
' "$response" "$effort"
done
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Route between two Copilot candidates and allow a deviating choice
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-multi"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
- github-copilot/gpt-5.4-mini
- github-copilot/gpt-4o
disallowedModels:
- github-copilot/gpt-4o
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# ROUTING_TARGET=selected sends the routed choice; ROUTING_TARGET=other
# sends the candidate the router did not pick. The selection is advisory,
# so both must succeed as long as model policy permits the model.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
plan="$(node -e '
const assert = require("node:assert/strict");
const reflect = JSON.parse(process.argv[1]);
const routing = reflect.routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const copilot = reflect.endpoints.find(endpoint => endpoint.provider === "copilot");
assert.ok(copilot, "Copilot is missing from /reflect");
const expectedModels = ["claude-haiku-4.5", "gpt-5.4-mini"];
assert.ok(Array.isArray(copilot.models), "Copilot model catalogue is missing");
assert.deepEqual(
[...copilot.models].sort(),
[...expectedModels].sort(),
"Copilot /reflect models should contain only policy-permitted candidates",
);
assert.deepEqual(
copilot.routing_models.map(model => model.model_id).sort(),
[...expectedModels].sort(),
"Copilot routing catalogue should be filtered by model policy",
);
assert.ok(
copilot.model_metadata === null || Array.isArray(copilot.model_metadata),
"Copilot model metadata should be null or an array",
);
if (copilot.model_metadata) {
assert.deepEqual(
copilot.model_metadata.map(model => model.id).sort(),
[...expectedModels].sort(),
"Copilot model metadata should be filtered by model policy",
);
}
const s = routing.selection;
const candidates = {
"github-copilot/claude-haiku-4.5": {
path: "/chat/completions",
body: { model: "claude-haiku-4.5", messages: [{ role: "user", content: "Reply with exactly ROUTED_OK." }], max_tokens: 16, stream: false },
},
"github-copilot/gpt-5.4-mini": {
path: "/responses",
body: { model: "gpt-5.4-mini", input: "Reply with exactly ROUTED_OK.", reasoning: { effort: s.effort ?? "low" }, max_output_tokens: 2048, stream: false },
},
};
assert.ok(Object.hasOwn(candidates, s.model), "router selected a model outside the allow-list: " + s.model);
assert.equal(s.endpoint, candidates[s.model].path, "unexpected endpoint for " + s.model);
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
const model = process.env.ROUTING_TARGET === "other"
? Object.keys(candidates).find(model => model !== s.model)
: s.model;
console.log(JSON.stringify({ model, ...candidates[model] }));
' "$reflect")"
path="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).path)' "$plan")"
body="$(node -e 'process.stdout.write(JSON.stringify(JSON.parse(process.argv[1]).body))' "$plan")"
target="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).model)' "$plan")"
response="$(curl --fail --silent --show-error --max-time 180 \
-H "Content-Type: application/json" --data "$body" "${COPILOT_API_URL}${path}")"
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content ?? (result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(String(text), /ROUTED_OK/, "Copilot did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_MULTI_CANDIDATE_INFERENCE=passed target=" + process.argv[3] + " model=" + process.argv[2]);
' "$response" "$target" "$ROUTING_TARGET"
if [ "${ROUTING_TARGET}" = other ]; then
policy_error="$(mktemp)"
status="$(curl --silent --show-error --max-time 60 -o "$policy_error" -w '%{http_code}' \
-H "Content-Type: application/json" \
--data '{"model":"gpt-4o","messages":[{"role":"user","content":"This request must be blocked by model policy."}],"max_tokens":16,"stream":false}' \
"${COPILOT_API_URL}/chat/completions")"
node -e '
const assert = require("node:assert/strict");
assert.equal(process.argv[1], "403", "model policy did not reject a disallowed model");
const error = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8")).error;
assert.equal(error?.type, "model_policy_violation");
assert.equal(error?.reason, "disallowed");
console.log("MODEL_POLICY=blocked disallowed model without failing routed run");
' "$status" "$policy_error"
rm -f "$policy_error"
fi
SCRIPT
)"
echo "::group::Routed choice is admitted"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" --env ROUTING_TARGET=selected -- bash -euc "$ROUTED_COMMAND"
echo "::endgroup::"
echo "::group::Non-selected candidate is admitted (advisory routing)"
# A deviating request is not a routing failure: the run must exit 0.
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" --env ROUTING_TARGET=other -- bash -euc "$ROUTED_COMMAND"
echo "::endgroup::"
echo "ROUTED_DEVIATION_EXIT=0"
- name: Install the Copilot CLI used by gh-aw
env:
# Keep in step with gh-aw's DefaultCopilotVersion.
COPILOT_CLI_VERSION: 1.0.89
run: |
set -euo pipefail
npm install --global --no-audit --no-fund "@github/copilot@${COPILOT_CLI_VERSION}"
copilot --version
- name: Run the Copilot CLI agent on the routed selection
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
ROUTING_DIR="${RUNNER_TEMP}/model-routing-cli"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- api.githubcopilot.com
apiProxy:
enabled: true
allowedModels:
- github-copilot/claude-haiku-4.5
- github-copilot/gpt-5.4-mini
routing:
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The shape a gh-aw engine step would take: read the routed selection
# from /reflect and hand it to the Copilot CLI. The selection is advisory:
# the CLI (or a sub-agent) may use another policy-permitted model.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
test -n "${COPILOT_API_URL:-}" || { echo "COPILOT_API_URL is missing" >&2; exit 1; }
reflect="$(curl --fail --silent --show-error --max-time 30 "${COPILOT_API_URL}/reflect")"
selection="$(node -e '
const assert = require("node:assert/strict");
const routing = JSON.parse(process.argv[1]).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
const wireApi = s.endpoint === "/responses" ? "responses" : "completions";
console.log(s.wire_model + " " + wireApi + " " + (s.effort ?? ""));
' "$reflect")"
read -r model wire_api effort <<<"$selection"
# AWF derives the wire API from COPILOT_MODEL at launch, before the
# router has chosen, so the routed endpoint must be applied here.
export COPILOT_PROVIDER_WIRE_API="$wire_api"
args=(--model "$model")
if [ -n "${effort:-}" ]; then args+=(--reasoning-effort "$effort"); fi
export HOME="$(mktemp -d)"
output="$(copilot "${args[@]}" \
--prompt "Reply with exactly ROUTED_OK and nothing else." \
--silent --no-color --allow-all-tools --disable-builtin-mcps \
--no-custom-instructions --no-auto-update --no-ask-user)"
printf '%s\n' "$output"
grep -q ROUTED_OK <<<"$output" || { echo "Copilot CLI did not return ROUTED_OK" >&2; exit 1; }
echo "ROUTED_COPILOT_CLI=passed model=${model} effort=${effort:-none}"
SCRIPT
)"
sudo --preserve-env=COPILOT_GITHUB_TOKEN,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" -- bash -euc "$ROUTED_COMMAND"
- name: Show Docker state on failure
if: failure()
run: |
docker ps -a
docker network ls
- name: Stop local image registry
if: always()
run: |
if docker container inspect awf-routing-registry >/dev/null 2>&1; then
docker rm --force awf-routing-registry
fi
native-provider-routing:
name: Exercise live ${{ matrix.provider }} routing
# Paid provider keys: run on the weekly schedule and on demand, not on every push.
if: github.ref == 'refs/heads/main' && github.event_name != 'push'
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- provider: openai
domain: api.openai.com
model: openai/gpt-5.4
wire_model: gpt-5.4
- provider: anthropic
domain: api.anthropic.com
model: anthropic/claude-sonnet-5
wire_model: claude-sonnet-5
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci
- name: Build AWF
run: npm run build
- name: Prepare smoke images
run: |
set -euo pipefail
docker pull "$AGENT_IMAGE"
docker pull "$SQUID_IMAGE"
docker pull "$ROUTER_IMAGE"
docker run --detach --name awf-routing-registry \
--publish 127.0.0.1:5000:5000 registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for attempt in $(seq 1 30); do
if curl --fail --silent http://127.0.0.1:5000/v2/ >/dev/null; then
break
fi
sleep 1
done
curl --fail --silent http://127.0.0.1:5000/v2/ >/dev/null
# Exercise the API-proxy source under test, not the last released image.
docker build --tag "$API_PROXY_LOCAL_TAG" containers/api-proxy
docker push "$API_PROXY_LOCAL_TAG"
api_proxy_digest="$(docker image inspect --format='{{index .RepoDigests 0}}' "$API_PROXY_LOCAL_TAG" | sed 's/.*@//')"
[[ "$api_proxy_digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "Could not resolve a sha256 digest for the locally built API proxy image" >&2
exit 1
}
printf 'API_PROXY_IMAGE=localhost:5000/awf/api-proxy:routing-smoke@%s\n' "$api_proxy_digest" >> "$GITHUB_ENV"
- name: Run a routed ${{ matrix.provider }} inference through AWF
env:
OPENAI_API_KEY: ${{ matrix.provider == 'openai' && secrets.OPENAI_API_KEY || '' }}
ANTHROPIC_API_KEY: ${{ matrix.provider == 'anthropic' && secrets.ANTHROPIC_API_KEY || '' }}
ROUTING_PROVIDER: ${{ matrix.provider }}
ROUTING_DOMAIN: ${{ matrix.domain }}
ROUTING_MODEL: ${{ matrix.model }}
ROUTING_WIRE_MODEL: ${{ matrix.wire_model }}
run: |
set -euo pipefail
case "${ROUTING_PROVIDER}" in
openai) test -n "${OPENAI_API_KEY}" || { echo "OPENAI_API_KEY secret is not available" >&2; exit 1; } ;;
anthropic) test -n "${ANTHROPIC_API_KEY}" || { echo "ANTHROPIC_API_KEY secret is not available" >&2; exit 1; } ;;
esac
ROUTING_DIR="${RUNNER_TEMP}/model-routing-${ROUTING_PROVIDER}"
mkdir -m 700 -p "$ROUTING_DIR"
cat > "${ROUTING_DIR}/conversation.json" <<'JSON'
[{"role":"user","parts":[{"text":"Reply with exactly ROUTED_OK."}]}]
JSON
cat > "${ROUTING_DIR}/awf-config.yml" <<YAML
experimental:
modelRouting: true
network:
allowDomains:
- ${ROUTING_DOMAIN}
apiProxy:
enabled: true
allowedModels:
- ${ROUTING_MODEL}
routing:
provider: ${ROUTING_PROVIDER}
objective:
goal: cost
mode: balanced
task:
conversationFile: ${ROUTING_DIR}/conversation.json
container:
images:
agent: ${AGENT_IMAGE}
apiProxy: ${API_PROXY_IMAGE}
squid: ${SQUID_IMAGE}
router: ${ROUTER_IMAGE}
YAML
# The agent reads the routed selection from /reflect (management port
# 10000) and sends exactly that model, effort, and native endpoint.
ROUTED_COMMAND="$(cat <<'SCRIPT'
set -euo pipefail
reflect="$(curl --fail --silent --show-error --max-time 30 "http://172.30.0.30:10000/reflect")"
plan="$(node -e '
const assert = require("node:assert/strict");
const [reflectJson, provider, model, wireModel] = process.argv.slice(1);
const routing = JSON.parse(reflectJson).routing;
assert.equal(routing?.status, "selected", "/reflect did not report a routed selection: " + JSON.stringify(routing));
const s = routing.selection;
assert.equal(s.provider, provider);
assert.equal(s.model, model);
assert.equal(s.wire_model, wireModel);
console.error("ROUTED_SELECTION=" + JSON.stringify(s));
const prompt = "Reply with exactly ROUTED_OK.";
let base, body;
if (provider === "anthropic") {
assert.equal(s.endpoint, "/v1/messages");
base = process.env.ANTHROPIC_BASE_URL;
body = { model: s.wire_model, max_tokens: 4096, messages: [{ role: "user", content: prompt }] };
if (s.effort !== null) body.output_config = { effort: s.effort };
} else {
base = process.env.OPENAI_BASE_URL;
if (s.effort === null) {
assert.equal(s.endpoint, "/chat/completions");
body = { model: s.wire_model, messages: [{ role: "user", content: prompt }], stream: false };
} else {
assert.equal(s.endpoint, "/responses");
body = { model: s.wire_model, input: prompt, reasoning: { effort: s.effort }, max_output_tokens: 4096, stream: false };
}
}
assert.ok(base, "provider base URL is missing from the agent environment");
const url = new URL(base).origin + (s.endpoint.startsWith("/v1/") ? s.endpoint : "/v1" + s.endpoint);
console.log(JSON.stringify({ url, body }));
' "$reflect" "$ROUTING_PROVIDER" "$ROUTING_MODEL" "$ROUTING_WIRE_MODEL")"
url="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).url)' "$plan")"
body="$(node -e 'process.stdout.write(JSON.stringify(JSON.parse(process.argv[1]).body))' "$plan")"
response="$(curl --fail-with-body --silent --show-error --max-time 180 \
-H "Content-Type: application/json" -H "anthropic-version: 2023-06-01" \
--data "$body" "$url")" || { echo "routed request failed: ${response:-}" >&2; exit 1; }
node -e '
const assert = require("node:assert/strict");
const result = JSON.parse(process.argv[1]);
const text = result.choices?.[0]?.message?.content ??
(Array.isArray(result.content) ? result.content.filter(p => p.type === "text").map(p => p.text).join("") : null) ??
(result.output || [])
.filter(item => item.type === "message")
.flatMap(item => item.content || [])
.filter(part => part.type === "output_text")
.map(part => part.text)
.join("");
assert.match(String(text), /ROUTED_OK/, "provider did not return the expected smoke response: " + JSON.stringify(result).slice(0, 500));
console.log("ROUTED_NATIVE_INFERENCE=passed provider=" + process.argv[2] + " model=" + process.argv[3]);
' "$response" "$ROUTING_PROVIDER" "$ROUTING_MODEL"
SCRIPT
)"
sudo --preserve-env=OPENAI_API_KEY,ANTHROPIC_API_KEY,ROUTING_PROVIDER,ROUTING_MODEL,ROUTING_WIRE_MODEL,GITHUB_WORKSPACE,HOME,PATH \
node dist/cli.js --config "${ROUTING_DIR}/awf-config.yml" \
--env ROUTING_PROVIDER="${ROUTING_PROVIDER}" --env ROUTING_MODEL="${ROUTING_MODEL}" --env ROUTING_WIRE_MODEL="${ROUTING_WIRE_MODEL}" \
-- bash -euc "$ROUTED_COMMAND"
- name: Show Docker state on failure
if: failure()
run: |
docker ps -a
docker network ls
- name: Stop local image registry
if: always()
run: |
if docker container inspect awf-routing-registry >/dev/null 2>&1; then
docker rm --force awf-routing-registry
fi