From 1011e074e58a0a20b4d37e8ddbc46e962512a8a7 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Sat, 26 Sep 2026 13:37:00 -0400 Subject: [PATCH 01/15] Rename the NFS mount root to /nfs_profiles create-user.sh mounts the profile share on a transient root, creates the user's directory there, bind-mounts it onto /home/ and unmounts the root again. The root had a deployment-specific name, /awipsprofiles. It is now the neutral /nfs_profiles, and the log line reads the path from NFS_MOUNT_ROOT instead of repeating it. Nothing else depends on the path: the release agent and the heartbeat key on /home/, and session-control.sh mounts under /run/linuxbroker. The share's contents are unchanged, so homes created before the rename keep working. Hosts keep the old, empty mount root until install-host-config.sh, added later in this phase, removes it. The tests use the new root and now also check the mount command and the log line. The legacy fixture, which proves old hosts reject the --password-stdin form, uses the new root too; its header now says so instead of calling itself a verbatim copy, and a missing line break in it is fixed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- linux_host/create-user.sh | 4 ++-- linux_host/tests/common.sh | 2 +- linux_host/tests/fixtures/create-user.legacy.sh | 10 ++++++---- linux_host/tests/test_create_user.sh | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/linux_host/create-user.sh b/linux_host/create-user.sh index 6b1db3d..98fc49c 100644 --- a/linux_host/create-user.sh +++ b/linux_host/create-user.sh @@ -8,7 +8,7 @@ LINUXBROKER_AGENT_VERSION="1.2.0" # Constants -NFS_MOUNT_ROOT="/awipsprofiles" +NFS_MOUNT_ROOT="/nfs_profiles" NFS_OPTIONS="vers=4,minorversion=1,sec=sys,nconnect=4" LOGFILE=/var/log/createuser.log LEASE_DIRECTORY="/var/lib/linuxbroker-release-session/leases" @@ -150,7 +150,7 @@ if [ ! -d "$NFS_MOUNT_ROOT" ]; then fi # Mount NFS root if not already mounted -log "Mount NFS root on /awipsprofiles" +log "Mount NFS root on $NFS_MOUNT_ROOT" if ! mountpoint -q "$NFS_MOUNT_ROOT"; then if [ "$PASSWORD_MODE" = "true" ]; then run_checked "Failed to mount NFS share: $NFS_SHARE" mount -t nfs "$NFS_SHARE" "$NFS_MOUNT_ROOT" -o "$NFS_OPTIONS" diff --git a/linux_host/tests/common.sh b/linux_host/tests/common.sh index de2a94f..08421de 100644 --- a/linux_host/tests/common.sh +++ b/linux_host/tests/common.sh @@ -24,7 +24,7 @@ cleanup_user() { if id "$username" >/dev/null 2>&1; then userdel -r "$username" >/dev/null 2>&1 || userdel "$username" >/dev/null 2>&1 || true fi - rm -rf "/home/$username" "/awipsprofiles/$username" "/var/lib/linuxbroker-release-session/leases/$username.lease" + rm -rf "/home/$username" "/nfs_profiles/$username" "/var/lib/linuxbroker-release-session/leases/$username.lease" } install_basic_shims() { diff --git a/linux_host/tests/fixtures/create-user.legacy.sh b/linux_host/tests/fixtures/create-user.legacy.sh index 267c372..b8b32fa 100644 --- a/linux_host/tests/fixtures/create-user.legacy.sh +++ b/linux_host/tests/fixtures/create-user.legacy.sh @@ -1,6 +1,8 @@ #!/bin/bash -# Verbatim copy of linux_host/create-user.sh before --password-stdin support. -# CI checks out with depth 1, so tests keep this fixture to prove old hosts reject the new form.# Usage: ./create-user.sh [LEASE_ID] +# Copy of linux_host/create-user.sh before --password-stdin support; only the NFS mount root has +# since been renamed. CI checks out with depth 1, so tests keep this fixture to prove old hosts +# reject the new form. +# Usage: ./create-user.sh [LEASE_ID] if [ $# -lt 3 ] || [ $# -gt 4 ]; then echo "Usage: $0 [LEASE_ID]" @@ -13,7 +15,7 @@ USERNAME="$3" LEASE_ID="${4:-}" # Constants -NFS_MOUNT_ROOT="/awipsprofiles" +NFS_MOUNT_ROOT="/nfs_profiles" NFS_OPTIONS="vers=4,minorversion=1,sec=sys,nconnect=4" NFS_USERHOME="$NFS_MOUNT_ROOT/$USERNAME" LOCAL_USERHOME="/home/$USERNAME" @@ -30,7 +32,7 @@ if [ ! -d "$NFS_MOUNT_ROOT" ]; then fi # Mount NFS root if not already mounted -echo "Mount NFS root on /awipsprofiles" >> $LOGFILE +echo "Mount NFS root on /nfs_profiles" >> $LOGFILE if ! mountpoint -q "$NFS_MOUNT_ROOT"; then mount -t nfs "$NFS_SHARE" "$NFS_MOUNT_ROOT" -o "$NFS_OPTIONS" if [ $? -ne 0 ]; then diff --git a/linux_host/tests/test_create_user.sh b/linux_host/tests/test_create_user.sh index bbf777d..c7cffc1 100644 --- a/linux_host/tests/test_create_user.sh +++ b/linux_host/tests/test_create_user.sh @@ -12,7 +12,7 @@ setup_case() { install_basic_shims export FAKE_CALLS="$WORK_DIR/calls.log" : > "$FAKE_CALLS" - mkdir -p /awipsprofiles /var/lib/linuxbroker-release-session/leases + mkdir -p /nfs_profiles /var/lib/linuxbroker-release-session/leases rm -f /var/log/createuser.log } @@ -36,7 +36,9 @@ new_form_success() { assert_eq "$(cat "$lease_file")" "$LEASE" assert_eq "$(stat -c %a "$lease_file")" "600" assert_not_contains_file /var/log/createuser.log 'S3cret!pass' - assert_file_contains "$FAKE_CALLS" "umount /awipsprofiles" + assert_file_contains "$FAKE_CALLS" "mount -t nfs nfs.example:/profiles /nfs_profiles" + assert_file_contains "$FAKE_CALLS" "umount /nfs_profiles" + assert_file_contains /var/log/createuser.log "Mount NFS root on /nfs_profiles" cleanup_user "$user" } From 3799ae27dd47695379d34cd8a92ddca6559b385d Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Sat, 26 Sep 2026 14:12:16 -0400 Subject: [PATCH 02/15] Add start on demand and scale to zero to the database When a checkout finds no ready host, dbo.ReserveVmForStart picks a stopped host to start for the waiting user under the scaling lock, or says why not and when to ask again. Checkout events gain a Starting outcome and the AVD host script's ClientVersion, and dbo.fnWaitingCheckoutUsers lists who is waiting. The scaling policy gains StartOnDemandEnabled (on by default) and MaxPendingStarts, and reports the AVD hosts whose script cannot wait for a host to start. Rules and windows may keep a minimum of 0 while start on demand is on; scaling reads 0 as 1 while it is off. Scaling counts waiting users as demand and keeps a host for each when scaling down. The dashboard procedures measure waits (served, median, 95th percentile, waiting now), leave Starting out of checkout totals and scaling runs, and stop reporting no ready hosts for an idle pool scaled to zero. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ..._add_start_on_demand_to_scaling_policy.sql | 34 ++ ...dd_starting_outcome_to_checkout_events.sql | 36 ++ ...46_alter_procedure-RecordCheckoutEvent.sql | 29 ++ sql_queries/147_allow_zero_minimum_hosts.sql | 47 ++ ...create_function-fnWaitingCheckoutUsers.sql | 21 + .../149_alter_procedure-GetScalingPolicy.sql | 60 +++ ...rocedure-SetScalingPolicyStartOnDemand.sql | 63 +++ ...51_alter_procedure-SaveScalingSchedule.sql | 151 ++++++ ...152_create_procedure-ReserveVmForStart.sql | 217 +++++++++ ...53_alter_procedure-TriggerScalingLogic.sql | 320 +++++++++++++ .../154_alter_procedure-GetCheckoutStats.sql | 96 ++++ ...5_alter_procedure-GetUtilizationSeries.sql | 100 ++++ .../156_alter_procedure-GetAttentionItems.sql | 128 +++++ sql_queries/README.md | 21 +- sql_queries/tests/conftest.py | 2 +- .../tests/test_scaling_policy_contract.py | 3 +- sql_queries/tests/test_sql_contract.py | 2 +- .../tests/test_start_on_demand_contract.py | 446 ++++++++++++++++++ 18 files changed, 1771 insertions(+), 5 deletions(-) create mode 100644 sql_queries/144_add_start_on_demand_to_scaling_policy.sql create mode 100644 sql_queries/145_add_starting_outcome_to_checkout_events.sql create mode 100644 sql_queries/146_alter_procedure-RecordCheckoutEvent.sql create mode 100644 sql_queries/147_allow_zero_minimum_hosts.sql create mode 100644 sql_queries/148_create_function-fnWaitingCheckoutUsers.sql create mode 100644 sql_queries/149_alter_procedure-GetScalingPolicy.sql create mode 100644 sql_queries/150_create_procedure-SetScalingPolicyStartOnDemand.sql create mode 100644 sql_queries/151_alter_procedure-SaveScalingSchedule.sql create mode 100644 sql_queries/152_create_procedure-ReserveVmForStart.sql create mode 100644 sql_queries/153_alter_procedure-TriggerScalingLogic.sql create mode 100644 sql_queries/154_alter_procedure-GetCheckoutStats.sql create mode 100644 sql_queries/155_alter_procedure-GetUtilizationSeries.sql create mode 100644 sql_queries/156_alter_procedure-GetAttentionItems.sql create mode 100644 sql_queries/tests/test_start_on_demand_contract.py diff --git a/sql_queries/144_add_start_on_demand_to_scaling_policy.sql b/sql_queries/144_add_start_on_demand_to_scaling_policy.sql new file mode 100644 index 0000000..5fd25dd --- /dev/null +++ b/sql_queries/144_add_start_on_demand_to_scaling_policy.sql @@ -0,0 +1,34 @@ +-- Start on demand (4.1). When a checkout finds no ready host, the broker may start a stopped +-- one for the user, whose AVD host waits for it instead of being turned away. +-- StartOnDemandEnabled On unless an administrator turns it off. While it is off, a checkout +-- that finds no host is refused as before, and a scaling minimum of 0 +-- is read as 1. +-- MaxPendingStarts At most this many hosts may be starting for waiting users at once +-- (1-20). A user who arrives while that many are starting waits for +-- one of them. + +IF COL_LENGTH('dbo.ScalingPolicy', 'StartOnDemandEnabled') IS NULL +BEGIN + ALTER TABLE dbo.ScalingPolicy + ADD StartOnDemandEnabled BIT NOT NULL + CONSTRAINT DF_ScalingPolicy_StartOnDemandEnabled DEFAULT (1); +END; +GO + +IF COL_LENGTH('dbo.ScalingPolicy', 'MaxPendingStarts') IS NULL +BEGIN + ALTER TABLE dbo.ScalingPolicy + ADD MaxPendingStarts INT NOT NULL + CONSTRAINT DF_ScalingPolicy_MaxPendingStarts DEFAULT (2); +END; +GO + +IF NOT EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_ScalingPolicy_MaxPendingStarts' AND parent_object_id = OBJECT_ID('dbo.ScalingPolicy') +) +BEGIN + ALTER TABLE dbo.ScalingPolicy WITH CHECK + ADD CONSTRAINT CK_ScalingPolicy_MaxPendingStarts CHECK (MaxPendingStarts BETWEEN 1 AND 20); +END; +GO diff --git a/sql_queries/145_add_starting_outcome_to_checkout_events.sql b/sql_queries/145_add_starting_outcome_to_checkout_events.sql new file mode 100644 index 0000000..e78de4d --- /dev/null +++ b/sql_queries/145_add_starting_outcome_to_checkout_events.sql @@ -0,0 +1,36 @@ +-- Checkout events gain the Starting outcome (4.1): no host was ready, so the broker started one +-- for the user, or one was already starting, and told their AVD host to ask again shortly. A +-- user who waits records one Starting event each time their AVD host asks, then Assigned once +-- a host takes them. +-- +-- ClientVersion is the version the AVD host's broker script reports with each checkout. A +-- script that reports one waits for a host to start; an older one gives up after three quick +-- attempts, so the scaling policy page lists the AVD hosts still running one. + +IF COL_LENGTH('dbo.CheckoutEvents', 'ClientVersion') IS NULL +BEGIN + ALTER TABLE dbo.CheckoutEvents ADD ClientVersion VARCHAR(32) NULL; +END; +GO + +IF EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_CheckoutEvents_Outcome' + AND parent_object_id = OBJECT_ID('dbo.CheckoutEvents') + AND definition NOT LIKE N'%''Starting''%' +) +BEGIN + ALTER TABLE dbo.CheckoutEvents DROP CONSTRAINT CK_CheckoutEvents_Outcome; +END; +GO + +IF NOT EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_CheckoutEvents_Outcome' AND parent_object_id = OBJECT_ID('dbo.CheckoutEvents') +) +BEGIN + ALTER TABLE dbo.CheckoutEvents WITH CHECK + ADD CONSTRAINT CK_CheckoutEvents_Outcome + CHECK (Outcome IN ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error', 'Starting')); +END; +GO diff --git a/sql_queries/146_alter_procedure-RecordCheckoutEvent.sql b/sql_queries/146_alter_procedure-RecordCheckoutEvent.sql new file mode 100644 index 0000000..8f7ed39 --- /dev/null +++ b/sql_queries/146_alter_procedure-RecordCheckoutEvent.sql @@ -0,0 +1,29 @@ +-- Redefines dbo.RecordCheckoutEvent to accept the Starting outcome and the AVD host script's +-- ClientVersion (145). An outcome it does not know is still stored as Error rather than failing +-- the checkout that reports it. + +CREATE PROCEDURE [dbo].[RecordCheckoutEvent] + @Username VARCHAR(255) = NULL, + @AvdHost VARCHAR(255) = NULL, + @Outcome VARCHAR(24), + @DurationMs INT = NULL, + @Hostname VARCHAR(255) = NULL, + @ClientVersion VARCHAR(32) = NULL +AS +BEGIN + SET NOCOUNT ON; + + INSERT INTO dbo.CheckoutEvents (Username, AvdHost, Outcome, DurationMs, Hostname, ClientVersion) + VALUES ( + LEFT(@Username, 255), + LEFT(@AvdHost, 255), + CASE WHEN @Outcome IN ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error', 'Starting') + THEN @Outcome ELSE 'Error' END, + CASE WHEN @DurationMs < 0 THEN 0 ELSE @DurationMs END, + LEFT(@Hostname, 255), + NULLIF(LEFT(@ClientVersion, 32), '') + ); + + SELECT CAST(SCOPE_IDENTITY() AS BIGINT) AS EventID; +END +GO diff --git a/sql_queries/147_allow_zero_minimum_hosts.sql b/sql_queries/147_allow_zero_minimum_hosts.sql new file mode 100644 index 0000000..535d414 --- /dev/null +++ b/sql_queries/147_allow_zero_minimum_hosts.sql @@ -0,0 +1,47 @@ +-- A pool may scale to zero while start on demand is on (4.1): the first user to arrive waits +-- for a host to start instead of being refused. The scaling rule and schedule windows accept a +-- minimum of 0; the API and dbo.SaveScalingSchedule allow it only while start on demand is on, +-- and scaling reads 0 as 1 whenever start on demand is off. A window's maximum must still be +-- above its minimum, so it is at least 1; the API checks the same for the rule. + +IF EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_VmScalingRules_MinVMs' + AND parent_object_id = OBJECT_ID('dbo.VmScalingRules') + AND definition LIKE N'%>=(1)%' +) +BEGIN + ALTER TABLE dbo.VmScalingRules DROP CONSTRAINT CK_VmScalingRules_MinVMs; +END; +GO + +IF NOT EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_VmScalingRules_MinVMs' AND parent_object_id = OBJECT_ID('dbo.VmScalingRules') +) +BEGIN + ALTER TABLE dbo.VmScalingRules WITH NOCHECK + ADD CONSTRAINT CK_VmScalingRules_MinVMs CHECK (MinVMs >= 0); +END; +GO + +IF EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_ScalingSchedules_MinMax' + AND parent_object_id = OBJECT_ID('dbo.ScalingSchedules') + AND definition LIKE N'%>=(1)%' +) +BEGIN + ALTER TABLE dbo.ScalingSchedules DROP CONSTRAINT CK_ScalingSchedules_MinMax; +END; +GO + +IF NOT EXISTS ( + SELECT 1 FROM sys.check_constraints + WHERE name = 'CK_ScalingSchedules_MinMax' AND parent_object_id = OBJECT_ID('dbo.ScalingSchedules') +) +BEGIN + ALTER TABLE dbo.ScalingSchedules WITH CHECK + ADD CONSTRAINT CK_ScalingSchedules_MinMax CHECK (MinVMs >= 0 AND MaxVMs > MinVMs); +END; +GO diff --git a/sql_queries/148_create_function-fnWaitingCheckoutUsers.sql b/sql_queries/148_create_function-fnWaitingCheckoutUsers.sql new file mode 100644 index 0000000..b857a86 --- /dev/null +++ b/sql_queries/148_create_function-fnWaitingCheckoutUsers.sql @@ -0,0 +1,21 @@ +-- The users waiting for a host to start (4.1): each user whose latest checkout event in the last +-- five minutes is Starting. An AVD host asks again at least every two minutes while its user +-- waits, and a checkout that ends in any other way records that outcome, so a user who gave up +-- stops counting within five minutes. Scaling, start on demand, the dashboard and the fleet +-- snapshot all count waiting users with this. + +CREATE OR ALTER FUNCTION dbo.fnWaitingCheckoutUsers () +RETURNS TABLE +AS +RETURN + SELECT latest.Username, latest.OccurredAt AS LastAskedAt + FROM ( + SELECT e.Username, e.Outcome, e.OccurredAt, + ROW_NUMBER() OVER (PARTITION BY e.Username ORDER BY e.OccurredAt DESC, e.EventID DESC) AS Recency + FROM dbo.CheckoutEvents e + WHERE e.Username IS NOT NULL + AND e.OccurredAt >= DATEADD(MINUTE, -5, SYSUTCDATETIME()) + ) latest + WHERE latest.Recency = 1 + AND latest.Outcome = 'Starting'; +GO diff --git a/sql_queries/149_alter_procedure-GetScalingPolicy.sql b/sql_queries/149_alter_procedure-GetScalingPolicy.sql new file mode 100644 index 0000000..03d3466 --- /dev/null +++ b/sql_queries/149_alter_procedure-GetScalingPolicy.sql @@ -0,0 +1,60 @@ +-- Redefines dbo.GetScalingPolicy (105) to add start on demand (4.1): +-- StartOnDemandEnabled, MaxPendingStarts The policy's start on demand settings (144). +-- ZeroMinimumCount How many scaling rules and enabled schedule windows have a minimum of 0. +-- While start on demand is off, scaling keeps one host running for them. +-- AvdHostsSeen The AVD hosts that asked for a checkout in the last seven days. +-- AvdHostsOutdated Those whose latest checkout reported no ClientVersion: their broker script +-- predates start on demand and gives up instead of waiting for a host. +-- OutdatedAvdHostsJson The first 20 of them by name, as [{"AvdHost": ...}]. +-- AvdClientVersionsJson How many of the others report each version, as +-- [{"ClientVersion": ..., "AvdHosts": n}]. + +CREATE PROCEDURE [dbo].[GetScalingPolicy] +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @NowUtc DATETIME2(0) = CAST(SYSUTCDATETIME() AS DATETIME2(0)); + DECLARE @AvdHosts TABLE (AvdHost VARCHAR(255) NOT NULL PRIMARY KEY, ClientVersion VARCHAR(32) NULL); + + INSERT INTO @AvdHosts (AvdHost, ClientVersion) + SELECT AvdHost, ClientVersion + FROM ( + SELECT AvdHost, ClientVersion, + ROW_NUMBER() OVER (PARTITION BY AvdHost ORDER BY OccurredAt DESC, EventID DESC) AS Position + FROM dbo.CheckoutEvents + WHERE OccurredAt >= DATEADD(DAY, -7, @NowUtc) AND AvdHost IS NOT NULL + ) latest + WHERE Position = 1; + + SELECT + p.TimeZone, + p.UpdatedBy, + CONVERT(VARCHAR(33), p.UpdatedAt, 126) + 'Z' AS UpdatedAtUtc, + CONVERT(VARCHAR(33), @NowUtc, 126) + 'Z' AS NowUtc, + CONVERT(VARCHAR(19), CAST(TODATETIMEOFFSET(@NowUtc, 0) AT TIME ZONE p.TimeZone AS DATETIME2(0)), 126) AS LocalTime, + phase.Source AS ActiveSource, + phase.ScheduleID AS ActiveScheduleID, + phase.PhaseName AS ActivePhaseName, + phase.MinVMs AS ActiveMinVMs, + phase.MaxVMs AS ActiveMaxVMs, + phase.ScaleUpRatio AS ActiveScaleUpRatio, + phase.ScaleUpIncrement AS ActiveScaleUpIncrement, + phase.ScaleDownRatio AS ActiveScaleDownRatio, + phase.ScaleDownIncrement AS ActiveScaleDownIncrement, + phase.StopMode AS ActiveStopMode, + phase.RuleID AS DefaultRuleID, + p.StartOnDemandEnabled, + p.MaxPendingStarts, + (SELECT COUNT(*) FROM dbo.VmScalingRules WHERE MinVMs = 0) + + (SELECT COUNT(*) FROM dbo.ScalingSchedules WHERE Enabled = 1 AND MinVMs = 0) AS ZeroMinimumCount, + (SELECT COUNT(*) FROM @AvdHosts) AS AvdHostsSeen, + (SELECT COUNT(*) FROM @AvdHosts WHERE ClientVersion IS NULL) AS AvdHostsOutdated, + (SELECT TOP (20) AvdHost FROM @AvdHosts WHERE ClientVersion IS NULL ORDER BY AvdHost FOR JSON PATH) AS OutdatedAvdHostsJson, + (SELECT ClientVersion, COUNT(*) AS AvdHosts FROM @AvdHosts WHERE ClientVersion IS NOT NULL + GROUP BY ClientVersion ORDER BY ClientVersion FOR JSON PATH) AS AvdClientVersionsJson + FROM dbo.ScalingPolicy p + OUTER APPLY dbo.fnActiveScalingPhase(@NowUtc) phase + WHERE p.PolicyID = 1; +END +GO diff --git a/sql_queries/150_create_procedure-SetScalingPolicyStartOnDemand.sql b/sql_queries/150_create_procedure-SetScalingPolicyStartOnDemand.sql new file mode 100644 index 0000000..c013e48 --- /dev/null +++ b/sql_queries/150_create_procedure-SetScalingPolicyStartOnDemand.sql @@ -0,0 +1,63 @@ +-- Turns start on demand on or off and sets how many hosts may start for waiting users at once +-- (4.1). A NULL parameter keeps its current value. Turning start on demand off is allowed while +-- rules or windows have a minimum of 0: scaling then keeps one host running for them, and +-- ZeroMinimumCount lets the caller say so. +-- +-- Result: Updated, Unchanged, or Invalid (with a Message naming the field). + +CREATE PROCEDURE [dbo].[SetScalingPolicyStartOnDemand] + @Enabled BIT = NULL, + @MaxPendingStarts INT = NULL, + @UpdatedBy NVARCHAR(256) = NULL +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @PreviousEnabled BIT; + DECLARE @PreviousMax INT; + DECLARE @NewEnabled BIT; + DECLARE @NewMax INT; + DECLARE @Message NVARCHAR(200) = NULL; + DECLARE @Result VARCHAR(24); + DECLARE @ZeroMinimumCount INT = + (SELECT COUNT(*) FROM dbo.VmScalingRules WHERE MinVMs = 0) + + (SELECT COUNT(*) FROM dbo.ScalingSchedules WHERE Enabled = 1 AND MinVMs = 0); + + SELECT @PreviousEnabled = StartOnDemandEnabled, @PreviousMax = MaxPendingStarts + FROM dbo.ScalingPolicy + WHERE PolicyID = 1; + + SET @NewEnabled = COALESCE(@Enabled, @PreviousEnabled); + SET @NewMax = COALESCE(@MaxPendingStarts, @PreviousMax); + + SET @Message = CASE + WHEN @Enabled IS NULL AND @MaxPendingStarts IS NULL THEN N'Provide enabled, maxpendingstarts, or both.' + WHEN @NewMax < 1 OR @NewMax > 20 THEN N'maxpendingstarts must be between 1 and 20.' + ELSE NULL + END; + + IF @Message IS NOT NULL + SET @Result = 'Invalid'; + ELSE IF @NewEnabled = @PreviousEnabled AND @NewMax = @PreviousMax + SET @Result = 'Unchanged'; + ELSE + BEGIN + UPDATE dbo.ScalingPolicy + SET StartOnDemandEnabled = @NewEnabled, + MaxPendingStarts = @NewMax, + UpdatedBy = @UpdatedBy, + UpdatedAt = SYSUTCDATETIME() + WHERE PolicyID = 1; + SET @Result = 'Updated'; + END + + SELECT + @Result AS Result, + @Message AS Message, + CASE WHEN @Result = 'Updated' THEN @NewEnabled ELSE @PreviousEnabled END AS StartOnDemandEnabled, + CASE WHEN @Result = 'Updated' THEN @NewMax ELSE @PreviousMax END AS MaxPendingStarts, + @PreviousEnabled AS PreviousStartOnDemandEnabled, + @PreviousMax AS PreviousMaxPendingStarts, + @ZeroMinimumCount AS ZeroMinimumCount; +END +GO diff --git a/sql_queries/151_alter_procedure-SaveScalingSchedule.sql b/sql_queries/151_alter_procedure-SaveScalingSchedule.sql new file mode 100644 index 0000000..4abcfd9 --- /dev/null +++ b/sql_queries/151_alter_procedure-SaveScalingSchedule.sql @@ -0,0 +1,151 @@ +-- Redefines dbo.SaveScalingSchedule (107) so a window may have a minimum of 0 while start on +-- demand is on (4.1). With start on demand off, a minimum of 0 is refused as before, because no +-- host would start for the first user to arrive. +-- +-- Creates a schedule window (@ScheduleID NULL) or replaces one. The values are validated as the +-- API validates them, and an enabled window may not overlap another enabled window: the check +-- and the write run under an application lock, so two saves cannot both pass it. +-- +-- Result: Created, Updated, NotFound, Overlap (with the window it overlaps), or Invalid (with +-- a Message naming the field). + +CREATE PROCEDURE [dbo].[SaveScalingSchedule] + @ScheduleID INT = NULL, + @Name NVARCHAR(64), + @Enabled BIT = 1, + @DaysOfWeek TINYINT, + @StartTime TIME(0), + @EndTime TIME(0), + @MinVMs INT, + @MaxVMs INT, + @ScaleUpRatio DECIMAL(5,2), + @ScaleUpIncrement INT, + @ScaleDownRatio DECIMAL(5,2), + @ScaleDownIncrement INT, + @StopMode VARCHAR(16) = NULL, + @UpdatedBy NVARCHAR(256) = NULL +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @Message NVARCHAR(200) = NULL; + DECLARE @CleanName NVARCHAR(64) = LTRIM(RTRIM(@Name)); + DECLARE @OverlapID INT; + DECLARE @OverlapName NVARCHAR(64); + DECLARE @LockResult INT; + DECLARE @StartedTransaction BIT = 0; + DECLARE @SavedID INT = @ScheduleID; + + SET @Message = CASE + WHEN @CleanName IS NULL OR @CleanName = N'' THEN N'name is required.' + WHEN @DaysOfWeek IS NULL OR @DaysOfWeek < 1 OR @DaysOfWeek > 127 THEN N'Choose at least one day.' + WHEN @StartTime IS NULL OR @EndTime IS NULL THEN N'start and end are required.' + WHEN @StartTime = @EndTime THEN N'start and end must differ.' + WHEN @MinVMs IS NULL OR @MinVMs < 0 THEN N'minvms must be at least 0.' + WHEN @MinVMs = 0 AND NOT EXISTS ( + SELECT 1 FROM dbo.ScalingPolicy WHERE PolicyID = 1 AND StartOnDemandEnabled = 1 + ) THEN N'minvms can be 0 only while start on demand is on.' + WHEN @MaxVMs IS NULL OR @MaxVMs <= @MinVMs THEN N'maxvms must be greater than minvms.' + WHEN @ScaleUpRatio IS NULL OR @ScaleUpRatio < 0 OR @ScaleUpRatio > 100 THEN N'scaleupratio must be between 0 and 100.' + WHEN @ScaleDownRatio IS NULL OR @ScaleDownRatio < 0 OR @ScaleDownRatio > 100 THEN N'scaledownratio must be between 0 and 100.' + WHEN @ScaleUpRatio <= @ScaleDownRatio THEN N'scaleupratio must be greater than scaledownratio.' + WHEN @ScaleUpIncrement IS NULL OR @ScaleUpIncrement < 1 THEN N'scaleupincrement must be at least 1.' + WHEN @ScaleDownIncrement IS NULL OR @ScaleDownIncrement < 1 THEN N'scaledownincrement must be at least 1.' + WHEN @StopMode IS NOT NULL AND @StopMode NOT IN ('PowerOff', 'Deallocate') THEN N'stopmode must be PowerOff or Deallocate.' + ELSE NULL + END; + + IF @Message IS NOT NULL + BEGIN + SELECT CAST('Invalid' AS VARCHAR(24)) AS Result, @Message AS Message, + CAST(NULL AS INT) AS ScheduleID, CAST(NULL AS INT) AS OverlapsScheduleID, CAST(NULL AS NVARCHAR(64)) AS OverlapsName; + RETURN; + END + + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END + + EXEC @LockResult = sp_getapplock @Resource = 'LinuxBroker.ScalingSchedules', @LockMode = 'Exclusive', + @LockOwner = 'Transaction', @LockTimeout = 10000; + + IF @LockResult < 0 + BEGIN + IF @StartedTransaction = 1 ROLLBACK TRANSACTION; + SELECT CAST('Busy' AS VARCHAR(24)) AS Result, N'Scaling schedules are busy. Please try again.' AS Message, + CAST(NULL AS INT) AS ScheduleID, CAST(NULL AS INT) AS OverlapsScheduleID, CAST(NULL AS NVARCHAR(64)) AS OverlapsName; + RETURN; + END + + IF @ScheduleID IS NOT NULL AND NOT EXISTS (SELECT 1 FROM dbo.ScalingSchedules WHERE ScheduleID = @ScheduleID) + BEGIN + IF @StartedTransaction = 1 COMMIT TRANSACTION; + SELECT CAST('NotFound' AS VARCHAR(24)) AS Result, CAST(NULL AS NVARCHAR(200)) AS Message, + @ScheduleID AS ScheduleID, CAST(NULL AS INT) AS OverlapsScheduleID, CAST(NULL AS NVARCHAR(64)) AS OverlapsName; + RETURN; + END + + IF COALESCE(@Enabled, 1) = 1 + BEGIN + SELECT TOP 1 @OverlapID = s.ScheduleID, @OverlapName = s.Name + FROM dbo.ScalingSchedules s + CROSS APPLY dbo.fnScheduleWeekIntervals(s.DaysOfWeek, s.StartTime, s.EndTime) existing + CROSS APPLY dbo.fnScheduleWeekIntervals(@DaysOfWeek, @StartTime, @EndTime) proposed + WHERE s.Enabled = 1 + AND (@ScheduleID IS NULL OR s.ScheduleID <> @ScheduleID) + AND existing.StartMinute < proposed.EndMinute + AND proposed.StartMinute < existing.EndMinute + ORDER BY s.ScheduleID; + + IF @OverlapID IS NOT NULL + BEGIN + IF @StartedTransaction = 1 COMMIT TRANSACTION; + SELECT CAST('Overlap' AS VARCHAR(24)) AS Result, CAST(NULL AS NVARCHAR(200)) AS Message, + @ScheduleID AS ScheduleID, @OverlapID AS OverlapsScheduleID, @OverlapName AS OverlapsName; + RETURN; + END + END + + IF @ScheduleID IS NULL + BEGIN + INSERT INTO dbo.ScalingSchedules ( + Name, Enabled, DaysOfWeek, StartTime, EndTime, MinVMs, MaxVMs, ScaleUpRatio, ScaleUpIncrement, + ScaleDownRatio, ScaleDownIncrement, StopMode, UpdatedBy + ) + VALUES ( + @CleanName, COALESCE(@Enabled, 1), @DaysOfWeek, @StartTime, @EndTime, @MinVMs, @MaxVMs, @ScaleUpRatio, + @ScaleUpIncrement, @ScaleDownRatio, @ScaleDownIncrement, @StopMode, @UpdatedBy + ); + SET @SavedID = CAST(SCOPE_IDENTITY() AS INT); + END + ELSE + BEGIN + UPDATE dbo.ScalingSchedules + SET Name = @CleanName, + Enabled = COALESCE(@Enabled, 1), + DaysOfWeek = @DaysOfWeek, + StartTime = @StartTime, + EndTime = @EndTime, + MinVMs = @MinVMs, + MaxVMs = @MaxVMs, + ScaleUpRatio = @ScaleUpRatio, + ScaleUpIncrement = @ScaleUpIncrement, + ScaleDownRatio = @ScaleDownRatio, + ScaleDownIncrement = @ScaleDownIncrement, + StopMode = @StopMode, + UpdatedBy = @UpdatedBy + WHERE ScheduleID = @ScheduleID; + END + + IF @StartedTransaction = 1 COMMIT TRANSACTION; + + SELECT CAST(CASE WHEN @ScheduleID IS NULL THEN 'Created' ELSE 'Updated' END AS VARCHAR(24)) AS Result, + CAST(NULL AS NVARCHAR(200)) AS Message, + @SavedID AS ScheduleID, + CAST(NULL AS INT) AS OverlapsScheduleID, + CAST(NULL AS NVARCHAR(64)) AS OverlapsName; +END +GO diff --git a/sql_queries/152_create_procedure-ReserveVmForStart.sql b/sql_queries/152_create_procedure-ReserveVmForStart.sql new file mode 100644 index 0000000..8fca00e --- /dev/null +++ b/sql_queries/152_create_procedure-ReserveVmForStart.sql @@ -0,0 +1,217 @@ +-- Starts a stopped host for a user whose checkout found no host ready (4.1, start on demand). +-- The API calls it after dbo.CheckoutVm returns no host, and acts on Result: +-- Disabled Start on demand is off. The checkout is refused as before. +-- Busy The scaling lock was not free within five seconds. Ask again shortly. +-- ReadyNow A host became ready since that checkout. Check out again. +-- Started A stopped host was started for the user (VMID, VMName, ActivityID). The +-- API asks Azure to start it, and records it as off again if Azure refuses. +-- AlreadyStarting Enough hosts are starting for the users waiting, MaxPendingStarts are +-- starting, or no other host can start while some still are. The user waits +-- for one of them. +-- AtMaximum No host is starting and the phase's MaxVMs hosts are all powered on. +-- NoCandidate No host is starting and none is stopped and free to start. +-- +-- A host counts as starting from its power-on until the reachability probe reaches it, for up +-- to ten minutes, as in scaling. The users waiting are dbo.fnWaitingCheckoutUsers and this +-- user. RetryAfterSeconds is when the AVD host should ask again: the median start time over the +-- last week (60 seconds without history), less how long the oldest host has been starting when +-- the user waits for one, kept between 30 and 120 seconds. BootingHostsJson lists the starting +-- hosts a checkout could take once they answer, oldest first, at most MaxPendingStarts, so the +-- API can probe them. +-- +-- The decision and the start run under the scaling lock, so start on demand and scaling never +-- pick the same host or together pass MaxVMs. A start is recorded exactly as a scaling start +-- is, with a 'Start On Demand' row in dbo.VmScalingActivityLog. + +CREATE PROCEDURE [dbo].[ReserveVmForStart] + @Username VARCHAR(255), + @AvdHost VARCHAR(255) = NULL +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @Now DATETIME = GETDATE(), @NowUtc DATETIME2(3) = SYSUTCDATETIME(), @Boot INT = 10; + DECLARE @Enabled BIT, @MaxPending INT, @LockResult INT, @StartedTransaction BIT = 0; + DECLARE @Result VARCHAR(24), @VMID INT, @VMName VARCHAR(255), @PreviousNetworkStatus VARCHAR(16), @ActivityID INT; + DECLARE @PoweredOn INT = 0, @Serviceable INT = 0, @InUse INT = 0, @Draining INT = 0, @Booting INT = 0, @Waiting INT = 0; + DECLARE @OldestBoot DATETIME, @MedianSeconds INT, @RetryAfter INT; + DECLARE @PhaseName NVARCHAR(64), @ScheduleID INT, @MinVMs INT, @MaxVMs INT; + DECLARE @BootingJson NVARCHAR(MAX); + DECLARE @Started TABLE (VMID INT, Hostname VARCHAR(255), PreviousNetworkStatus VARCHAR(16)); + + SELECT @Enabled = StartOnDemandEnabled, @MaxPending = MaxPendingStarts + FROM dbo.ScalingPolicy + WHERE PolicyID = 1; + + IF COALESCE(@Enabled, 0) = 0 + BEGIN + SELECT CAST('Disabled' AS VARCHAR(24)) AS Result, CAST(NULL AS INT) AS VMID, CAST(NULL AS VARCHAR(255)) AS VMName, + CAST(NULL AS VARCHAR(16)) AS PreviousNetworkStatus, CAST(NULL AS INT) AS ActivityID, + CAST(NULL AS INT) AS RetryAfterSeconds, CAST(N'[]' AS NVARCHAR(MAX)) AS BootingHostsJson, + 0 AS Waiting, 0 AS Booting; + RETURN; + END + + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END + ELSE + BEGIN + SAVE TRANSACTION ReserveVmForStartSave; + END + + EXEC @LockResult = sp_getapplock @Resource = 'LinuxBroker.Scaling', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = 5000; + + IF @LockResult < 0 + BEGIN + IF @StartedTransaction = 1 + BEGIN + ROLLBACK TRANSACTION; + END + ELSE + BEGIN + ROLLBACK TRANSACTION ReserveVmForStartSave; + END + + SELECT CAST('Busy' AS VARCHAR(24)) AS Result, CAST(NULL AS INT) AS VMID, CAST(NULL AS VARCHAR(255)) AS VMName, + CAST(NULL AS VARCHAR(16)) AS PreviousNetworkStatus, CAST(NULL AS INT) AS ActivityID, + 30 AS RetryAfterSeconds, CAST(N'[]' AS NVARCHAR(MAX)) AS BootingHostsJson, + 0 AS Waiting, 0 AS Booting; + RETURN; + END + + -- CheckoutVm's test for a host it can give a new user. + IF EXISTS ( + SELECT 1 FROM dbo.VirtualMachines + WHERE PowerState = 'On' AND NetworkStatus = 'Reachable' AND VmStatus = 'Available' + AND Username IS NULL AND CleanupPending = 0 AND DrainRequested = 0 + ) + BEGIN + IF @StartedTransaction = 1 COMMIT TRANSACTION; + + SELECT CAST('ReadyNow' AS VARCHAR(24)) AS Result, CAST(NULL AS INT) AS VMID, CAST(NULL AS VARCHAR(255)) AS VMName, + CAST(NULL AS VARCHAR(16)) AS PreviousNetworkStatus, CAST(NULL AS INT) AS ActivityID, + CAST(NULL AS INT) AS RetryAfterSeconds, CAST(N'[]' AS NVARCHAR(MAX)) AS BootingHostsJson, + 0 AS Waiting, 0 AS Booting; + RETURN; + END + + SELECT @PoweredOn = COALESCE(SUM(CASE WHEN PowerState = 'On' THEN 1 ELSE 0 END), 0), + @Serviceable = COALESCE(SUM(CASE WHEN PowerState = 'On' AND VmStatus <> 'Maintenance' AND DrainRequested = 0 + AND (NetworkStatus = 'Reachable' OR PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now)) + AND NOT (VmStatus = 'Available' AND (Username IS NOT NULL OR LeaseId IS NOT NULL)) + THEN 1 ELSE 0 END), 0), + @InUse = COALESCE(SUM(CASE WHEN PowerState = 'On' AND DrainRequested = 0 + AND (VmStatus IN ('CheckedOut', 'Released') OR CleanupPending = 1) + THEN 1 ELSE 0 END), 0), + @Draining = COALESCE(SUM(CASE WHEN DrainRequested = 1 THEN 1 ELSE 0 END), 0), + @Booting = COALESCE(SUM(CASE WHEN PowerState = 'On' AND COALESCE(NetworkStatus, 'Unreachable') <> 'Reachable' + AND VmStatus = 'Available' AND Username IS NULL AND LeaseId IS NULL + AND DrainRequested = 0 AND PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now) + THEN 1 ELSE 0 END), 0), + @OldestBoot = MIN(CASE WHEN PowerState = 'On' AND COALESCE(NetworkStatus, 'Unreachable') <> 'Reachable' + AND VmStatus = 'Available' AND Username IS NULL AND LeaseId IS NULL + AND DrainRequested = 0 AND PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now) + THEN PowerStateChangedDate END) + FROM dbo.VirtualMachines; + + SELECT @Waiting = COUNT(*) + FROM ( + SELECT Username FROM dbo.fnWaitingCheckoutUsers() + UNION + SELECT @Username + ) waiting; + + SELECT TOP 1 @MedianSeconds = CAST(ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY Seconds) OVER (), 0) AS INT) + FROM dbo.HostStartEvents + WHERE ReadyAt >= DATEADD(DAY, -7, @NowUtc); + SET @MedianSeconds = COALESCE(@MedianSeconds, 60); + + IF @Booting >= @MaxPending OR @Booting >= @Waiting + BEGIN + SET @Result = 'AlreadyStarting'; + END + ELSE + BEGIN + SELECT @PhaseName = PhaseName, @ScheduleID = ScheduleID, @MinVMs = MinVMs, @MaxVMs = MaxVMs + FROM dbo.fnActiveScalingPhase(NULL); + + IF @MaxVMs IS NULL OR @PoweredOn < @MaxVMs + BEGIN + SELECT TOP 1 @VMID = VMID + FROM dbo.VirtualMachines WITH (UPDLOCK, ROWLOCK) + WHERE PowerState = 'Off' AND VmStatus = 'Available' AND Username IS NULL AND LeaseId IS NULL + AND DrainRequested = 0 AND CleanupPending = 0 + ORDER BY VMID; + END + + IF @VMID IS NOT NULL + BEGIN + UPDATE vm + SET PowerState = 'On', NetworkStatus = 'Unreachable', PowerStateChangedDate = @Now, + StartRequestedAt = @NowUtc, LastUpdateDate = @Now + OUTPUT INSERTED.VMID, INSERTED.Hostname, DELETED.NetworkStatus INTO @Started (VMID, Hostname, PreviousNetworkStatus) + FROM dbo.VirtualMachines vm + WHERE vm.VMID = @VMID; + + SELECT @VMName = Hostname, @PreviousNetworkStatus = PreviousNetworkStatus FROM @Started; + + INSERT INTO dbo.VmScalingActivityLog (CheckTimestamp, CurrentRunningVMs, CurrentInUseVMs, ActionTaken, VMsPoweredOn, VMsPoweredOff, + NewTotalVMs, Outcome, Notes, PhaseName, ScheduleID, MinVMs, MaxVMs, ServiceableVMs, DrainingVMs) + VALUES (@Now, @PoweredOn, @InUse, N'Start On Demand', 1, 0, @PoweredOn + 1, + CONCAT(N'Requested power-on of ', @VMName, N' for a waiting user'), + CONCAT(N'No host was ready for ', @Username, + CASE WHEN @AvdHost IS NOT NULL THEN CONCAT(N' (AVD host ', @AvdHost, N')') ELSE N'' END, + N', so ', @VMName, N' was started. Counts poweredOn=', @PoweredOn, N', serviceable=', @Serviceable, + N', inUse=', @InUse, N', starting=', @Booting, N', waiting=', @Waiting, + N'. Phase=', COALESCE(@PhaseName, N'Default rule'), N'.'), + @PhaseName, @ScheduleID, @MinVMs, @MaxVMs, @Serviceable, @Draining); + SET @ActivityID = SCOPE_IDENTITY(); + + SET @Result = 'Started'; + END + ELSE IF @Booting > 0 + SET @Result = 'AlreadyStarting'; + ELSE IF @MaxVMs IS NOT NULL AND @PoweredOn >= @MaxVMs + SET @Result = 'AtMaximum'; + ELSE + SET @Result = 'NoCandidate'; + END + + SET @RetryAfter = CASE + WHEN @Result = 'Started' THEN @MedianSeconds + WHEN @Result = 'AlreadyStarting' THEN @MedianSeconds - COALESCE(DATEDIFF(SECOND, @OldestBoot, @Now), 0) + ELSE NULL + END; + SET @RetryAfter = CASE WHEN @RetryAfter IS NULL THEN NULL WHEN @RetryAfter < 30 THEN 30 WHEN @RetryAfter > 120 THEN 120 ELSE @RetryAfter END; + + IF @Result = 'AlreadyStarting' + BEGIN + SET @BootingJson = ( + SELECT TOP (@MaxPending) VMID, Hostname, IPAddress + FROM dbo.VirtualMachines + WHERE PowerState = 'On' AND COALESCE(NetworkStatus, 'Unreachable') <> 'Reachable' + AND VmStatus = 'Available' AND Username IS NULL AND LeaseId IS NULL + AND DrainRequested = 0 AND CleanupPending = 0 + AND PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now) + ORDER BY PowerStateChangedDate, VMID + FOR JSON PATH + ); + END + + IF @StartedTransaction = 1 COMMIT TRANSACTION; + + SELECT @Result AS Result, + @VMID AS VMID, + @VMName AS VMName, + @PreviousNetworkStatus AS PreviousNetworkStatus, + @ActivityID AS ActivityID, + @RetryAfter AS RetryAfterSeconds, + COALESCE(@BootingJson, N'[]') AS BootingHostsJson, + @Waiting AS Waiting, + CASE WHEN @Result = 'Started' THEN @Booting + 1 ELSE @Booting END AS Booting; +END +GO diff --git a/sql_queries/153_alter_procedure-TriggerScalingLogic.sql b/sql_queries/153_alter_procedure-TriggerScalingLogic.sql new file mode 100644 index 0000000..ddfb8cb --- /dev/null +++ b/sql_queries/153_alter_procedure-TriggerScalingLogic.sql @@ -0,0 +1,320 @@ +-- Redefines dbo.TriggerScalingLogic for start on demand and scale to zero (4.1): +-- +-- * Users waiting for a host to start (dbo.fnWaitingCheckoutUsers) count as demand: utilization +-- is (InUse + Waiting) / Serviceable, and 100% when nothing is serviceable but someone is +-- waiting or in use. +-- * A phase's MinVMs may be 0 while start on demand is on (dbo.ScalingPolicy. +-- StartOnDemandEnabled), so idle hosts stop until someone needs one. With start on demand off, +-- 0 is read as 1, because no host would start for the first user, and the notes say so. +-- * Scaling down for low utilization keeps at least InUse + Waiting hosts serviceable while +-- anyone is waiting, never fewer than MinVMs. With nobody waiting, nothing changes. +-- * The notes add waiting=, the reason names the users waiting, and the dry run returns Waiting +-- and StartOnDemandEnabled. +-- +-- Everything else is unchanged from 140: +-- +-- * While an active maintenance run is waiting for a spare host (dbo.MaintenanceRuns. +-- SurgeRequested), scaling keeps one more host serviceable than the phase's MinVMs, never +-- more than its MaxVMs, and the dry run returns it as MaintenanceSurge. +-- * The values come from dbo.fnActiveScalingPhase: the enabled schedule window that covers the +-- policy's local time, or else the default rule. +-- * @DryRun = 1 makes the same decision against the current counts without changing anything, +-- and returns one decision row for the portal's preview. @AtUtc resolves the phase at another +-- time and @OverrideJson replaces its values (MinVMs, MaxVMs, ScaleUpRatio, ScaleUpIncrement, +-- ScaleDownRatio, ScaleDownIncrement, StopMode, PhaseName); both apply to a dry run only. +-- * Scaling down to get under MaxVMs never takes serviceable hosts below MinVMs. +-- * A run waits up to five seconds for the scaling lock, which maintenance admission and start +-- on demand hold briefly, rather than skipping. +-- * Each power-on stamps StartRequestedAt for the start-to-ready measure, a power-off clears it. +-- +-- A normal call returns the same columns as before, so the previous API build is unaffected. + +CREATE PROCEDURE [dbo].[TriggerScalingLogic] + @DryRun BIT = 0, + @AtUtc DATETIME2(0) = NULL, + @OverrideJson NVARCHAR(MAX) = NULL +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @Actions TABLE ( + ActionType VARCHAR(16), VMName VARCHAR(255), VMID INT, + StopMode VARCHAR(16), PreviousNetworkStatus VARCHAR(16), ActivityID INT NULL + ); + DECLARE @Candidates TABLE (VMID INT, Hostname VARCHAR(255)); + DECLARE @Now DATETIME = GETDATE(), @NowUtc DATETIME2(3) = SYSUTCDATETIME(), @Boot INT = 10, @LockResult INT; + DECLARE @StartedTransaction BIT = 0; + DECLARE @IsDryRun BIT = COALESCE(@DryRun, 0); + DECLARE @PhaseAtUtc DATETIME2(0) = CASE + WHEN COALESCE(@DryRun, 0) = 1 AND @AtUtc IS NOT NULL THEN @AtUtc + ELSE CAST(SYSUTCDATETIME() AS DATETIME2(0)) + END; + + IF @IsDryRun = 0 + BEGIN + IF @@TRANCOUNT = 0 + BEGIN + BEGIN TRANSACTION; + SET @StartedTransaction = 1; + END + ELSE + BEGIN + SAVE TRANSACTION ScalingLogicSave; + END + EXEC @LockResult = sp_getapplock @Resource = 'LinuxBroker.Scaling', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = 5000; + + IF @LockResult < 0 + BEGIN + IF @StartedTransaction = 1 + BEGIN + ROLLBACK TRANSACTION; + END + ELSE + BEGIN + ROLLBACK TRANSACTION ScalingLogicSave; + END + SELECT ActionType, VMName, VMID, StopMode, PreviousNetworkStatus, ActivityID FROM @Actions; + RETURN; + END + END + + DECLARE @PhaseSource VARCHAR(16), @ScheduleID INT, @PhaseName NVARCHAR(64), @RuleID INT, + @MinVMs INT, @MaxVMs INT, @ScaleUpRatio DECIMAL(5,2), @ScaleUpIncrement INT, + @ScaleDownRatio DECIMAL(5,2), @ScaleDownIncrement INT, @StopMode VARCHAR(16), + @TimeZone NVARCHAR(64), @LocalTime DATETIME2(0); + DECLARE @OriginalMin INT, @OriginalMax INT, @OriginalUpInc INT, @OriginalDownInc INT; + DECLARE @Surge BIT = 0; + DECLARE @StartOnDemand BIT = COALESCE((SELECT StartOnDemandEnabled FROM dbo.ScalingPolicy WHERE PolicyID = 1), 0); + + SELECT @PhaseSource = Source, @ScheduleID = ScheduleID, @PhaseName = PhaseName, @RuleID = RuleID, + @MinVMs = MinVMs, @MaxVMs = MaxVMs, @ScaleUpRatio = ScaleUpRatio, @ScaleUpIncrement = ScaleUpIncrement, + @ScaleDownRatio = ScaleDownRatio, @ScaleDownIncrement = ScaleDownIncrement, + @StopMode = COALESCE(StopMode, 'PowerOff'), @TimeZone = TimeZone, @LocalTime = LocalTime + FROM dbo.fnActiveScalingPhase(@PhaseAtUtc); + + IF @IsDryRun = 1 AND @OverrideJson IS NOT NULL AND ISJSON(@OverrideJson) = 1 + BEGIN + SET @MinVMs = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.MinVMs') AS INT), @MinVMs); + SET @MaxVMs = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.MaxVMs') AS INT), @MaxVMs); + SET @ScaleUpRatio = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.ScaleUpRatio') AS DECIMAL(5,2)), @ScaleUpRatio); + SET @ScaleUpIncrement = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.ScaleUpIncrement') AS INT), @ScaleUpIncrement); + SET @ScaleDownRatio = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.ScaleDownRatio') AS DECIMAL(5,2)), @ScaleDownRatio); + SET @ScaleDownIncrement = COALESCE(TRY_CAST(JSON_VALUE(@OverrideJson, '$.ScaleDownIncrement') AS INT), @ScaleDownIncrement); + SET @StopMode = CASE WHEN JSON_VALUE(@OverrideJson, '$.StopMode') IN ('PowerOff', 'Deallocate') + THEN JSON_VALUE(@OverrideJson, '$.StopMode') ELSE COALESCE(@StopMode, 'PowerOff') END; + SET @PhaseName = COALESCE(CAST(JSON_VALUE(@OverrideJson, '$.PhaseName') AS NVARCHAR(64)), @PhaseName, N'Proposed values'); + SET @PhaseSource = 'Proposed'; + END + + DECLARE @PoweredOn INT = 0, @Serviceable INT = 0, @InUse INT = 0, @Draining INT = 0, @Utilization DECIMAL(7,2) = 0, + @Headroom INT = 0, @RequestCount INT = 0, @ActualOn INT = 0, @ActualOff INT = 0, + @ActionTaken NVARCHAR(50) = N'No Action', @Outcome NVARCHAR(255), @Notes NVARCHAR(MAX), + @Reason NVARCHAR(400), @Corrections NVARCHAR(600) = N'', @ActivityID INT, @Mode VARCHAR(8) = NULL, + @Waiting INT = 0, @DownFloor INT; + + SELECT @PoweredOn = SUM(CASE WHEN PowerState = 'On' THEN 1 ELSE 0 END), + -- Capacity a user can be given: an Available host must also pass CheckoutVm's test. + @Serviceable = SUM(CASE WHEN PowerState = 'On' AND VmStatus <> 'Maintenance' AND DrainRequested = 0 + AND (NetworkStatus = 'Reachable' OR PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now)) + AND NOT (VmStatus = 'Available' AND (Username IS NOT NULL OR LeaseId IS NOT NULL)) + THEN 1 ELSE 0 END), + @InUse = SUM(CASE WHEN PowerState = 'On' AND DrainRequested = 0 AND (VmStatus IN ('CheckedOut', 'Released') OR CleanupPending = 1) THEN 1 ELSE 0 END), + @Draining = SUM(CASE WHEN DrainRequested = 1 THEN 1 ELSE 0 END) + FROM dbo.VirtualMachines; + + SET @PoweredOn = COALESCE(@PoweredOn, 0); + SET @Serviceable = COALESCE(@Serviceable, 0); + SET @InUse = COALESCE(@InUse, 0); + SET @Draining = COALESCE(@Draining, 0); + SELECT @Waiting = COUNT(*) FROM dbo.fnWaitingCheckoutUsers(); + SET @Utilization = CASE WHEN @Serviceable = 0 THEN CASE WHEN @InUse + @Waiting > 0 THEN 100 ELSE 0 END + ELSE CAST((@InUse + @Waiting) * 100.0 / @Serviceable AS DECIMAL(7,2)) END; + + IF @MinVMs IS NULL OR @MaxVMs IS NULL + BEGIN + IF @IsDryRun = 1 + BEGIN + SELECT CAST('None' AS VARCHAR(16)) AS Action, 0 AS RequestCount, 0 AS CandidateCount, + CAST(N'[]' AS NVARCHAR(MAX)) AS CandidatesJson, CAST(N'No scaling rule is configured.' AS NVARCHAR(400)) AS Reason, + CAST(NULL AS VARCHAR(16)) AS PhaseSource, CAST(NULL AS INT) AS ScheduleID, CAST(NULL AS NVARCHAR(64)) AS PhaseName, + CAST(NULL AS INT) AS MinVMs, CAST(NULL AS INT) AS MaxVMs, CAST(NULL AS DECIMAL(5,2)) AS ScaleUpRatio, + CAST(NULL AS INT) AS ScaleUpIncrement, CAST(NULL AS DECIMAL(5,2)) AS ScaleDownRatio, CAST(NULL AS INT) AS ScaleDownIncrement, + CAST(NULL AS VARCHAR(16)) AS StopMode, @PoweredOn AS PoweredOn, @Serviceable AS Serviceable, @InUse AS InUse, + @Draining AS Draining, @Utilization AS Utilization, @TimeZone AS TimeZone, + CONVERT(VARCHAR(19), @LocalTime, 126) AS LocalTime, CONVERT(VARCHAR(33), @PhaseAtUtc, 126) + 'Z' AS AtUtc, + CAST(0 AS BIT) AS MaintenanceSurge, @Waiting AS Waiting, @StartOnDemand AS StartOnDemandEnabled; + RETURN; + END + + INSERT INTO dbo.VmScalingActivityLog (CheckTimestamp, CurrentRunningVMs, CurrentInUseVMs, ActionTaken, VMsPoweredOn, VMsPoweredOff, NewTotalVMs, Outcome, Notes, ServiceableVMs, DrainingVMs) + VALUES (@Now, @PoweredOn, @InUse, N'No Action', 0, 0, @PoweredOn, N'No scaling action was necessary', N'No scaling rule is configured.', @Serviceable, @Draining); + IF @StartedTransaction = 1 COMMIT TRANSACTION; + SELECT ActionType, VMName, VMID, StopMode, PreviousNetworkStatus, ActivityID FROM @Actions; + RETURN; + END + + SET @OriginalMin = @MinVMs; SET @OriginalMax = @MaxVMs; SET @OriginalUpInc = @ScaleUpIncrement; SET @OriginalDownInc = @ScaleDownIncrement; + SET @MinVMs = CASE WHEN @MinVMs < 0 THEN 0 WHEN @MinVMs < 1 AND @StartOnDemand = 0 THEN 1 ELSE @MinVMs END; + SET @MaxVMs = CASE WHEN @MaxVMs < @MinVMs THEN @MinVMs ELSE @MaxVMs END; + SET @ScaleUpIncrement = CASE WHEN COALESCE(@ScaleUpIncrement, 0) < 1 THEN 1 ELSE @ScaleUpIncrement END; + SET @ScaleDownIncrement = CASE WHEN COALESCE(@ScaleDownIncrement, 0) < 1 THEN 1 ELSE @ScaleDownIncrement END; + + IF @OriginalMin = 0 AND @MinVMs = 1 SET @Corrections = CONCAT(@Corrections, N' Start on demand is off, so MinVMs 0 was read as 1.'); + ELSE IF @OriginalMin <> @MinVMs SET @Corrections = CONCAT(@Corrections, N' Corrected MinVMs from ', @OriginalMin, N' to ', @MinVMs, N'.'); + IF @OriginalMax <> @MaxVMs SET @Corrections = CONCAT(@Corrections, N' Corrected MaxVMs from ', @OriginalMax, N' to ', @MaxVMs, N'.'); + IF COALESCE(@OriginalUpInc, 0) <> @ScaleUpIncrement SET @Corrections = CONCAT(@Corrections, N' Corrected ScaleUpIncrement from ', @OriginalUpInc, N' to ', @ScaleUpIncrement, N'.'); + IF COALESCE(@OriginalDownInc, 0) <> @ScaleDownIncrement SET @Corrections = CONCAT(@Corrections, N' Corrected ScaleDownIncrement from ', @OriginalDownInc, N' to ', @ScaleDownIncrement, N'.'); + + IF @MinVMs < @MaxVMs AND EXISTS (SELECT 1 FROM dbo.MaintenanceRuns WHERE Status = 'Active' AND SurgeRequested = 1) + BEGIN + SET @MinVMs = @MinVMs + 1; + SET @Surge = 1; + END + + SET @Headroom = @MaxVMs - @PoweredOn; + -- Scaling down for low utilization leaves a host for everyone in use or waiting. + SET @DownFloor = CASE WHEN @Waiting > 0 AND @InUse + @Waiting > @MinVMs THEN @InUse + @Waiting ELSE @MinVMs END; + + IF @Serviceable < @MinVMs AND @Headroom > 0 + BEGIN + SET @Mode = 'Up'; + SET @RequestCount = CASE WHEN @MinVMs - @Serviceable < @Headroom THEN @MinVMs - @Serviceable ELSE @Headroom END; + SET @Reason = N'Serviceable hosts are below the minimum.'; + END + ELSE IF @PoweredOn > @MaxVMs + BEGIN + -- The minimum wins: never stop so many that fewer than MinVMs hosts can take a user. + SET @RequestCount = @PoweredOn - @MaxVMs; + IF @RequestCount > @Serviceable - @MinVMs SET @RequestCount = @Serviceable - @MinVMs; + + IF @RequestCount > 0 + BEGIN + SET @Mode = 'Down'; + SET @Reason = N'Powered-on hosts exceed the maximum.'; + END + ELSE + BEGIN + SET @RequestCount = 0; + SET @Reason = N'Powered-on hosts exceed the maximum, but stopping any would leave fewer serviceable hosts than the minimum.'; + END + END + ELSE IF @Utilization >= @ScaleUpRatio AND @Headroom > 0 + BEGIN + SET @Mode = 'Up'; + SET @RequestCount = CASE WHEN @ScaleUpIncrement < @Headroom THEN @ScaleUpIncrement ELSE @Headroom END; + SET @Reason = N'Utilization is at or above the scale-up ratio.'; + END + ELSE IF @Utilization <= @ScaleDownRatio AND @Serviceable > @DownFloor + BEGIN + SET @Mode = 'Down'; + SET @RequestCount = CASE WHEN @ScaleDownIncrement < @Serviceable - @DownFloor THEN @ScaleDownIncrement ELSE @Serviceable - @DownFloor END; + SET @Reason = N'Utilization is at or below the scale-down ratio.'; + END + ELSE + BEGIN + SET @Reason = N'No scaling threshold was crossed.'; + END + + IF @Waiting > 0 + SET @Reason = CONCAT(@Reason, N' ', @Waiting, CASE WHEN @Waiting = 1 THEN N' user is' ELSE N' users are' END, N' waiting for a host to start.'); + + IF @Mode = 'Up' AND @RequestCount > 0 + BEGIN + INSERT INTO @Candidates (VMID, Hostname) + SELECT TOP (@RequestCount) VMID, Hostname + FROM dbo.VirtualMachines WITH (UPDLOCK, ROWLOCK) + WHERE PowerState = 'Off' AND VmStatus = 'Available' AND Username IS NULL AND LeaseId IS NULL AND DrainRequested = 0 + ORDER BY CleanupPending, VMID; + END + ELSE IF @Mode = 'Down' AND @RequestCount > 0 + BEGIN + INSERT INTO @Candidates (VMID, Hostname) + SELECT TOP (@RequestCount) VMID, Hostname + FROM dbo.VirtualMachines WITH (UPDLOCK, ROWLOCK) + WHERE PowerState = 'On' AND NetworkStatus = 'Reachable' AND VmStatus = 'Available' + AND Username IS NULL AND LeaseId IS NULL AND CleanupPending = 0 AND DrainRequested = 0 + AND (PowerStateChangedDate IS NULL OR PowerStateChangedDate < DATEADD(MINUTE, -@Boot, @Now)) + ORDER BY VMID DESC; + END + + IF @IsDryRun = 1 + BEGIN + DECLARE @CandidateCount INT = (SELECT COUNT(*) FROM @Candidates); + IF @RequestCount > 0 AND @CandidateCount < @RequestCount + BEGIN + SET @Reason = CONCAT(@Reason, N' Only ', @CandidateCount, N' of ', @RequestCount, N' requested hosts are available to ', + CASE WHEN @Mode = 'Up' THEN N'start.' ELSE N'stop.' END); + END + + SELECT CAST(CASE WHEN @Mode = 'Up' AND @CandidateCount > 0 THEN 'PowerOn' + WHEN @Mode = 'Down' AND @CandidateCount > 0 THEN 'PowerOff' + ELSE 'None' END AS VARCHAR(16)) AS Action, + @RequestCount AS RequestCount, + @CandidateCount AS CandidateCount, + COALESCE((SELECT Hostname FROM @Candidates ORDER BY Hostname FOR JSON PATH), N'[]') AS CandidatesJson, + @Reason AS Reason, + @PhaseSource AS PhaseSource, @ScheduleID AS ScheduleID, @PhaseName AS PhaseName, + @MinVMs AS MinVMs, @MaxVMs AS MaxVMs, @ScaleUpRatio AS ScaleUpRatio, @ScaleUpIncrement AS ScaleUpIncrement, + @ScaleDownRatio AS ScaleDownRatio, @ScaleDownIncrement AS ScaleDownIncrement, @StopMode AS StopMode, + @PoweredOn AS PoweredOn, @Serviceable AS Serviceable, @InUse AS InUse, @Draining AS Draining, + @Utilization AS Utilization, @TimeZone AS TimeZone, + CONVERT(VARCHAR(19), @LocalTime, 126) AS LocalTime, CONVERT(VARCHAR(33), @PhaseAtUtc, 126) + 'Z' AS AtUtc, + @Surge AS MaintenanceSurge, @Waiting AS Waiting, @StartOnDemand AS StartOnDemandEnabled; + RETURN; + END + + IF @Mode = 'Up' AND @RequestCount > 0 + BEGIN + UPDATE vm + SET PowerState = 'On', NetworkStatus = 'Unreachable', PowerStateChangedDate = @Now, StartRequestedAt = @NowUtc, LastUpdateDate = @Now + OUTPUT 'PowerOn', INSERTED.Hostname, INSERTED.VMID, NULL, DELETED.NetworkStatus, NULL + INTO @Actions (ActionType, VMName, VMID, StopMode, PreviousNetworkStatus, ActivityID) + FROM dbo.VirtualMachines vm INNER JOIN @Candidates c ON c.VMID = vm.VMID; + + SET @ActualOn = @@ROWCOUNT; + SET @ActionTaken = CASE WHEN @ActualOn > 0 THEN N'Scale Up' ELSE N'No Action' END; + SET @Outcome = CASE WHEN @ActualOn > 0 THEN CONCAT(N'Requested power-on of ', @ActualOn, N' VM(s)') ELSE N'No scaling action was necessary' END; + IF @ActualOn < @RequestCount SET @Reason = CONCAT(@Reason, N' Only ', @ActualOn, N' of ', @RequestCount, N' requested hosts were available to start.'); + END + ELSE IF @Mode = 'Down' AND @RequestCount > 0 + BEGIN + UPDATE vm + SET PowerState = 'Off', NetworkStatus = 'Unreachable', PowerStateChangedDate = @Now, StartRequestedAt = NULL, LastUpdateDate = @Now + OUTPUT 'PowerOff', INSERTED.Hostname, INSERTED.VMID, @StopMode, DELETED.NetworkStatus, NULL + INTO @Actions (ActionType, VMName, VMID, StopMode, PreviousNetworkStatus, ActivityID) + FROM dbo.VirtualMachines vm INNER JOIN @Candidates c ON c.VMID = vm.VMID; + + SET @ActualOff = @@ROWCOUNT; + SET @ActionTaken = CASE WHEN @ActualOff > 0 THEN N'Scale Down' ELSE N'No Action' END; + SET @Outcome = CASE WHEN @ActualOff > 0 AND @StopMode = 'Deallocate' THEN CONCAT(N'Requested deallocation of ', @ActualOff, N' VM(s)') WHEN @ActualOff > 0 THEN CONCAT(N'Requested power-off of ', @ActualOff, N' VM(s)') ELSE N'No scaling action was necessary' END; + IF @ActualOff < @RequestCount SET @Reason = CONCAT(@Reason, N' Only ', @ActualOff, N' of ', @RequestCount, N' requested hosts were available to stop.'); + END + ELSE + BEGIN + SET @Outcome = N'No scaling action was necessary'; + END + + SET @Notes = CONCAT(N'Rule Min=', @MinVMs, N', Max=', @MaxVMs, N', UpRatio=', @ScaleUpRatio, N', DownRatio=', @ScaleDownRatio, + N', UpIncrement=', @ScaleUpIncrement, N', DownIncrement=', @ScaleDownIncrement, N'. Counts poweredOn=', @PoweredOn, + N', serviceable=', @Serviceable, N', inUse=', @InUse, N', waiting=', @Waiting, N', draining=', @Draining, N', utilization=', @Utilization, N'%. ', @Reason, @Corrections, + N' Phase=', COALESCE(@PhaseName, N'Default rule'), N'.', + CASE WHEN @Surge = 1 THEN N' A maintenance run asked for one more ready host.' ELSE N'' END); + + INSERT INTO dbo.VmScalingActivityLog (CheckTimestamp, CurrentRunningVMs, CurrentInUseVMs, ActionTaken, VMsPoweredOn, VMsPoweredOff, NewTotalVMs, Outcome, Notes, + PhaseName, ScheduleID, MinVMs, MaxVMs, ServiceableVMs, DrainingVMs) + VALUES (@Now, @PoweredOn, @InUse, @ActionTaken, @ActualOn, @ActualOff, @PoweredOn + @ActualOn - @ActualOff, @Outcome, @Notes, + @PhaseName, @ScheduleID, @MinVMs, @MaxVMs, @Serviceable, @Draining); + + SET @ActivityID = SCOPE_IDENTITY(); + UPDATE @Actions SET ActivityID = @ActivityID; + -- VmScalingRules.LastChecked is deliberately not updated: the table is system-versioned, + -- so a write on every run would add a history row every few minutes and bury real rule + -- edits. VmScalingActivityLog.CheckTimestamp already records each run. + IF @StartedTransaction = 1 COMMIT TRANSACTION; + + SELECT ActionType, VMName, VMID, StopMode, PreviousNetworkStatus, ActivityID + FROM @Actions + ORDER BY CASE WHEN ActionType = 'PowerOn' THEN 0 ELSE 1 END, VMID; +END +GO diff --git a/sql_queries/154_alter_procedure-GetCheckoutStats.sql b/sql_queries/154_alter_procedure-GetCheckoutStats.sql new file mode 100644 index 0000000..c22a3a3 --- /dev/null +++ b/sql_queries/154_alter_procedure-GetCheckoutStats.sql @@ -0,0 +1,96 @@ +-- Redefines dbo.GetCheckoutStats (120) for start on demand (4.1). A user who waits for a host to +-- start records a Starting event each time their AVD host asks, then the outcome that ended +-- the wait, so: +-- * Total and the outcome counts leave Starting events out: they count checkouts that ended, +-- as before, and the API's denied share (NoneAvailable / Total) keeps its meaning. +-- * Starting counts the times an AVD host was told to wait. +-- * A wait runs from a user's first Starting event to the next event of theirs that is not +-- Starting. Waits counts the waits that began in the period, WaitsServed those that ended in +-- a checkout (Assigned or Reused) within 20 minutes, and WaitP50Seconds and WaitP95Seconds +-- how long those took. Events up to 20 minutes either side of the period are read, so a wait +-- that crosses its edges is still measured. +-- * WaitingNow counts the users waiting now (dbo.fnWaitingCheckoutUsers). +-- Everything else is unchanged: the median and 95th percentile time of successful checkouts, +-- denials in the last hour, and the median and 95th percentile host start-to-ready time. + +CREATE PROCEDURE [dbo].[GetCheckoutStats] + @FromUtc DATETIME2(0), + @ToUtc DATETIME2(0) = NULL +AS +BEGIN + SET NOCOUNT ON; + + -- Without an end, everything up to now: a second ahead, so nothing just recorded is missed. + DECLARE @To DATETIME2(0) = COALESCE(@ToUtc, DATEADD(SECOND, 1, SYSUTCDATETIME())); + DECLARE @From DATETIME2(0) = COALESCE(@FromUtc, DATEADD(HOUR, -24, @To)); + DECLARE @P50Ms INT, @P95Ms INT, @StartP50 INT, @StartP95 INT, @WaitP50 INT, @WaitP95 INT; + DECLARE @Waits TABLE (FirstStartingAt DATETIME2(3) NOT NULL, EndedAt DATETIME2(3) NULL, EndOutcome VARCHAR(24) NULL); + + SELECT TOP 1 + @P50Ms = CAST(ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY DurationMs) OVER (), 0) AS INT), + @P95Ms = CAST(ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY DurationMs) OVER (), 0) AS INT) + FROM dbo.CheckoutEvents + WHERE OccurredAt >= @From AND OccurredAt < @To + AND Outcome IN ('Assigned', 'Reused') + AND DurationMs IS NOT NULL; + + SELECT TOP 1 + @StartP50 = CAST(ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY Seconds) OVER (), 0) AS INT), + @StartP95 = CAST(ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY Seconds) OVER (), 0) AS INT) + FROM dbo.HostStartEvents + WHERE ReadyAt >= @From AND ReadyAt < @To; + + -- Each of a user's events belongs to the wait that the next event other than Starting ends: + -- the number of such events before it identifies the wait. + INSERT INTO @Waits (FirstStartingAt, EndedAt, EndOutcome) + SELECT MIN(CASE WHEN e.Outcome = 'Starting' THEN e.OccurredAt END), + MAX(CASE WHEN e.Outcome <> 'Starting' THEN e.OccurredAt END), + MAX(CASE WHEN e.Outcome <> 'Starting' THEN e.Outcome END) + FROM ( + SELECT Username, Outcome, OccurredAt, + COALESCE(SUM(CASE WHEN Outcome = 'Starting' THEN 0 ELSE 1 END) + OVER (PARTITION BY Username ORDER BY OccurredAt, EventID ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0) AS WaitNumber + FROM dbo.CheckoutEvents + WHERE Username IS NOT NULL + AND OccurredAt >= DATEADD(MINUTE, -20, @From) + AND OccurredAt < DATEADD(MINUTE, 20, @To) + ) e + GROUP BY e.Username, e.WaitNumber + HAVING SUM(CASE WHEN e.Outcome = 'Starting' THEN 1 ELSE 0 END) > 0; + + DELETE FROM @Waits WHERE FirstStartingAt < @From OR FirstStartingAt >= @To; + + SELECT TOP 1 + @WaitP50 = CAST(ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY DATEDIFF(SECOND, FirstStartingAt, EndedAt)) OVER (), 0) AS INT), + @WaitP95 = CAST(ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY DATEDIFF(SECOND, FirstStartingAt, EndedAt)) OVER (), 0) AS INT) + FROM @Waits + WHERE EndOutcome IN ('Assigned', 'Reused') + AND EndedAt <= DATEADD(MINUTE, 20, FirstStartingAt); + + SELECT + COALESCE(SUM(CASE WHEN Outcome <> 'Starting' THEN 1 ELSE 0 END), 0) AS Total, + COALESCE(SUM(CASE WHEN Outcome = 'Assigned' THEN 1 ELSE 0 END), 0) AS Assigned, + COALESCE(SUM(CASE WHEN Outcome = 'Reused' THEN 1 ELSE 0 END), 0) AS Reused, + COALESCE(SUM(CASE WHEN Outcome = 'NoneAvailable' THEN 1 ELSE 0 END), 0) AS NoneAvailable, + COALESCE(SUM(CASE WHEN Outcome = 'ProvisionFailed' THEN 1 ELSE 0 END), 0) AS ProvisionFailed, + COALESCE(SUM(CASE WHEN Outcome = 'Error' THEN 1 ELSE 0 END), 0) AS Errors, + @P50Ms AS P50Ms, + @P95Ms AS P95Ms, + (SELECT COUNT(*) FROM dbo.CheckoutEvents + WHERE Outcome = 'NoneAvailable' AND OccurredAt >= DATEADD(HOUR, -1, SYSUTCDATETIME())) AS DeniedLastHour, + (SELECT CONVERT(VARCHAR(33), MAX(OccurredAt), 126) + 'Z' FROM dbo.CheckoutEvents + WHERE Outcome = 'NoneAvailable' AND OccurredAt >= @From AND OccurredAt < @To) AS LastDeniedUtc, + (SELECT COUNT(*) FROM dbo.HostStartEvents WHERE ReadyAt >= @From AND ReadyAt < @To) AS HostStarts, + @StartP50 AS StartP50Seconds, + @StartP95 AS StartP95Seconds, + COALESCE(SUM(CASE WHEN Outcome = 'Starting' THEN 1 ELSE 0 END), 0) AS Starting, + (SELECT COUNT(*) FROM @Waits) AS Waits, + (SELECT COUNT(*) FROM @Waits + WHERE EndOutcome IN ('Assigned', 'Reused') AND EndedAt <= DATEADD(MINUTE, 20, FirstStartingAt)) AS WaitsServed, + @WaitP50 AS WaitP50Seconds, + @WaitP95 AS WaitP95Seconds, + (SELECT COUNT(*) FROM dbo.fnWaitingCheckoutUsers()) AS WaitingNow + FROM dbo.CheckoutEvents + WHERE OccurredAt >= @From AND OccurredAt < @To; +END +GO diff --git a/sql_queries/155_alter_procedure-GetUtilizationSeries.sql b/sql_queries/155_alter_procedure-GetUtilizationSeries.sql new file mode 100644 index 0000000..11df091 --- /dev/null +++ b/sql_queries/155_alter_procedure-GetUtilizationSeries.sql @@ -0,0 +1,100 @@ +-- Redefines dbo.GetUtilizationSeries (119) for start on demand (4.1): +-- * Runs and the averages come from scaling runs only, not the 'Start On Demand' rows that +-- start on demand adds to the activity log. +-- * Checkouts leaves out Starting events, which record a user told to wait rather than a +-- checkout that ended, and the new Waited column counts the users told to wait in each bucket. +-- +-- Capacity over time for the dashboard's charts: one row per bucket of @BucketMinutes from +-- @FromUtc (rounded down to the minute) up to @ToUtc, empty buckets included. +-- * From the scaling runs in each bucket: the average powered-on, in-use and serviceable +-- counts, the peak in use, and the phase's minimum and maximum. The averages keep the lines +-- comparable: in use never averages above serviceable unless it was above it at some point. +-- * From the checkout events: how many checkouts there were, how many found no host, how many +-- failed in the broker, and how many users waited for a host to start. +-- Times are UTC; the activity log's database-local CheckTimestamp is converted with the +-- database's current offset (zero on Azure SQL). At most 2,000 buckets. + +CREATE PROCEDURE [dbo].[GetUtilizationSeries] + @FromUtc DATETIME2(0), + @ToUtc DATETIME2(0), + @BucketMinutes INT = 60 +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @Bucket INT = CASE WHEN @BucketMinutes IS NULL OR @BucketMinutes < 5 THEN 5 + WHEN @BucketMinutes > 1440 THEN 1440 ELSE @BucketMinutes END; + DECLARE @BucketSeconds INT = @Bucket * 60; + -- Without an end, everything up to now: a second ahead, so nothing just recorded is missed. + DECLARE @To DATETIME2(0) = COALESCE(@ToUtc, DATEADD(SECOND, 1, SYSUTCDATETIME())); + DECLARE @From DATETIME2(0) = COALESCE(@FromUtc, DATEADD(HOUR, -24, @To)); + SET @From = DATEADD(MINUTE, DATEDIFF(MINUTE, CAST('20000101' AS DATETIME2(0)), @From), CAST('20000101' AS DATETIME2(0))); + DECLARE @Span INT = DATEDIFF(SECOND, @From, @To); + DECLARE @Buckets INT = @Span / @BucketSeconds + CASE WHEN @Span % @BucketSeconds = 0 THEN 0 ELSE 1 END; + DECLARE @OffsetMinutes INT = DATEDIFF(MINUTE, GETDATE(), GETUTCDATE()); + + IF @Span <= 0 OR @Buckets > 2000 + BEGIN + SELECT CAST(NULL AS VARCHAR(33)) AS BucketStartUtc, 0 AS Runs, + CAST(NULL AS DECIMAL(9,1)) AS PoweredOn, CAST(NULL AS DECIMAL(9,1)) AS InUse, + CAST(NULL AS DECIMAL(9,1)) AS Serviceable, CAST(NULL AS INT) AS PeakInUse, + CAST(NULL AS INT) AS MinVMs, CAST(NULL AS INT) AS MaxVMs, + 0 AS Checkouts, 0 AS Denied, 0 AS Failed, 0 AS Waited + WHERE 1 = 0; + RETURN; + END + + ;WITH Tally AS ( + SELECT TOP (@Buckets) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS BucketIndex + FROM sys.all_columns a CROSS JOIN sys.all_columns b + ), + Runs AS ( + SELECT + DATEDIFF(SECOND, @From, DATEADD(MINUTE, @OffsetMinutes, CAST(CheckTimestamp AS DATETIME2(0)))) / @BucketSeconds AS BucketIndex, + CurrentRunningVMs, CurrentInUseVMs, ServiceableVMs, MinVMs, MaxVMs + FROM dbo.VmScalingActivityLog + WHERE CheckTimestamp >= DATEADD(MINUTE, -@OffsetMinutes, @From) + AND CheckTimestamp < DATEADD(MINUTE, -@OffsetMinutes, @To) + AND (ActionTaken IS NULL OR ActionTaken <> N'Start On Demand') + ), + RunBuckets AS ( + SELECT BucketIndex, + COUNT(*) AS Runs, + CAST(AVG(CAST(CurrentRunningVMs AS DECIMAL(9,2))) AS DECIMAL(9,1)) AS PoweredOn, + CAST(AVG(CAST(CurrentInUseVMs AS DECIMAL(9,2))) AS DECIMAL(9,1)) AS InUse, + CAST(AVG(CAST(ServiceableVMs AS DECIMAL(9,2))) AS DECIMAL(9,1)) AS Serviceable, + MAX(CurrentInUseVMs) AS PeakInUse, + MAX(MinVMs) AS MinVMs, + MAX(MaxVMs) AS MaxVMs + FROM Runs + GROUP BY BucketIndex + ), + CheckoutBuckets AS ( + SELECT DATEDIFF(SECOND, @From, OccurredAt) / @BucketSeconds AS BucketIndex, + SUM(CASE WHEN Outcome <> 'Starting' THEN 1 ELSE 0 END) AS Checkouts, + SUM(CASE WHEN Outcome = 'NoneAvailable' THEN 1 ELSE 0 END) AS Denied, + SUM(CASE WHEN Outcome IN ('ProvisionFailed', 'Error') THEN 1 ELSE 0 END) AS Failed, + COUNT(DISTINCT CASE WHEN Outcome = 'Starting' THEN Username END) AS Waited + FROM dbo.CheckoutEvents + WHERE OccurredAt >= @From AND OccurredAt < @To + GROUP BY DATEDIFF(SECOND, @From, OccurredAt) / @BucketSeconds + ) + SELECT + CONVERT(VARCHAR(33), DATEADD(MINUTE, t.BucketIndex * @Bucket, @From), 126) + 'Z' AS BucketStartUtc, + COALESCE(r.Runs, 0) AS Runs, + r.PoweredOn, + r.InUse, + r.Serviceable, + r.PeakInUse, + r.MinVMs, + r.MaxVMs, + COALESCE(c.Checkouts, 0) AS Checkouts, + COALESCE(c.Denied, 0) AS Denied, + COALESCE(c.Failed, 0) AS Failed, + COALESCE(c.Waited, 0) AS Waited + FROM Tally t + LEFT JOIN RunBuckets r ON r.BucketIndex = t.BucketIndex + LEFT JOIN CheckoutBuckets c ON c.BucketIndex = t.BucketIndex + ORDER BY t.BucketIndex; +END +GO diff --git a/sql_queries/156_alter_procedure-GetAttentionItems.sql b/sql_queries/156_alter_procedure-GetAttentionItems.sql new file mode 100644 index 0000000..21a3fe0 --- /dev/null +++ b/sql_queries/156_alter_procedure-GetAttentionItems.sql @@ -0,0 +1,128 @@ +-- Redefines dbo.GetAttentionItems (121) for scale to zero (4.1). With start on demand on and a +-- phase whose MinVMs is 0, having no ready host is the intended idle state, so no-ready-hosts +-- is raised only while a user is waiting for one. Everything else is unchanged. +-- +-- What needs an operator now, for the dashboard's Attention panel. One row per item: +-- no-ready-hosts hosts are registered but none can take a new user and none is starting, +-- while scaling should keep one (MinVMs is at least one, or start on demand +-- is off, or a user is waiting). The pool is exhausted or its hosts are +-- failing. +-- denied-checkouts checkouts that found no host in the last @DeniedMinutes (ItemCount). +-- unreachable powered on, not in maintenance, not just starting, and not reachable for +-- at least @UnreachableMinutes. The time comes from the temporal history: +-- the later of the end of the host's last reachable version and its power-on. +-- cleanup-stuck on and reachable, but the previous user has not been removed for at least +-- @CleanupMinutes, measured from the end of its last version without a +-- pending cleanup. Cleanup is retried every two minutes, so this means +-- several attempts failed. +-- never-connected checked out at least @NotConnectedMinutes ago, not checked out again +-- since, and the host's current heartbeat does not report the user. +-- AgeSeconds is how long the condition has lasted. The history is searched for the last seven +-- days only; a condition older than that reports seven days or its power-on time. + +CREATE PROCEDURE [dbo].[GetAttentionItems] + @UnreachableMinutes INT = 10, + @CleanupMinutes INT = 15, + @NotConnectedMinutes INT = 30, + @DeniedMinutes INT = 60 +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @NowUtc DATETIME2(0) = CAST(SYSUTCDATETIME() AS DATETIME2(0)); + DECLARE @HistoryFloor DATETIME2(0) = DATEADD(DAY, -7, @NowUtc); + DECLARE @Now DATETIME = GETDATE(); + DECLARE @OffsetMinutes INT = DATEDIFF(MINUTE, GETDATE(), GETUTCDATE()); + DECLARE @Boot INT = 10; + DECLARE @ReconcileIntervalSeconds INT, @FreshSeconds INT; + DECLARE @Items TABLE ( + Kind VARCHAR(32) NOT NULL, VMID INT NULL, Hostname VARCHAR(255) NULL, Username VARCHAR(255) NULL, + AgeSeconds INT NULL, ItemCount INT NULL + ); + + SELECT TOP 1 @ReconcileIntervalSeconds = ReconcileIntervalSeconds + FROM dbo.LinuxHostSettings WHERE SettingsScope = 'Global' ORDER BY SettingsID; + -- The API's heartbeat_stale_after: three reconcile intervals, never under three minutes. + SET @FreshSeconds = CASE WHEN 3 * COALESCE(@ReconcileIntervalSeconds, 60) > 180 THEN 3 * COALESCE(@ReconcileIntervalSeconds, 60) ELSE 180 END; + + IF EXISTS (SELECT 1 FROM dbo.VirtualMachines) + AND NOT ( + EXISTS (SELECT 1 FROM dbo.ScalingPolicy WHERE PolicyID = 1 AND StartOnDemandEnabled = 1) + AND EXISTS (SELECT 1 FROM dbo.fnActiveScalingPhase(NULL) WHERE MinVMs = 0) + AND NOT EXISTS (SELECT 1 FROM dbo.fnWaitingCheckoutUsers()) + ) + AND NOT EXISTS ( + SELECT 1 FROM dbo.VirtualMachines + WHERE VmStatus = 'Available' AND PowerState = 'On' AND NetworkStatus = 'Reachable' + AND CleanupPending = 0 AND DrainRequested = 0 AND Username IS NULL AND LeaseId IS NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM dbo.VirtualMachines + WHERE PowerState = 'On' AND NetworkStatus = 'Unreachable' AND VmStatus <> 'Maintenance' + AND DrainRequested = 0 AND PowerStateChangedDate >= DATEADD(MINUTE, -@Boot, @Now) + ) + BEGIN + INSERT INTO @Items (Kind) VALUES ('no-ready-hosts'); + END + + INSERT INTO @Items (Kind, ItemCount, AgeSeconds) + SELECT 'denied-checkouts', COUNT(*), DATEDIFF(SECOND, MAX(OccurredAt), SYSUTCDATETIME()) + FROM dbo.CheckoutEvents + WHERE Outcome = 'NoneAvailable' + AND OccurredAt >= DATEADD(MINUTE, -@DeniedMinutes, SYSUTCDATETIME()) + HAVING COUNT(*) > 0; + + INSERT INTO @Items (Kind, VMID, Hostname, AgeSeconds) + SELECT 'unreachable', vm.VMID, vm.Hostname, DATEDIFF(SECOND, since.SinceUtc, @NowUtc) + FROM dbo.VirtualMachines vm + OUTER APPLY ( + SELECT MAX(h.SysEndTime) AS LastReachableEnd + FROM dbo.VirtualMachinesHistory h + WHERE h.VMID = vm.VMID AND h.NetworkStatus = 'Reachable' AND h.SysEndTime >= @HistoryFloor + ) lr + CROSS APPLY ( + SELECT DATEADD(MINUTE, @OffsetMinutes, CAST(COALESCE(vm.PowerStateChangedDate, vm.CreateDate, @Now) AS DATETIME2(0))) AS PoweredOnUtc + ) po + CROSS APPLY ( + SELECT CASE WHEN lr.LastReachableEnd > po.PoweredOnUtc THEN CAST(lr.LastReachableEnd AS DATETIME2(0)) ELSE po.PoweredOnUtc END AS SinceUtc + ) since + WHERE vm.PowerState = 'On' + AND vm.NetworkStatus = 'Unreachable' + AND vm.VmStatus <> 'Maintenance' + AND (vm.PowerStateChangedDate IS NULL OR vm.PowerStateChangedDate < DATEADD(MINUTE, -@Boot, @Now)) + AND DATEDIFF(SECOND, since.SinceUtc, @NowUtc) >= @UnreachableMinutes * 60; + + INSERT INTO @Items (Kind, VMID, Hostname, Username, AgeSeconds) + SELECT 'cleanup-stuck', vm.VMID, vm.Hostname, vm.CleanupUsername, + DATEDIFF(SECOND, COALESCE(CAST(np.LastNotPendingEnd AS DATETIME2(0)), @HistoryFloor), @NowUtc) + FROM dbo.VirtualMachines vm + OUTER APPLY ( + SELECT MAX(h.SysEndTime) AS LastNotPendingEnd + FROM dbo.VirtualMachinesHistory h + WHERE h.VMID = vm.VMID AND h.CleanupPending = 0 AND h.SysEndTime >= @HistoryFloor + ) np + WHERE vm.CleanupPending = 1 + AND vm.PowerState = 'On' + AND vm.NetworkStatus = 'Reachable' + AND DATEDIFF(SECOND, COALESCE(CAST(np.LastNotPendingEnd AS DATETIME2(0)), @HistoryFloor), @NowUtc) >= @CleanupMinutes * 60; + + INSERT INTO @Items (Kind, VMID, Hostname, Username, AgeSeconds) + SELECT 'never-connected', vm.VMID, vm.Hostname, vm.Username, DATEDIFF(SECOND, vm.AssignedDate, @Now) + FROM dbo.VirtualMachines vm + INNER JOIN dbo.HostHeartbeats hb ON hb.Hostname = vm.Hostname + WHERE vm.VmStatus = 'CheckedOut' + AND vm.AssignedDate < DATEADD(MINUTE, -@NotConnectedMinutes, @Now) + AND vm.LastCheckoutDate < DATEADD(MINUTE, -@NotConnectedMinutes, @Now) + AND hb.ReceivedAt >= DATEADD(SECOND, -@FreshSeconds, SYSUTCDATETIME()) + AND NOT EXISTS ( + SELECT 1 FROM OPENJSON(hb.SessionsJson) WITH (Username VARCHAR(64) '$.username') s + WHERE s.Username = vm.Username + ); + + SELECT Kind, VMID, Hostname, Username, AgeSeconds, ItemCount + FROM @Items + ORDER BY CASE Kind WHEN 'no-ready-hosts' THEN 0 WHEN 'denied-checkouts' THEN 1 WHEN 'unreachable' THEN 2 + WHEN 'cleanup-stuck' THEN 3 ELSE 4 END, + AgeSeconds DESC, Hostname; +END +GO diff --git a/sql_queries/README.md b/sql_queries/README.md index 2b666ec..40e9b90 100644 --- a/sql_queries/README.md +++ b/sql_queries/README.md @@ -67,6 +67,9 @@ Use that path when you need to: - `116_create_table-host_start_events.sql`: creates `dbo.HostStartEvents`, how long each start took to become reachable - `124_create_table-maintenance_runs.sql`: creates `dbo.MaintenanceRuns` - `125_create_table-maintenance_run_hosts.sql`: creates `dbo.MaintenanceRunHosts`, each host's progress through a run +- `144_add_start_on_demand_to_scaling_policy.sql`: adds `StartOnDemandEnabled` (on by default) and `MaxPendingStarts` (1–20, default 2) to `dbo.ScalingPolicy` +- `145_add_starting_outcome_to_checkout_events.sql`: lets `dbo.CheckoutEvents` record `Starting`, a request answered while a host starts for the user, and adds `ClientVersion`, the version the AVD host's broker script reports +- `147_allow_zero_minimum_hosts.sql`: lets the default rule and schedule windows keep a minimum of 0 hosts; a window's maximum must still be above its minimum The table scripts above are written to be rerunnable. @@ -187,6 +190,16 @@ The scripts do not contain `USE ` statements. The target database come - `140_alter_procedure-TriggerScalingLogic.sql`: keeps one more host serviceable while a run waits for a spare ready host, never past `MaxVMs` - `141_create_procedure-GetVmsPaged.sql`, `142_create_procedure-GetVmStatusCounts.sql`: the host list's page, filtered, searched and sorted, and its status counts - `143_create_procedure-ImportLinuxHostVm.sql`: registers a host found in Azure as `Unreachable` with its Azure power state, or reports `Exists` +- `146_alter_procedure-RecordCheckoutEvent.sql`: accepts the `Starting` outcome and `@ClientVersion` +- `148_create_function-fnWaitingCheckoutUsers.sql`: the users waiting for a host to start: those whose latest checkout event in the last five minutes is `Starting` +- `149_alter_procedure-GetScalingPolicy.sql`: adds `StartOnDemandEnabled`, `MaxPendingStarts`, `ZeroMinimumCount` (the default rule and enabled windows with a minimum of 0), and the AVD hosts seen in the last week whose broker script cannot wait for a host to start +- `150_create_procedure-SetScalingPolicyStartOnDemand.sql`: turns start on demand on or off and sets the pending-start limit (`Updated`, `Unchanged`, `Invalid`) +- `151_alter_procedure-SaveScalingSchedule.sql`: accepts a minimum of 0 only while start on demand is on +- `152_create_procedure-ReserveVmForStart.sql`: when a checkout finds no ready host, picks a stopped host to start for the user under the scaling application lock, or says why not (`Started`, `AlreadyStarting`, `ReadyNow`, `AtMaximum`, `NoCandidate`, `Busy`, `Disabled`), with how long to wait before asking again +- `153_alter_procedure-TriggerScalingLogic.sql`: counts waiting users as demand, leaves a host for each of them when scaling down, and reads a minimum of 0 as 1 while start on demand is off +- `154_alter_procedure-GetCheckoutStats.sql`: leaves `Starting` out of the total and adds the waits for a host to start, how many were served, their median and 95th percentile, and who is waiting now +- `155_alter_procedure-GetUtilizationSeries.sql`: counts only scaling runs, leaves `Starting` out of checkouts, and adds `Waited`, the users who waited in each bucket +- `156_alter_procedure-GetAttentionItems.sql`: does not report no ready hosts for a pool scaled to zero that nobody is waiting on `033` exists as its own file rather than being folded into `014` because `014` runs before `029` adds those columns, and SQL Server validates column references against existing tables when a procedure is created. @@ -227,6 +240,8 @@ Two current behaviors are worth calling out: - `dbo.CheckoutEvents` and `dbo.HostStartEvents` are kept for the API's `CHECKOUT_EVENT_RETENTION_DAYS` and purged with the audit log. - Only one maintenance run is active, paused or stopping at a time. Admission takes the scaling application lock, so neither admission nor scaling can take ready capacity below the minimum while the other acts, and a run waiting for a spare ready host makes scaling keep one more host on. `dbo.SetMaintenanceHostState` is a compare-and-set, so overlapping advances cannot both act on a step. - `dbo.GetVmsPaged` and `dbo.GetVmStatusCounts` apply the same status tests as `dbo.CheckoutVm`, so a host the list calls ready is one a checkout could take. Imported hosts start `Unreachable`. +- Start on demand: when `dbo.CheckoutVm` finds no ready host, `dbo.ReserveVmForStart` takes the scaling application lock and marks one stopped host `On` for the waiting user, the way scaling would, logging it as `Start On Demand`. It starts at most one host per waiting user and at most `MaxPendingStarts` at once, never past the active phase's `MaxVMs`. The API commits at once to release the lock, then asks Azure to start the host. Two first requests close together can see one start where two were needed; the second user's next request starts another host, so a race can only delay a start, never add one. +- A minimum of 0 is allowed only while start on demand is on. If start on demand is turned off with a minimum of 0 saved, `dbo.TriggerScalingLogic` reads it as 1 and notes it on the run, and `dbo.GetScalingPolicy` reports how many such minimums remain in `ZeroMinimumCount`. Linux host settings are a single fleet-wide profile: @@ -416,7 +431,9 @@ WHERE name IN ( 'GetMaintenanceAttention', 'GetVmsPaged', 'GetVmStatusCounts', - 'ImportLinuxHostVm' + 'ImportLinuxHostVm', + 'SetScalingPolicyStartOnDemand', + 'ReserveVmForStart' ) ORDER BY name; ``` @@ -427,7 +444,7 @@ ORDER BY name; SELECT name FROM sys.objects WHERE type IN ('FN', 'IF', 'TF') - AND name IN ('fnScheduleWeekIntervals', 'fnActiveScalingPhase', 'fnMaintenanceRunSummary') + AND name IN ('fnScheduleWeekIntervals', 'fnActiveScalingPhase', 'fnMaintenanceRunSummary', 'fnWaitingCheckoutUsers') ORDER BY name; ``` diff --git a/sql_queries/tests/conftest.py b/sql_queries/tests/conftest.py index 4bf737a..b488f8d 100644 --- a/sql_queries/tests/conftest.py +++ b/sql_queries/tests/conftest.py @@ -103,7 +103,7 @@ def clean_database(conn): "DELETE FROM dbo.VmScalingActivityLog", "DELETE FROM dbo.VmScalingRules", "DELETE FROM dbo.ScalingSchedules", - "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL", + "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL, StartOnDemandEnabled=1, MaxPendingStarts=2", "UPDATE dbo.LinuxHostSettings SET GracePeriodSeconds=1200, ReconcileIntervalSeconds=60, WatcherDebounceSeconds=10, WatcherSettleSeconds=2, IdleTimeoutSeconds=0, IdleWarningSeconds=120, ScreenLockEnabled=0, DisableLockScreen=1, ScreenIdleDelaySeconds=0, ScreenLockDelaySeconds=0, ScreenLockSettingsLocked=1, PreserveSessionsOnDisconnect=0, UpdatedBy=NULL WHERE SettingsScope='Global'", "INSERT INTO dbo.VmScalingRules (MinVMs, MaxVMs, ScaleUpRatio, ScaleUpIncrement, ScaleDownRatio, ScaleDownIncrement, StopMode) VALUES (2, 10, 70.00, 2, 30.00, 1, NULL)", ]: diff --git a/sql_queries/tests/test_scaling_policy_contract.py b/sql_queries/tests/test_scaling_policy_contract.py index bcbe6f7..eb3d304 100644 --- a/sql_queries/tests/test_scaling_policy_contract.py +++ b/sql_queries/tests/test_scaling_policy_contract.py @@ -148,7 +148,8 @@ def test_saving_validates_like_the_api(conn): ({"name": " "}, "name is required."), ({"days": 0}, "Choose at least one day."), ({"start": "08:00", "end": "08:00"}, "start and end must differ."), - ({"MinVMs": 0}, "minvms must be at least 1."), + ({"MinVMs": -1}, "minvms must be at least 0."), + ({"MinVMs": 0, "MaxVMs": 0}, "maxvms must be greater than minvms."), ({"MinVMs": 5, "MaxVMs": 5}, "maxvms must be greater than minvms."), ({"ScaleUpRatio": 101}, "scaleupratio must be between 0 and 100."), ({"ScaleUpRatio": 20, "ScaleDownRatio": 20}, "scaleupratio must be greater than scaledownratio."), diff --git a/sql_queries/tests/test_sql_contract.py b/sql_queries/tests/test_sql_contract.py index be43274..941c59b 100644 --- a/sql_queries/tests/test_sql_contract.py +++ b/sql_queries/tests/test_sql_contract.py @@ -144,7 +144,7 @@ def test_scaling_rules_constraints_history_and_create_lock(conn, second_conn): assert "StopMode" in exec_sql(conn, "EXEC dbo.GetVmScalingRulesHistoryPaged")[0] for sql in [ - "UPDATE dbo.VmScalingRules SET MinVMs=0", + "UPDATE dbo.VmScalingRules SET MinVMs=-1", "UPDATE dbo.VmScalingRules SET ScaleUpIncrement=0", "UPDATE dbo.VmScalingRules SET ScaleUpRatio=101", "UPDATE dbo.VmScalingRules SET StopMode='Stop'", diff --git a/sql_queries/tests/test_start_on_demand_contract.py b/sql_queries/tests/test_start_on_demand_contract.py new file mode 100644 index 0000000..b87d570 --- /dev/null +++ b/sql_queries/tests/test_start_on_demand_contract.py @@ -0,0 +1,446 @@ +"""Contract tests for 4.1 start on demand and scale to zero (144-156): the policy settings, the +Starting outcome, waiting users, dbo.ReserveVmForStart, scaling with waiting users and a minimum +of 0, and the dashboard's wait figures.""" + +import json +from datetime import datetime, timedelta, timezone + +from conftest import add_vm, exec_sql, one, rows + + +def set_rule(conn, **values): + rule = {"MinVMs": 0, "MaxVMs": 5, "ScaleUpRatio": 70, "ScaleUpIncrement": 1, "ScaleDownRatio": 30, + "ScaleDownIncrement": 1, "StopMode": "PowerOff"} + rule.update(values) + exec_sql(conn, "DELETE FROM dbo.VmScalingRules") + exec_sql(conn, "INSERT INTO dbo.VmScalingRules (MinVMs,MaxVMs,ScaleUpRatio,ScaleUpIncrement,ScaleDownRatio,ScaleDownIncrement,StopMode) " + "VALUES (%s,%s,%s,%s,%s,%s,%s)", tuple(rule[k] for k in ("MinVMs", "MaxVMs", "ScaleUpRatio", "ScaleUpIncrement", + "ScaleDownRatio", "ScaleDownIncrement", "StopMode"))) + + +def set_start_on_demand(conn, enabled=None, max_pending=None, updated_by="admin@contoso.com"): + return exec_sql(conn, "EXEC dbo.SetScalingPolicyStartOnDemand @Enabled=%s, @MaxPendingStarts=%s, @UpdatedBy=%s", + (enabled, max_pending, updated_by))[0] + + +def reserve(conn, username="alice", avdhost="avd-01"): + return exec_sql(conn, "EXEC dbo.ReserveVmForStart @Username=%s, @AvdHost=%s", (username, avdhost))[0] + + +def record(conn, outcome, username="alice", seconds_ago=None, avdhost="avd-01", client_version=None): + event_id = exec_sql(conn, "EXEC dbo.RecordCheckoutEvent @Username=%s, @AvdHost=%s, @Outcome=%s, @DurationMs=%s, @Hostname=%s, " + "@ClientVersion=%s", (username, avdhost, outcome, 100, None, client_version))[0]["EventID"] + if seconds_ago is not None: + exec_sql(conn, "UPDATE dbo.CheckoutEvents SET OccurredAt=DATEADD(SECOND, -%s, SYSUTCDATETIME()) WHERE EventID=%s", + (seconds_ago, event_id)) + return event_id + + +def waiting(conn): + return sorted(r["Username"] for r in rows(conn, "SELECT Username FROM dbo.fnWaitingCheckoutUsers()")) + + +def booting(conn, hostname, seconds_ago=0, cleanup=False): + vmid = add_vm(conn, hostname, net="Unreachable", cleanup=cleanup) + exec_sql(conn, "UPDATE dbo.VirtualMachines SET PowerStateChangedDate=DATEADD(SECOND, -%s, GETDATE()) WHERE VMID=%s", + (seconds_ago, vmid)) + return vmid + + +def vm(conn, vmid): + return one(conn, "SELECT Hostname, PowerState, NetworkStatus, VmStatus, StartRequestedAt, " + "DATEDIFF(SECOND, PowerStateChangedDate, GETDATE()) AS ChangedSecondsAgo FROM dbo.VirtualMachines WHERE VMID=%s", (vmid,)) + + +def dry_run(conn): + return exec_sql(conn, "EXEC dbo.TriggerScalingLogic @DryRun=1")[0] + + +def start_times(conn, *seconds): + for value in seconds: + exec_sql(conn, "INSERT INTO dbo.HostStartEvents (Hostname, RequestedAt, ReadyAt, Seconds) " + "VALUES ('hist', SYSUTCDATETIME(), SYSUTCDATETIME(), %s)", (value,)) + + +# ----------------------------------------------------------------------- policy and constraints + + +def test_start_on_demand_is_on_by_default_and_can_be_changed(conn): + policy = exec_sql(conn, "EXEC dbo.GetScalingPolicy")[0] + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["ZeroMinimumCount"]) == (True, 2, 0) + + assert set_start_on_demand(conn)["Result"] == "Invalid" + invalid = set_start_on_demand(conn, max_pending=21) + assert (invalid["Result"], invalid["Message"]) == ("Invalid", "maxpendingstarts must be between 1 and 20.") + assert set_start_on_demand(conn, max_pending=0)["Result"] == "Invalid" + + updated = set_start_on_demand(conn, enabled=False, max_pending=4) + assert updated == { + "Result": "Updated", "Message": None, "StartOnDemandEnabled": False, "MaxPendingStarts": 4, + "PreviousStartOnDemandEnabled": True, "PreviousMaxPendingStarts": 2, "ZeroMinimumCount": 0, + } + assert set_start_on_demand(conn, enabled=False)["Result"] == "Unchanged" + # A NULL keeps the other value. + assert set_start_on_demand(conn, enabled=True)["MaxPendingStarts"] == 4 + + policy = exec_sql(conn, "EXEC dbo.GetScalingPolicy")[0] + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["UpdatedBy"]) == (True, 4, "admin@contoso.com") + + for sql in ("UPDATE dbo.ScalingPolicy SET MaxPendingStarts=0", "UPDATE dbo.ScalingPolicy SET MaxPendingStarts=21"): + try: + exec_sql(conn, sql) + except Exception: + conn.rollback() + else: + raise AssertionError(f"{sql} should be refused") + + +def test_a_minimum_of_zero_needs_start_on_demand(conn): + set_rule(conn, MinVMs=0) + save = ("EXEC dbo.SaveScalingSchedule @ScheduleID=NULL, @Name=%s, @Enabled=%s, @DaysOfWeek=31, @StartTime='08:00', " + "@EndTime='18:00', @MinVMs=%s, @MaxVMs=4, @ScaleUpRatio=60, @ScaleUpIncrement=1, @ScaleDownRatio=20, " + "@ScaleDownIncrement=1, @StopMode=NULL, @UpdatedBy=NULL") + assert exec_sql(conn, save, ("Quiet days", True, 0))[0]["Result"] == "Created" + exec_sql(conn, "UPDATE dbo.ScalingSchedules SET Enabled=0") + assert exec_sql(conn, save, ("Busy days", True, 1))[0]["Result"] == "Created" + exec_sql(conn, "UPDATE dbo.ScalingSchedules SET Enabled=1, MinVMs=0 WHERE Name='Busy days'") + + # The rule and one enabled window have a minimum of 0; the disabled window does not count. + assert exec_sql(conn, "EXEC dbo.GetScalingPolicy")[0]["ZeroMinimumCount"] == 2 + turned_off = set_start_on_demand(conn, enabled=False) + assert (turned_off["Result"], turned_off["ZeroMinimumCount"]) == ("Updated", 2) + + refused = exec_sql(conn, save, ("Night", False, 0))[0] + assert (refused["Result"], refused["Message"]) == ("Invalid", "minvms can be 0 only while start on demand is on.") + + for sql in ("UPDATE dbo.VmScalingRules SET MinVMs=-1", "UPDATE dbo.ScalingSchedules SET MinVMs=-1", + "UPDATE dbo.ScalingSchedules SET MinVMs=0, MaxVMs=0"): + try: + exec_sql(conn, sql) + except Exception: + conn.rollback() + else: + raise AssertionError(f"{sql} should be refused") + + +def test_the_starting_outcome_is_stored(conn): + record(conn, "Starting") + assert one(conn, "SELECT Outcome FROM dbo.CheckoutEvents")["Outcome"] == "Starting" + exec_sql(conn, "INSERT INTO dbo.CheckoutEvents (Username, Outcome) VALUES ('bob', 'Starting')") + + +def test_the_policy_lists_avd_hosts_whose_script_cannot_wait(conn): + policy = exec_sql(conn, "EXEC dbo.GetScalingPolicy")[0] + assert (policy["AvdHostsSeen"], policy["AvdHostsOutdated"]) == (0, 0) + assert (policy["OutdatedAvdHostsJson"], policy["AvdClientVersionsJson"]) == (None, None) + + record(conn, "Assigned", avdhost="avd-01", client_version="2.0.0") + record(conn, "Assigned", avdhost="avd-02", client_version="2.0.0", seconds_ago=60) + record(conn, "Assigned", avdhost="avd-03") + record(conn, "Reused", avdhost="avd-04", client_version="2.1.0") + # The latest checkout decides: avd-05 was updated, avd-06 went back to an old script. + record(conn, "Assigned", avdhost="avd-05", seconds_ago=600) + record(conn, "Reused", avdhost="avd-05", client_version="2.0.0") + record(conn, "Assigned", avdhost="avd-06", client_version="2.0.0", seconds_ago=600) + record(conn, "Reused", avdhost="avd-06", client_version="") + # An AVD host not seen for a week is left out. + record(conn, "Assigned", avdhost="avd-07", seconds_ago=8 * 86400) + + policy = exec_sql(conn, "EXEC dbo.GetScalingPolicy")[0] + assert (policy["AvdHostsSeen"], policy["AvdHostsOutdated"]) == (6, 2) + assert json.loads(policy["OutdatedAvdHostsJson"]) == [{"AvdHost": "avd-03"}, {"AvdHost": "avd-06"}] + assert json.loads(policy["AvdClientVersionsJson"]) == [{"ClientVersion": "2.0.0", "AvdHosts": 3}, + {"ClientVersion": "2.1.0", "AvdHosts": 1}] + assert one(conn, "SELECT ClientVersion FROM dbo.CheckoutEvents WHERE AvdHost='avd-06' AND Outcome='Reused'")["ClientVersion"] is None + + +# ----------------------------------------------------------------------- waiting users + + +def test_a_user_waits_while_their_latest_event_in_five_minutes_is_starting(conn): + record(conn, "Starting", "alice", seconds_ago=90) + record(conn, "Starting", "alice", seconds_ago=30) + record(conn, "Starting", "bob", seconds_ago=120) + record(conn, "Assigned", "bob", seconds_ago=10) + record(conn, "Starting", "carol", seconds_ago=400) + record(conn, "Assigned", "dave", seconds_ago=200) + record(conn, "Starting", "dave", seconds_ago=60) + record(conn, "Starting", "erin", seconds_ago=100) + record(conn, "NoneAvailable", "erin", seconds_ago=50) + + assert waiting(conn) == ["alice", "dave"] + + +# ----------------------------------------------------------------------- reserving a start + + +def test_reserve_is_disabled_while_start_on_demand_is_off(conn): + add_vm(conn, "off-1", power="Off", net="Unreachable") + set_start_on_demand(conn, enabled=False) + result = reserve(conn) + assert (result["Result"], result["VMID"], result["RetryAfterSeconds"]) == ("Disabled", None, None) + assert one(conn, "SELECT PowerState FROM dbo.VirtualMachines")["PowerState"] == "Off" + + +def test_reserve_says_ready_now_when_a_host_became_ready(conn): + add_vm(conn, "ready-1") + add_vm(conn, "off-1", power="Off", net="Unreachable") + assert reserve(conn)["Result"] == "ReadyNow" + assert rows(conn, "SELECT COUNT(*) AS c FROM dbo.VmScalingActivityLog")[0]["c"] == 0 + + +def test_reserve_starts_the_first_free_stopped_host_as_scaling_would(conn, second_conn): + set_rule(conn, MinVMs=0, MaxVMs=5) + add_vm(conn, "busy-1", status="CheckedOut", username="zoe", avdhost="avd", lease="7F6E3C1A-8E0C-4E5B-9A8C-0D1E2F3A4B5C") + drained = add_vm(conn, "off-drain", power="Off", net="Unreachable") + exec_sql(conn, "UPDATE dbo.VirtualMachines SET DrainRequested=1 WHERE VMID=%s", (drained,)) + add_vm(conn, "off-cleanup", power="Off", net="Unreachable", cleanup=True) + add_vm(conn, "off-maint", power="Off", net="Unreachable", status="Maintenance") + first = add_vm(conn, "off-1", power="Off", net="Unreachable") + add_vm(conn, "off-2", power="Off", net="Unreachable") + + result = reserve(conn, "alice", "avd-07") + assert (result["Result"], result["VMID"], result["VMName"]) == ("Started", first, "off-1") + assert (result["PreviousNetworkStatus"], result["RetryAfterSeconds"], result["Waiting"], result["Booting"]) == ("Unreachable", 60, 1, 1) + started = vm(conn, first) + assert (started["PowerState"], started["NetworkStatus"], started["VmStatus"]) == ("On", "Unreachable", "Available") + assert started["StartRequestedAt"] is not None and 0 <= started["ChangedSecondsAgo"] < 60 + + log = one(conn, "SELECT * FROM dbo.VmScalingActivityLog WHERE ActivityID=%s", (result["ActivityID"],)) + assert (log["ActionTaken"], log["VMsPoweredOn"], log["VMsPoweredOff"], log["CurrentRunningVMs"], log["NewTotalVMs"]) == ( + "Start On Demand", 1, 0, 1, 2) + assert (log["MinVMs"], log["MaxVMs"], log["PhaseName"]) == (0, 5, "Default rule") + assert "alice" in log["Notes"] and "avd-07" in log["Notes"] and "off-1" in log["Outcome"] + + # The scaling lock is released with the caller's transaction. + cur = second_conn().cursor() + cur.execute("BEGIN TRAN; DECLARE @r INT; EXEC @r = sp_getapplock @Resource='LinuxBroker.Scaling', @LockMode='Exclusive', " + "@LockOwner='Transaction', @LockTimeout=0; SELECT @r AS r") + assert cur.fetchone()[0] >= 0 + cur.execute("ROLLBACK") + + +def test_a_waiting_user_waits_for_the_host_already_starting(conn): + start_times(conn, 50, 70, 80) + first = booting(conn, "boot-1", seconds_ago=20) + add_vm(conn, "off-1", power="Off", net="Unreachable") + record(conn, "Starting", "alice", seconds_ago=20) + + result = reserve(conn, "alice") + assert (result["Result"], result["VMID"], result["Waiting"], result["Booting"]) == ("AlreadyStarting", None, 1, 1) + # The median start is 70 seconds, and the host has been starting for 20. + assert 48 <= result["RetryAfterSeconds"] <= 50 + assert json.loads(result["BootingHostsJson"]) == [{"VMID": first, "Hostname": "boot-1", "IPAddress": "10.0.0.1"}] + assert one(conn, "SELECT PowerState FROM dbo.VirtualMachines WHERE Hostname='off-1'")["PowerState"] == "Off" + + +def test_retry_after_is_kept_between_30_and_120_seconds(conn): + add_vm(conn, "off-1", power="Off", net="Unreachable") + add_vm(conn, "off-2", power="Off", net="Unreachable") + start_times(conn, 300, 400) + assert reserve(conn, "alice")["RetryAfterSeconds"] == 120 + record(conn, "Starting", "alice") + + exec_sql(conn, "DELETE FROM dbo.HostStartEvents") + start_times(conn, 10) + second = reserve(conn, "bob") + assert (second["Result"], second["RetryAfterSeconds"]) == ("Started", 30) + record(conn, "Starting", "bob") + + # A host that has been starting for longer than the median still asks for 30 seconds. + exec_sql(conn, "UPDATE dbo.VirtualMachines SET PowerStateChangedDate=DATEADD(SECOND, -200, GETDATE()) WHERE PowerState='On'") + third = reserve(conn, "alice") + assert (third["Result"], third["RetryAfterSeconds"]) == ("AlreadyStarting", 30) + + +def test_each_waiting_user_gets_a_host_up_to_the_pending_limit(conn): + for index in range(4): + add_vm(conn, f"off-{index}", power="Off", net="Unreachable") + + assert reserve(conn, "alice")["Result"] == "Started" + record(conn, "Starting", "alice") + second = reserve(conn, "bob") + assert (second["Result"], second["VMName"], second["Waiting"], second["Booting"]) == ("Started", "off-1", 2, 2) + record(conn, "Starting", "bob") + + third = reserve(conn, "carol") + assert (third["Result"], third["Waiting"], third["Booting"]) == ("AlreadyStarting", 3, 2) + assert [h["Hostname"] for h in json.loads(third["BootingHostsJson"])] == ["off-0", "off-1"] + + set_start_on_demand(conn, max_pending=3) + assert reserve(conn, "carol")["Result"] == "Started" + + +def test_booting_counts_only_free_hosts_started_in_the_last_ten_minutes(conn): + exec_sql(conn, "UPDATE dbo.ScalingPolicy SET MaxPendingStarts=5") + booting(conn, "stale", seconds_ago=11 * 60) + booting(conn, "cleaning", seconds_ago=30, cleanup=True) + add_vm(conn, "off-1", power="Off", net="Unreachable") + record(conn, "Starting", "bob") + + # One host is starting (the one being cleaned), for two waiting users. + result = reserve(conn, "alice") + assert (result["Result"], result["Booting"], result["Waiting"]) == ("Started", 2, 2) + + record(conn, "Starting", "alice") + waiting_again = reserve(conn, "alice") + assert waiting_again["Result"] == "AlreadyStarting" + # A host being cleaned counts as starting, but is not offered for a probe: a checkout could not take it yet. + assert [h["Hostname"] for h in json.loads(waiting_again["BootingHostsJson"])] == ["off-1"] + + +def test_reserve_stops_at_the_phase_maximum(conn): + set_rule(conn, MinVMs=0, MaxVMs=1) + add_vm(conn, "busy-1", status="CheckedOut", username="zoe", avdhost="avd", lease="7F6E3C1A-8E0C-4E5B-9A8C-0D1E2F3A4B5C") + add_vm(conn, "off-1", power="Off", net="Unreachable") + assert reserve(conn, "alice")["Result"] == "AtMaximum" + + # With a host already starting, a user at the maximum waits for it instead. + exec_sql(conn, "DELETE FROM dbo.VirtualMachines WHERE Hostname='busy-1'") + booting(conn, "boot-1", seconds_ago=10) + record(conn, "Starting", "bob") + result = reserve(conn, "alice") + assert (result["Result"], result["Booting"], result["Waiting"]) == ("AlreadyStarting", 1, 2) + + +def test_reserve_reports_no_candidate_when_nothing_can_start(conn): + add_vm(conn, "busy-1", status="CheckedOut", username="zoe", avdhost="avd", lease="7F6E3C1A-8E0C-4E5B-9A8C-0D1E2F3A4B5C") + add_vm(conn, "off-cleanup", power="Off", net="Unreachable", cleanup=True) + result = reserve(conn, "alice") + assert (result["Result"], result["RetryAfterSeconds"], result["BootingHostsJson"]) == ("NoCandidate", None, "[]") + + +def test_reserve_is_busy_while_scaling_holds_the_lock(conn, second_conn): + add_vm(conn, "off-1", power="Off", net="Unreachable") + cur = second_conn().cursor() + cur.execute("BEGIN TRAN; DECLARE @r INT; EXEC @r = sp_getapplock @Resource='LinuxBroker.Scaling', @LockMode='Exclusive', " + "@LockOwner='Transaction', @LockTimeout=0; SELECT @r") + try: + result = reserve(conn, "alice") + assert (result["Result"], result["RetryAfterSeconds"]) == ("Busy", 30) + assert one(conn, "SELECT PowerState FROM dbo.VirtualMachines")["PowerState"] == "Off" + finally: + cur.execute("ROLLBACK") + + +# ----------------------------------------------------------------------- scaling + + +def test_scaling_stops_idle_hosts_down_to_zero_while_start_on_demand_is_on(conn): + set_rule(conn, MinVMs=0, MaxVMs=5) + add_vm(conn, "idle-1", changed_minutes=30) + + preview = dry_run(conn) + assert (preview["Action"], preview["MinVMs"], preview["Waiting"], preview["StartOnDemandEnabled"]) == ("PowerOff", 0, 0, True) + stopped = exec_sql(conn, "EXEC dbo.TriggerScalingLogic") + assert [(r["ActionType"], r["VMName"]) for r in stopped] == [("PowerOff", "idle-1")] + + # Nothing is running and nobody is waiting: nothing to do. + preview = dry_run(conn) + assert (preview["Action"], preview["Serviceable"], preview["Utilization"]) == ("None", 0, 0) + + +def test_a_minimum_of_zero_is_read_as_one_while_start_on_demand_is_off(conn): + set_rule(conn, MinVMs=0, MaxVMs=5) + set_start_on_demand(conn, enabled=False) + add_vm(conn, "off-1", power="Off", net="Unreachable") + + preview = dry_run(conn) + assert (preview["Action"], preview["MinVMs"], preview["StartOnDemandEnabled"]) == ("PowerOn", 1, False) + assert preview["Reason"] == "Serviceable hosts are below the minimum." + exec_sql(conn, "EXEC dbo.TriggerScalingLogic") + notes = one(conn, "SELECT TOP 1 Notes FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC")["Notes"] + assert "Start on demand is off, so MinVMs 0 was read as 1." in notes and "Rule Min=1" in notes + + +def test_waiting_users_count_as_demand(conn): + set_rule(conn, MinVMs=0, MaxVMs=5, ScaleUpIncrement=1) + add_vm(conn, "off-1", power="Off", net="Unreachable") + add_vm(conn, "off-2", power="Off", net="Unreachable") + record(conn, "Starting", "alice") + + preview = dry_run(conn) + assert (preview["Action"], preview["Waiting"], float(preview["Utilization"]), preview["RequestCount"]) == ("PowerOn", 1, 100.0, 1) + assert preview["Reason"] == "Utilization is at or above the scale-up ratio. 1 user is waiting for a host to start." + + exec_sql(conn, "EXEC dbo.TriggerScalingLogic") + notes = one(conn, "SELECT TOP 1 Notes FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC")["Notes"] + assert "inUse=0, waiting=1," in notes + + +def test_scaling_down_leaves_a_host_for_each_waiting_user(conn): + set_rule(conn, MinVMs=0, MaxVMs=20, ScaleDownIncrement=20) + for index in range(10): + add_vm(conn, f"idle-{index}", changed_minutes=30) + + assert dry_run(conn)["RequestCount"] == 10 + record(conn, "Starting", "alice") + preview = dry_run(conn) + assert (preview["Action"], preview["RequestCount"], float(preview["Utilization"])) == ("PowerOff", 9, 10.0) + assert preview["Reason"].endswith("1 user is waiting for a host to start.") + + +# ----------------------------------------------------------------------- dashboard + + +def test_checkout_stats_measure_waits_and_leave_starting_out_of_the_total(conn): + record(conn, "Starting", "alice", seconds_ago=600) + record(conn, "Starting", "alice", seconds_ago=540) + record(conn, "Assigned", "alice", seconds_ago=480) + record(conn, "Starting", "bob", seconds_ago=300) + record(conn, "NoneAvailable", "bob", seconds_ago=240) + record(conn, "Starting", "carol", seconds_ago=900) + record(conn, "Reused", "carol", seconds_ago=660) + record(conn, "Starting", "dave", seconds_ago=30) + # A wait that began before the period is left out of it. + record(conn, "Starting", "erin", seconds_ago=3900) + record(conn, "Assigned", "erin", seconds_ago=3780) + + since = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=1) + stats = one(conn, "EXEC dbo.GetCheckoutStats @FromUtc=%s", (since,)) + assert (stats["Total"], stats["Assigned"], stats["Reused"], stats["NoneAvailable"], stats["Starting"]) == (3, 1, 1, 1, 5) + assert (stats["Waits"], stats["WaitsServed"], stats["WaitingNow"]) == (4, 2, 1) + # alice waited 120 seconds and carol 240. + assert (stats["WaitP50Seconds"], stats["WaitP95Seconds"]) == (180, 234) + + +def test_utilization_series_count_waiting_users_and_scaling_runs_only(conn): + now = datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0) + start = now - timedelta(minutes=30) + for outcome, user in (("Starting", "alice"), ("Starting", "alice"), ("Starting", "bob"), ("Assigned", "alice")): + exec_sql(conn, "INSERT INTO dbo.CheckoutEvents (OccurredAt, Username, Outcome) VALUES (%s, %s, %s)", + (now - timedelta(minutes=5), user, outcome)) + for action, running in (("No Action", 2), ("Start On Demand", 3)): + exec_sql(conn, "INSERT INTO dbo.VmScalingActivityLog (CheckTimestamp, CurrentRunningVMs, CurrentInUseVMs, ActionTaken, NewTotalVMs, " + "ServiceableVMs, MinVMs, MaxVMs) VALUES (DATEADD(MINUTE, -5, GETDATE()), %s, 0, %s, %s, %s, 0, 5)", + (running, action, running, running)) + + series = rows(conn, "EXEC dbo.GetUtilizationSeries @FromUtc=%s, @ToUtc=%s, @BucketMinutes=60", (start, now + timedelta(minutes=1))) + assert len(series) == 1 + bucket = series[0] + assert (bucket["Runs"], float(bucket["PoweredOn"]), bucket["Checkouts"], bucket["Waited"]) == (1, 2.0, 1, 2) + + +def test_no_ready_hosts_is_the_idle_state_of_a_pool_scaled_to_zero(conn): + set_rule(conn, MinVMs=0, MaxVMs=5) + add_vm(conn, "off-1", power="Off", net="Unreachable") + attention = lambda: [r["Kind"] for r in rows(conn, "EXEC dbo.GetAttentionItems")] + assert attention() == [] + + record(conn, "Starting", "alice") + assert attention() == ["no-ready-hosts"] + + exec_sql(conn, "DELETE FROM dbo.CheckoutEvents") + set_start_on_demand(conn, enabled=False) + assert attention() == ["no-ready-hosts"] + + set_start_on_demand(conn, enabled=True) + set_rule(conn, MinVMs=1, MaxVMs=5) + assert attention() == ["no-ready-hosts"] + + +def test_start_on_demand_objects_exist(conn): + procedures = {r["name"] for r in rows(conn, "SELECT name FROM sys.procedures")} + assert {"ReserveVmForStart", "SetScalingPolicyStartOnDemand"} <= procedures + functions = {r["name"] for r in rows(conn, "SELECT name FROM sys.objects WHERE type IN ('IF', 'TF', 'FN')")} + assert "fnWaitingCheckoutUsers" in functions From 5fac106d1ca6607a64f486ae644f90151635fc90 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Sat, 26 Sep 2026 16:41:29 -0400 Subject: [PATCH 03/15] Start a host on demand at checkout When a checkout finds no ready host, the broker API asks dbo.ReserveVmForStart for a host to start, commits to release the scaling lock, starts the host in Azure and answers 202 with Retry-After. A start that Azure refuses is recorded as off again and the checkout is refused with 409. While users wait, each request probes the starting hosts on port 22, so a host is handed out as soon as it answers. The checkout records the AVD script's clientVersion. The scaling policy gains the start on demand settings and reports which broker script the AVD session hosts run, and a rule or window may keep a minimum of 0 while start on demand is on. The portal adds a Start on demand card, allows a minimum of 0 accordingly, and shows waits on the dashboard. Connect-LinuxBroker.ps1 2.0.0 waits for a starting host in a window with Cancel, honours Retry-After up to -MaxWaitSeconds, retries transient failures and renews a refused token once. Update-AvdHostBrokerScript.ps1 replaces the script on existing AVD session hosts through Run Command, and Migrate-ExistingEnvironment.ps1 runs it after the Linux host migration. Because it writes the API URL and ScriptSourceRoot into PowerShell code, it accepts only https URLs of plain ASCII URL characters, and it refuses a script that is not plain ASCII, which Windows PowerShell could misread. A new CI job runs the Pester tests and PSScriptAnalyzer under Windows PowerShell 5.1. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/front-end-tests.yml | 46 + README.md | 12 +- api/README.md | 26 +- api/app.py | 439 ++++++- api/config.py | 22 + api/tests/test_lifecycle_scaling_settings.py | 29 +- api/tests/test_scaling_policy.py | 3 +- api/tests/test_start_on_demand.py | 576 ++++++++++ api/tests/test_trends.py | 14 +- api/tests_integration/conftest.py | 2 +- api/tests_integration/test_api_against_sql.py | 4 + .../test_start_on_demand_against_sql.py | 261 +++++ avd_host/broker/Connect-LinuxBroker.ps1 | 659 +++++++++-- avd_host/tests/Connect-LinuxBroker.Tests.ps1 | 776 +++++++++++++ avd_host/tests/PSScriptAnalyzerSettings.psd1 | 13 + .../Update-AvdHostBrokerScript.Tests.ps1 | 1009 +++++++++++++++++ deploy/DEPLOYMENT.md | 50 +- deploy/Migrate-ExistingEnvironment.ps1 | 69 +- deploy/Update-AvdHostBrokerScript.ps1 | 556 +++++++++ docs/ROADMAP.md | 31 +- front_end/README.md | 4 +- front_end/route_scaling_management.py | 26 +- front_end/tests/test_scaling_policy_bff.py | 27 + front_end/web/src/App.test.tsx | 106 +- .../components/dashboard/AttentionPanel.tsx | 7 +- .../src/components/dashboard/CapacityCard.tsx | 56 +- .../dashboard/CheckoutHealthCard.tsx | 28 + .../components/dashboard/dashboard.test.tsx | 70 +- .../components/scaling/ScalingPreviewCard.tsx | 13 + .../components/scaling/StartOnDemandCard.tsx | 229 ++++ .../web/src/components/ui/Badge.test.tsx | 1 + front_end/web/src/components/ui/Badge.tsx | 1 + front_end/web/src/hooks/useBroker.ts | 24 +- .../web/src/pages/scaling/CreateRule.tsx | 4 +- front_end/web/src/pages/scaling/RuleForm.tsx | 36 +- .../web/src/pages/scaling/ScalingPolicy.tsx | 10 + .../web/src/pages/scaling/ScheduleForm.tsx | 7 +- .../web/src/pages/scaling/UpdateRule.tsx | 4 +- front_end/web/src/types/broker.ts | 54 +- 39 files changed, 5130 insertions(+), 174 deletions(-) create mode 100644 api/tests/test_start_on_demand.py create mode 100644 api/tests_integration/test_start_on_demand_against_sql.py create mode 100644 avd_host/tests/Connect-LinuxBroker.Tests.ps1 create mode 100644 avd_host/tests/PSScriptAnalyzerSettings.psd1 create mode 100644 avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 create mode 100644 deploy/Update-AvdHostBrokerScript.ps1 create mode 100644 front_end/web/src/components/scaling/StartOnDemandCard.tsx diff --git a/.github/workflows/front-end-tests.yml b/.github/workflows/front-end-tests.yml index ce74a69..a08af39 100644 --- a/.github/workflows/front-end-tests.yml +++ b/.github/workflows/front-end-tests.yml @@ -8,8 +8,11 @@ on: - 'task/**' - 'sql_queries/**' - 'linux_host/**' + - 'avd_host/**' - 'custom_script_extensions/**' - 'deploy/bicep/**' + - 'deploy/Update-AvdHostBrokerScript.ps1' + - 'deploy/Migrate-ExistingEnvironment.ps1' - '.github/workflows/front-end-tests.yml' push: paths: @@ -18,8 +21,11 @@ on: - 'task/**' - 'sql_queries/**' - 'linux_host/**' + - 'avd_host/**' - 'custom_script_extensions/**' - 'deploy/bicep/**' + - 'deploy/Update-AvdHostBrokerScript.ps1' + - 'deploy/Migrate-ExistingEnvironment.ps1' - '.github/workflows/front-end-tests.yml' jobs: @@ -198,6 +204,46 @@ jobs: - name: Run the host script tests run: bash linux_host/tests/run.sh + avd-scripts: + name: AVD session host scripts + # Connect-LinuxBroker.ps1 runs in Windows PowerShell 5.1 on the session hosts. + runs-on: windows-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + + - name: Install Pester and PSScriptAnalyzer + shell: powershell + run: | + Install-Module Pester -RequiredVersion 5.7.1 -Force -Scope CurrentUser -SkipPublisherCheck + Install-Module PSScriptAnalyzer -RequiredVersion 1.24.0 -Force -Scope CurrentUser -SkipPublisherCheck + + # Also runs Update-AvdHostBrokerScript.ps1 and Migrate-ExistingEnvironment.ps1 with Azure mocked. + - name: Run the Pester tests + shell: powershell + run: | + Import-Module Pester -RequiredVersion 5.7.1 + $configuration = New-PesterConfiguration + $configuration.Run.Path = 'avd_host/tests' + $configuration.Run.Exit = $true + $configuration.Output.Verbosity = 'Detailed' + Invoke-Pester -Configuration $configuration + + - name: Lint the scripts + shell: powershell + run: | + Import-Module PSScriptAnalyzer -RequiredVersion 1.24.0 + $findings = @( + 'avd_host/broker/Connect-LinuxBroker.ps1', + 'deploy/Update-AvdHostBrokerScript.ps1' + ) | ForEach-Object { Invoke-ScriptAnalyzer -Path $_ -Settings avd_host/tests/PSScriptAnalyzerSettings.psd1 } + if (@($findings).Count -gt 0) { + $findings | Format-Table -Property ScriptName, Line, Severity, RuleName, Message -AutoSize -Wrap | Out-String -Width 240 | Write-Output + throw "PSScriptAnalyzer reported $(@($findings).Count) finding(s)." + } + bicep-build: name: Bicep templates runs-on: ubuntu-latest diff --git a/README.md b/README.md index b8b323f..296a7b0 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The architecture ensures secure, efficient, and scalable management of Linux hos 2. **Selects Linux Host Connection**: The user opens the **Linux Desktop** RemoteApp, which the `azd` deployment publishes, for a full RDP session to a Linux host. 3. **Broker Agent Initiates Connection**: - The Broker Agent script (`Connect-LinuxBroker.ps1`) connects to the Broker API using the AVD host's managed identity. - - It checks out an available Linux VM for the user. + - It checks out an available Linux VM for the user. When none is ready, the broker can start a stopped one, and the user waits a minute or two in a small window that says their desktop is starting. - The user's ID is added to the Linux host with a unique 25-character password. - The user is added to appropriate user groups on the Linux host for RDP access. - The host also receives the key that unlocks the user's login keyring, so applications that save passwords do not ask for one. @@ -329,7 +329,7 @@ azd env new azd up ``` -For existing environments that need in-place rollout instead of new-environment provisioning, use [deploy/Migrate-ExistingEnvironment.ps1](deploy/Migrate-ExistingEnvironment.ps1) from the `deploy/` directory. `azd up` remains the supported greenfield path. +For existing environments that need in-place rollout instead of new-environment provisioning, use [deploy/Migrate-ExistingEnvironment.ps1](deploy/Migrate-ExistingEnvironment.ps1) from the `deploy/` directory. It updates the apps, the database, the Linux hosts' agent and the AVD session hosts' broker script. `azd up` remains the supported greenfield path. The deployment targets Azure commercial by default. Set `azureCloudName` to `AzureUSGovernment` or `AzureCustom` to deploy elsewhere; commercial and Government resolve their endpoints automatically, while custom and sovereign clouds require their own authority, Graph, STS, and App Service FQDNs. Air-gapped environments should also set `scriptSourceRoot` to a reachable mirror of this repository, because the Linux hosts download their agent scripts from it during bootstrap. @@ -361,9 +361,15 @@ The distribution and desktop support release needs `azd provision` and agent 1.2 - **Review the idle timeout.** It had never disconnected anyone before this release, and migrated Ubuntu hosts now enforce any timeout already set. - **Replace or bootstrap again any Ubuntu hosts.** Earlier releases deployed them with no desktop and without the packages NFS homes need. +The start on demand release needs no new Azure resources or roles, but the AVD session hosts need `Connect-LinuxBroker.ps1` 2.0.0. See [Upgrading To Start On Demand](deploy/DEPLOYMENT.md#upgrading-to-start-on-demand). + +- **A checkout can start a host.** When no host is ready, the broker starts a stopped one and the user's **Linux Desktop** waits for it, for up to 10 minutes. It is on after the upgrade, and the **Scaling** page turns it off or limits how many hosts start at once. +- **Update the AVD session hosts.** `deploy/Migrate-ExistingEnvironment.ps1` now also runs `deploy/Update-AvdHostBrokerScript.ps1`, which replaces the script on every running session host. An older script tells its user that no host is available while one starts for them. +- **A pool can scale to zero.** With start on demand on, a rule or schedule window may keep a minimum of 0 hosts. Set one only after the **Scaling** page shows that every session host runs a script that can wait. + ## Roadmap -Planned work beyond this release, including RHEL 10 and Ubuntu 26.04 support, starting a host on demand, golden images and multi-session hosts, is described in [docs/ROADMAP.md](docs/ROADMAP.md). +Planned work beyond this release, including RHEL 10 and Ubuntu 26.04 support, golden images and multi-session hosts, is described in [docs/ROADMAP.md](docs/ROADMAP.md). ## Contributing diff --git a/api/README.md b/api/README.md index 591c7fe..a9bf57b 100644 --- a/api/README.md +++ b/api/README.md @@ -19,7 +19,7 @@ Role groups used below: **READ** = `Reader`, `Operator`, `FullAccess`; **OPERATE | GET | `/api/me` | any valid token | The caller's app roles and `permissions` (`read`, `operate`, `admin`), plus `legacyScopeAccess`. The portal uses it to adapt its interface. | | GET | `/api/vms` | READ, `ScheduledTask` | Lists all broker VM records, including `ReleasedDate`, `CleanupPending`, `CleanupUsername`, `PowerStateChangedDate`, `DrainRequested`, and `DrainRequestedDate`. With any of `page`, `per_page`, `q`, `status`, `sort` or `dir` it answers one page of the host list instead; see [Host List and Import](#host-list-and-import). | | GET | `/api/vms/summary` | READ, `ScheduledTask` | Returns dashboard counters: `TotalVMs`, `Available`, `CheckedOut`, `Maintenance`, `Released`, `PoweredOn`, `PoweredOff`, `Unreachable`, `Ready`, `CleanupPending`, and `Draining`. | -| POST | `/api/vms/checkout` | `AvdHost`, `FullAccess`, or `AVD_HOST_GROUP_ID` membership | Checks out a ready Linux host and provisions the user in one SSH session. | +| POST | `/api/vms/checkout` | `AvdHost`, `FullAccess`, or `AVD_HOST_GROUP_ID` membership | Checks out a ready Linux host and provisions the user in one SSH session. With no host ready it starts one on demand and answers `202` with `Retry-After`; see [Start on Demand](#start-on-demand). | | POST | `/api/vms//update-attributes` | ADMIN, `ScheduledTask` | Repairs the broker's record of a VM's power, network, or broker status. It does not start or stop anything. `ScheduledTask` keeps access for task builds older than `/network-status`. | | POST | `/api/vms//network-status` | `ScheduledTask`, `FullAccess` | Records a reachability probe result; writes only when the status changes. | | POST | `/api/vms//maintenance` | OPERATE | Moves an unassigned host between `Available` and `Maintenance`, clearing any drain flag. | @@ -47,8 +47,8 @@ Role groups used below: **READ** = `Reader`, `Operator`, `FullAccess`; **OPERATE | POST | `/api/scaling/rules//update` | ADMIN | Updates the scaling rule; the resulting rule is validated. | | POST | `/api/scaling/rules//delete` | ADMIN | Deletes a scaling rule. | | POST | `/api/scaling/rules/history` | READ | Returns scaling rule history, optionally paged with `page` and `per_page`. | -| GET | `/api/scaling/policy` | READ | The policy time zone, the default rule, every schedule window, and what applies now and next; `404` until the database has the scaling policy procedures. | -| POST | `/api/scaling/policy/update` | ADMIN | Sets the time zone every schedule window is read in (`{"timezone": ""}`). | +| GET | `/api/scaling/policy` | READ | The policy time zone, the default rule, every schedule window, what applies now and next, the start on demand settings, and which broker script the AVD hosts run; `404` until the database has the scaling policy procedures. | +| POST | `/api/scaling/policy/update` | ADMIN | Sets any of the time zone every schedule window is read in (`timezone`, a Windows zone name), `startondemandenabled` and `maxpendingstarts` (1–20), together or not at all. | | GET | `/api/scaling/timezones` | READ | The time zones the policy can use, from `sys.time_zone_info`. | | POST | `/api/scaling/schedules/create` | ADMIN | Adds a schedule window; `409` when it overlaps another enabled window. | | POST | `/api/scaling/schedules//update` | ADMIN | Replaces a schedule window. | @@ -133,7 +133,7 @@ Fleet health reports only. Checkout readiness is still decided by the task funct ## Audit Log -`@audited` records every call a portal user or an administrator principal makes to a mutating route, with the outcome taken from the status (`success`, `failure`, or `denied` for `403`), the target (the hostname where known), and a curated detail: the fields changed and their previous values, the result, and the error envelope's message. It never records a response body, so a password or lease ID cannot reach the log. `token_required` records authorization denials on mutating routes from any caller, up to 30 a minute per caller in each worker process; past that it only logs them, so a caller retrying a refused call cannot fill the table. The broker's own changes are recorded where they happen: scaling power actions and their failures, power states corrected from Azure, expired releases, completed cleanups and drains, and each purge. Routine agent calls are not audited: AVD checkouts, Linux host releases, acknowledgements and heartbeats, and the task's probes are high volume, and `VirtualMachinesHistory` already records what they change. +`@audited` records every call a portal user or an administrator principal makes to a mutating route, with the outcome taken from the status (`success`, `failure`, or `denied` for `403`), the target (the hostname where known), and a curated detail: the fields changed and their previous values, the result, and the error envelope's message. It never records a response body, so a password or lease ID cannot reach the log. `token_required` records authorization denials on mutating routes from any caller, up to 30 a minute per caller in each worker process; past that it only logs them, so a caller retrying a refused call cannot fill the table. The broker's own changes are recorded where they happen: scaling power actions and their failures, hosts started on demand for a waiting user, power states corrected from Azure, expired releases, completed cleanups and drains, and each purge. Routine agent calls are not audited: AVD checkouts, Linux host releases, acknowledgements and heartbeats, and the task's probes are high volume, and `VirtualMachinesHistory` already records what they change. The actor comes from the validated token: a portal user's sign-in name, or for a managed identity the VM or function app named in `xms_mirid`. The correlation ID is the request's OpenTelemetry trace ID when it is traced. Writing an entry never fails the operation: a failure is logged and the call carries on. Every entry is also written to the `linuxbroker.api.audit` logger with `audit_*` attributes, so Application Insights keeps a copy with its own retention, for example for export to a SIEM. @@ -161,9 +161,25 @@ A profile reset is requested and then applied at the user's next new assignment, Schedule windows override the default rule on chosen days and times, read in one policy time zone (a Windows zone name, `UTC` until one is chosen): for example Monday to Friday 08:00–18:00 keeping at least five hosts ready. A window may run past midnight; outside every enabled window the default rule applies. Overlapping enabled windows are refused in SQL under a lock, and in the API with a message naming the clash. `TriggerScalingLogic` takes its values from the active phase, and `GET /api/scaling/preview` runs the same decision as a dry run against the current counts, so the portal can show what the next run would do, and why, before anything is saved. `MinVMs` wins over `MaxVMs`: draining and maintenance hosts count toward the maximum, and a scale-down never takes serviceable hosts below the minimum. Windows take effect at the next five-minute scaling run. +## Start on Demand + +When a checkout finds no ready host, the API asks `ReserveVmForStart` what to do instead of refusing it. Under the scaling application lock, so it never races a scaling run, the procedure starts a stopped, free host for the user, or tells them to wait for one already starting, and the checkout answers `202`: + +```json +{"status": "Starting", "reason": "Started", "retryAfterSeconds": 60, "message": "A Linux host is starting for you. Ask again in 60 seconds."} +``` + +with a `Retry-After` header of the same number of seconds, 5–300. The AVD host asks again then, and gets the host once the reachability probe, or the API's own probe, reaches it. `reason` is `Started`, `AlreadyStarting` (enough hosts are starting for the users waiting, or `MaxPendingStarts` are starting), `Busy` (the scaling lock was not free within five seconds) or `ReadyNow` (a host became ready but was taken again). The wait is the median host start time over the last week, less how long the oldest starting host has been starting, kept between 30 and 120 seconds. While a user waits, each request probes the starting hosts on port 22 (`START_ON_DEMAND_PROBE_*`) and records the ones that answer as reachable, so the user need not wait for the task's next minute-by-minute probe. + +A start is recorded exactly as a scaling start is, with a `Start On Demand` row in the scaling activity log and a `vm.start_on_demand` audit entry. If Azure refuses it, the host is recorded as off again and the checkout is refused with `409`, as it is when start on demand is off, the phase's `MaxVMs` hosts are all on, or no stopped host is free. A database without `ReserveVmForStart` refuses as before, logged once. + +Scaling counts waiting users (`fnWaitingCheckoutUsers`: users whose latest checkout event in the last five minutes is `Starting`) as demand, and while anyone waits, a scale-down keeps at least as many serviceable hosts as there are users in use and waiting. With start on demand on, a rule or window may set `MinVMs` to `0`, so idle hosts stop until someone needs one; the API and `SaveScalingSchedule` refuse `0` while it is off. Turning it off while such rules exist is allowed: scaling then keeps one host running for them, and the update's message says so. `no-ready-hosts` is raised in that idle state only while a user is waiting. + +`POST /api/scaling/policy/update` takes `startondemandenabled` and `maxpendingstarts` with or without `timezone`, and applies them in one transaction. `GET /api/scaling/policy` returns `StartOnDemandEnabled`, `MaxPendingStarts`, `ZeroMinimumCount` (rules and enabled windows with a minimum of `0`), and `AvdHostScripts`: of the AVD hosts that asked for a checkout in the last seven days, how many run each broker script version and which report none (`Outdated`, `OutdatedHostnames`). A script that reports no version predates start on demand: it treats `202` as a failure, so its users see an error while a host starts for them. Update it with [`deploy/Update-AvdHostBrokerScript.ps1`](../deploy/Update-AvdHostBrokerScript.ps1). The checkout body's optional `clientVersion` (at most 32 letters, digits, `.`, `_`, `+` or `-`) is recorded with each checkout event; `AVD_HOST_SCRIPT_VERSION` in [`config.py`](config.py) is the current one, and a unit test requires it to match the script's `$ScriptVersion`. + ## Trends and Attention -Every checkout records its outcome (`Assigned`, `Reused`, `NoneAvailable`, `ProvisionFailed` or `Error`) and duration in `CheckoutEvents`. Recording never fails the checkout, and a missing procedure is logged once. `GET /api/metrics/utilization` combines the five-minute scaling runs with those events: running, in use, serviceable and the scaling maximum per bucket (15 minutes over a day, an hour over a week), denied checkouts, checkout time percentiles, and how long started hosts took to become reachable. `GET /api/metrics/attention` lists what needs an operator now: no ready host, denied checkouts in the last hour, hosts unreachable for 10 minutes, stuck in cleanup for 15, or checked out 30 minutes ago without the user connecting, hosts a maintenance run could not patch, and the fleet health flags, grouped. The daily audit purge also removes events older than `CHECKOUT_EVENT_RETENTION_DAYS`. Both endpoints answer `404` against a database without the procedures, and the portal then hides the charts. +Every checkout records its outcome (`Assigned`, `Reused`, `NoneAvailable`, `ProvisionFailed`, `Error`, or `Starting` each time a user is told to wait for a host) and duration in `CheckoutEvents`. Recording never fails the checkout, and a missing procedure is logged once. `GET /api/metrics/utilization` combines the five-minute scaling runs with those events: running, in use, serviceable and the scaling maximum per bucket (15 minutes over a day, an hour over a week), denied checkouts, the users told to wait (`Waited`), checkout time percentiles, and how long started hosts took to become reachable. `Total` and the denied share count checkouts that ended, so `Starting` events are counted apart, with `Waits` (waits that began in the period), `WaitsServed` (those that ended in a checkout within 20 minutes), `WaitP50Seconds` and `WaitP95Seconds`, and `WaitingNow`. `GET /api/metrics/attention` lists what needs an operator now: no ready host, denied checkouts in the last hour, hosts unreachable for 10 minutes, stuck in cleanup for 15, or checked out 30 minutes ago without the user connecting, hosts a maintenance run could not patch, and the fleet health flags, grouped. The daily audit purge also removes events older than `CHECKOUT_EVENT_RETENTION_DAYS`. Both endpoints answer `404` against a database without the procedures, and the portal then hides the charts. ## Rolling Maintenance diff --git a/api/app.py b/api/app.py index a8cfeb3..066007f 100644 --- a/api/app.py +++ b/api/app.py @@ -317,6 +317,8 @@ def reset_caches(): with _keyring_lock: _keyring_state.update({'client': None, 'unavailable_until': 0.0}) _checkout_event_state['missing_logged'] = False + _checkout_event_state['client_version_retry_at'] = 0.0 + _start_on_demand_state['missing_logged'] = False cache.clear() def is_duplicate_key_error(error) -> bool: @@ -2053,8 +2055,34 @@ def get_vm_summary(): logger.exception("Failed to build the VM summary.") return error_response("Unable to retrieve the virtual machine summary.", 500) -CHECKOUT_OUTCOMES = ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error') -_checkout_event_state = {'missing_logged': False} +CHECKOUT_OUTCOMES = ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error', 'Starting') +CLIENT_VERSION_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._+-]*$') +_checkout_event_state = {'missing_logged': False, 'client_version_retry_at': 0.0} +_start_on_demand_state = {'missing_logged': False} +# How long to leave out @ClientVersion after a database that predates it refused it. +CLIENT_VERSION_RECHECK_SECONDS = 600 + + +def parse_client_version(value): + """The version the AVD host script reports, or None when it reports none or an unusable one.""" + if not isinstance(value, str): + return None + value = value.strip() + if not value or len(value) > CLIENT_VERSION_MAX_CHARS or not CLIENT_VERSION_RE.match(value): + return None + return value + + +def _write_checkout_event(params, client_version): + sql = "EXEC RecordCheckoutEvent @Username = %s, @AvdHost = %s, @Outcome = %s, @DurationMs = %s, @Hostname = %s" + if client_version is not None: + sql += ", @ClientVersion = %s" + params = params + (client_version,) + with db_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(sql, params) + cursor.fetchone() + conn.commit() def record_checkout_event(event, started): @@ -2062,18 +2090,25 @@ def record_checkout_event(event, started): Never raises: a checkout must not fail because its statistics could not be written. A database without dbo.RecordCheckoutEvent, while the API is upgraded ahead of SQL, is - logged once per process rather than on every checkout. + logged once per process rather than on every checkout. One whose procedure predates + @ClientVersion (146) gets the event without it. """ duration_ms = max(0, int((time.monotonic() - started) * 1000)) + params = (event.get('username'), event.get('avdhost'), event['outcome'], duration_ms, event.get('hostname')) + client_version = event.get('clientVersion') + if client_version is not None and time.monotonic() < _checkout_event_state['client_version_retry_at']: + client_version = None try: - with db_connection() as conn: - with conn.cursor() as cursor: - cursor.execute( - "EXEC RecordCheckoutEvent @Username = %s, @AvdHost = %s, @Outcome = %s, @DurationMs = %s, @Hostname = %s", - (event.get('username'), event.get('avdhost'), event['outcome'], duration_ms, event.get('hostname')) - ) - cursor.fetchone() - conn.commit() + try: + _write_checkout_event(params, client_version) + except DatabaseUnavailable: + raise + except Exception as e: + if client_version is None or 'ClientVersion' not in str(e): + raise + _checkout_event_state['client_version_retry_at'] = time.monotonic() + CLIENT_VERSION_RECHECK_SECONDS + logger.warning("RecordCheckoutEvent does not take @ClientVersion yet; AVD host script versions are not recorded until the SQL upgrade.") + _write_checkout_event(params, None) except Exception as e: if not is_missing_procedure_error(e): logger.warning("Could not record the checkout event (%s).", event.get('outcome'), exc_info=True) @@ -2082,6 +2117,164 @@ def record_checkout_event(event, started): logger.warning("RecordCheckoutEvent is not deployed yet; checkouts are not counted until the SQL upgrade.") +def call_checkout_vm(username, avdhost): + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.callproc('CheckoutVm', (username, avdhost)) + rows = cursor.fetchall() + conn.commit() + return rows + + +def found_no_host(rows): + return not rows or 'Message' in rows[0] + + +def reserve_vm_for_start(username, avdhost): + """dbo.ReserveVmForStart's decision for a user whose checkout found no ready host. + + The procedure holds the scaling lock until the caller commits, so this commits at once, + before anything asks Azure. Returns None when the database made no decision: a database + that predates start on demand is logged once per process, any other failure every time, + and either way the checkout is refused as it was before start on demand. + """ + try: + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC ReserveVmForStart @Username = %s, @AvdHost = %s", (username, avdhost)) + row = cursor.fetchone() + conn.commit() + return row or None + except DatabaseUnavailable: + raise + except Exception as e: + if not is_missing_procedure_error(e): + logger.exception("Could not decide whether to start a host for %s.", username) + elif not _start_on_demand_state['missing_logged']: + _start_on_demand_state['missing_logged'] = True + logger.warning("ReserveVmForStart is not deployed yet; a checkout that finds no ready host is refused until the SQL upgrade.") + return None + + +def probe_ssh(ip_address): + """Whether a host accepts a connection on the port the reachability probe uses.""" + try: + with socket.create_connection((ip_address, START_ON_DEMAND_PROBE_PORT), timeout=START_ON_DEMAND_PROBE_TIMEOUT_SECONDS): + return True + except (OSError, ValueError): + return False + + +def mark_started_hosts_reachable(hosts): + """Probe the hosts starting for waiting users, and record the ones that answer as Reachable. + + Returns how many answered. The scheduled task probes every host each minute; this hands a + waiting user a host as soon as it is up. Never raises, except for an unreachable database. + """ + candidates = [host for host in hosts or [] if isinstance(host, dict) and host.get('VMID') and host.get('IPAddress')] + if not candidates: + return 0 + + with ThreadPoolExecutor(max_workers=min(START_ON_DEMAND_PROBE_CONCURRENCY, len(candidates))) as pool: + answers = list(pool.map(lambda host: probe_ssh(str(host['IPAddress'])), candidates)) + answered = [host for host, up in zip(candidates, answers) if up] + if not answered: + return 0 + + try: + with db_connection() as conn: + for host in answered: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC SetVmNetworkStatus @VMID = %s, @NetworkStatus = %s", (host['VMID'], 'Reachable')) + cursor.fetchone() + conn.commit() + except DatabaseUnavailable: + raise + except Exception: + logger.exception("Could not record the started hosts that answered as reachable.") + return 0 + + for host in answered: + logger.info("%s answered while starting for a waiting user, so it was recorded as reachable.", host.get('Hostname')) + return len(answered) + + +def _retry_after(value): + try: + seconds = int(value) + except (TypeError, ValueError): + return START_ON_DEMAND_RETRY_SECONDS + return min(300, max(5, seconds)) + + +def start_host_on_demand(username, avdhost, client_version): + """What to do for a user whose checkout found no ready host. + + Returns {'retry': bool, 'wait': None or {'reason', 'retryAfterSeconds', 'hostname'}}. retry + means a host may now be ready, so the checkout should be tried once more; wait means the + user should ask again after retryAfterSeconds, rather than be refused. + """ + row = reserve_vm_for_start(username, avdhost) or {} + result = row.get('Result') + retry_after = _retry_after(row.get('RetryAfterSeconds')) + + if result == 'ReadyNow': + return {'retry': True, 'wait': {'reason': 'ReadyNow', 'retryAfterSeconds': START_ON_DEMAND_RETRY_SECONDS}} + + if result == 'Busy': + return {'retry': False, 'wait': {'reason': 'Busy', 'retryAfterSeconds': retry_after}} + + if result == 'AlreadyStarting': + answered = mark_started_hosts_reachable(_json_column(row.get('BootingHostsJson'), list)) + return {'retry': answered > 0, 'wait': {'reason': 'AlreadyStarting', 'retryAfterSeconds': retry_after}} + + if result == 'Started': + vm_name = row.get('VMName') + detail = {key: value for key, value in ( + ('username', username), ('avdhost', avdhost), ('clientVersion', client_version), + ('activityId', row.get('ActivityID')), ('waiting', row.get('Waiting')), ('starting', row.get('Booting')), + ) if value is not None} + try: + if not VM_SUBSCRIPTION_ID or not VM_RESOURCE_GROUP: + raise RuntimeError("The Azure subscription or resource group of the Linux hosts is not configured.") + get_compute_client().virtual_machines.begin_start(VM_RESOURCE_GROUP, vm_name) + except Exception: + logger.exception("Azure refused to start %s for %s.", vm_name, username) + revert_power_action(row, 'Off') + append_scaling_note(row.get('ActivityID'), f"Starting {vm_name} failed, so it was recorded as off again.") + audit('vm.start_on_demand', 'vm', vm_name, AUDIT_FAILURE, + dict(detail, error='The Azure start operation could not be requested.')) + return {'retry': False, 'wait': None} + + logger.info("Started %s for %s, who found no ready host.", vm_name, username) + audit('vm.start_on_demand', 'vm', vm_name, AUDIT_SUCCESS, detail) + return {'retry': False, 'wait': {'reason': 'Started', 'retryAfterSeconds': retry_after, 'hostname': vm_name}} + + if result in ('AtMaximum', 'NoCandidate'): + logger.warning("No host could be started for %s (%s).", username, result) + return {'retry': False, 'wait': None} + + +START_ON_DEMAND_MESSAGES = { + 'Started': "A Linux host is starting for you.", + 'AlreadyStarting': "A Linux host is starting.", +} + + +def starting_response(wait): + """202: no host is ready yet, so the AVD host should ask again after Retry-After seconds.""" + seconds = wait['retryAfterSeconds'] + lead = START_ON_DEMAND_MESSAGES.get(wait['reason'], "No Linux host is ready yet.") + response = jsonify({ + 'status': 'Starting', + 'reason': wait['reason'], + 'retryAfterSeconds': seconds, + 'message': f"{lead} Ask again in {seconds} seconds.", + }) + response.headers['Retry-After'] = str(seconds) + return response, 202 + + @app.route('/api/vms/checkout', methods=['POST']) @token_required([ROLE_AVD_HOST, ROLE_ADMIN], required_group_ids=[AVD_HOST_GROUP_ID]) @audited('vm.checkout', target_type='vm') @@ -2111,23 +2304,38 @@ def _checkout_vm(event): if not username: return error_response("The username contains no characters a Linux account can use.", 400) - event.update(username=username, avdhost=avdhost, outcome='Error') + client_version = parse_client_version(req_body.get('clientVersion')) + event.update(username=username, avdhost=avdhost, outcome='Error', clientVersion=client_version) g.audit_detail = {'username': username, 'avdhost': avdhost} + if client_version: + g.audit_detail['clientVersion'] = client_version user_password = generate_secure_password() - with db_connection() as conn: - with conn.cursor(as_dict=True) as cursor: - cursor.callproc('CheckoutVm', (username, avdhost)) - rows = cursor.fetchall() - conn.commit() - + rows = call_checkout_vm(username, avdhost) if rows and is_procedure_error(rows[0]): logger.error("CheckoutVm failed with SQL error %s.", rows[0].get('ErrorNumber')) return error_response("Unable to check out a virtual machine.", 500) - if not rows or 'Message' in rows[0]: - event['outcome'] = 'NoneAvailable' - return error_response("No available VM found. Please try again.", 409) + if found_no_host(rows): + # Start on demand: start a host for this user, or wait for one already starting, + # instead of refusing them. + decision = start_host_on_demand(username, avdhost, client_version) + if decision['retry']: + rows = call_checkout_vm(username, avdhost) + if rows and is_procedure_error(rows[0]): + logger.error("CheckoutVm failed with SQL error %s.", rows[0].get('ErrorNumber')) + return error_response("Unable to check out a virtual machine.", 500) + + if found_no_host(rows): + wait = decision['wait'] + if wait: + event['outcome'] = 'Starting' + event['hostname'] = wait.get('hostname') + g.audit_detail['startOnDemand'] = wait['reason'] + return starting_response(wait) + + event['outcome'] = 'NoneAvailable' + return error_response("No available VM found. Please try again.", 409) checked_out_vm = rows[0] vmid = checked_out_vm.get("VMID") @@ -3340,14 +3548,31 @@ def parse_stop_mode(value): raise RuleValidationError("stopmode must be PowerOff or Deallocate.") return STOP_MODES[value.strip().lower()] -def validate_rule(rule): +def start_on_demand_enabled(): + """Whether start on demand is on. A database that predates it (144, 149) has it off.""" + try: + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC GetScalingPolicy") + row = cursor.fetchone() + except DatabaseUnavailable: + raise + except Exception as e: + if is_missing_procedure_error(e): + return False + raise + return bool((row or {}).get('StartOnDemandEnabled')) + +def validate_rule(rule, zero_minimum_allowed=None): """Check a complete rule. Mirrors the CHECK constraints on dbo.VmScalingRules. - A minimum of zero is rejected because a pool with no running host cannot recover: - nothing starts a VM until a checkout succeeds, and no checkout can succeed. + A minimum of zero is accepted only while start on demand is on: without it, a pool with no + running host cannot recover, because nothing starts a VM until a checkout succeeds and no + checkout can succeed. zero_minimum_allowed says whether start on demand is on; when it is + None the policy is read, and only for a minimum of zero. """ - if rule['minvms'] < 1: - raise RuleValidationError("minvms must be at least 1.") + if rule['minvms'] < 0: + raise RuleValidationError("minvms must be at least 0.") if rule['maxvms'] <= rule['minvms']: raise RuleValidationError("maxvms must be greater than minvms.") for field in ('scaleupratio', 'scaledownratio'): @@ -3358,6 +3583,10 @@ def validate_rule(rule): for field in ('scaleupincrement', 'scaledownincrement'): if rule[field] < 1: raise RuleValidationError(f"{field} must be at least 1.") + if rule['minvms'] == 0: + allowed = start_on_demand_enabled() if zero_minimum_allowed is None else zero_minimum_allowed + if not allowed: + raise RuleValidationError("minvms can be 0 only while start on demand is on.") def _rule_from_row(row): return { @@ -3777,6 +4006,40 @@ def _phase_from_policy(row): }) +def _optional_bool(value): + return None if value is None else bool(value) + + +def avd_host_scripts(policy): + """Which broker script the AVD hosts that asked for a checkout in the last seven days run. + + None when the database predates the summary (149). Outdated counts the hosts whose script + reports no version: it predates start on demand and gives up instead of waiting for a host. + """ + if 'AvdHostsSeen' not in policy: + return None + current = version_tuple(AVD_HOST_SCRIPT_VERSION) + versions = [] + for entry in _json_column(policy.get('AvdClientVersionsJson'), list) or []: + if isinstance(entry, dict) and entry.get('ClientVersion'): + reported = version_tuple(entry['ClientVersion']) + versions.append({ + 'ClientVersion': entry['ClientVersion'], + 'AvdHosts': int(entry.get('AvdHosts') or 0), + 'Current': reported is not None and current is not None and reported >= current, + }) + return { + 'Seen': int(policy.get('AvdHostsSeen') or 0), + 'Outdated': int(policy.get('AvdHostsOutdated') or 0), + 'OutdatedHostnames': [ + entry['AvdHost'] for entry in _json_column(policy.get('OutdatedAvdHostsJson'), list) or [] + if isinstance(entry, dict) and entry.get('AvdHost') + ], + 'Versions': versions, + 'CurrentVersion': AVD_HOST_SCRIPT_VERSION, + } + + @app.route('/api/scaling/policy', methods=['GET']) @token_required(READ_ROLES) def get_scaling_policy(): @@ -3805,6 +4068,11 @@ def get_scaling_policy(): 'Schedules': schedules, 'NextChange': next_phase_change(schedules, policy.get('LocalTime')), 'LastRun': last_run, + # None on a database that predates start on demand (144, 149). + 'StartOnDemandEnabled': _optional_bool(policy.get('StartOnDemandEnabled')), + 'MaxPendingStarts': policy.get('MaxPendingStarts'), + 'ZeroMinimumCount': policy.get('ZeroMinimumCount'), + 'AvdHostScripts': avd_host_scripts(policy), })), 200 except DatabaseUnavailable as e: @@ -3842,34 +4110,118 @@ def get_time_zones(): return error_response("Unable to list time zones.", 500) +def _policy_update_fields(body): + """The policy fields a request sets. Raises RuleValidationError.""" + fields = {} + zone = body.get('timezone') + if zone is not None: + if not isinstance(zone, str) or not zone.strip() or len(zone) > 64: + raise RuleValidationError("Provide the timezone as a name from /api/scaling/timezones.") + fields['timezone'] = zone.strip() + + enabled = body.get('startondemandenabled') + if enabled is not None: + if not isinstance(enabled, bool): + raise RuleValidationError("startondemandenabled must be true or false.") + fields['enabled'] = enabled + + pending = body.get('maxpendingstarts') + if pending is not None: + if isinstance(pending, str) and re.fullmatch(r'[0-9]{1,3}', pending.strip()): + pending = int(pending.strip()) + if isinstance(pending, bool) or not isinstance(pending, int) or not 1 <= pending <= 20: + raise RuleValidationError("maxpendingstarts must be a whole number from 1 to 20.") + fields['maxpendingstarts'] = pending + + if not fields: + raise RuleValidationError("Provide timezone, startondemandenabled, maxpendingstarts, or any of them.") + return fields + + +def _start_on_demand_message(row): + if not row.get('StartOnDemandEnabled'): + zero = int(row.get('ZeroMinimumCount') or 0) + if zero: + places = 'rule or window has' if zero == 1 else 'rules and windows have' + return (f"Start on demand is off. {zero} scaling {places} a minimum of 0, so scaling keeps one " + "host running for them.") + return "Start on demand is off. A user who finds no ready host is refused." + pending = row.get('MaxPendingStarts') + return f"Start on demand is on. Up to {pending} host{'s' if pending != 1 else ''} may start at once for waiting users." + + @app.route('/api/scaling/policy/update', methods=['POST']) @token_required(ADMIN_ROLES) @audited('scaling.policy_update', target_type='scaling') def update_scaling_policy(): - """Set the time zone every schedule window is read in.""" + """Set the time zone schedule windows are read in, and start on demand. + + Any of timezone, startondemandenabled and maxpendingstarts; they are applied together or + not at all. + """ try: g.audit_target_id = 'Policy' body = request.get_json(silent=True) body = body if isinstance(body, dict) else {} - zone = body.get('timezone') - if not isinstance(zone, str) or not zone.strip() or len(zone) > 64: - return error_response("Provide the timezone as a name from /api/scaling/timezones.", 400) + try: + fields = _policy_update_fields(body) + except RuleValidationError as e: + return error_response(e.client_message, 400) _, updated_by, _ = audit_actor() + zone_row = None + start_row = None + wants_start = 'enabled' in fields or 'maxpendingstarts' in fields with db_connection() as conn: with conn.cursor(as_dict=True) as cursor: - cursor.execute("EXEC SetScalingPolicyTimeZone @TimeZone = %s, @UpdatedBy = %s", (zone.strip(), updated_by)) - row = cursor.fetchone() or {} + if 'timezone' in fields: + cursor.execute("EXEC SetScalingPolicyTimeZone @TimeZone = %s, @UpdatedBy = %s", + (fields['timezone'], updated_by)) + zone_row = cursor.fetchone() or {} + if zone_row.get('Result') == 'InvalidTimeZone': + # Returning before the commit applies nothing else in the request either. + return error_response( + f"'{fields['timezone']}' is not a time zone SQL Server knows. Choose one from the list.", 400) + if wants_start: + try: + cursor.execute( + "EXEC SetScalingPolicyStartOnDemand @Enabled = %s, @MaxPendingStarts = %s, @UpdatedBy = %s", + (fields.get('enabled'), fields.get('maxpendingstarts'), updated_by)) + except Exception as e: + if is_missing_procedure_error(e): + logger.warning("SetScalingPolicyStartOnDemand is not deployed yet; start on demand cannot be changed.") + return error_response("Start on demand is not available until the database is upgraded.", 404) + raise + start_row = cursor.fetchone() or {} + if start_row.get('Result') == 'Invalid': + return error_response(str(start_row.get('Message') or 'The start on demand settings are not valid.'), 400) conn.commit() - result = row.get('Result') - if result == 'InvalidTimeZone': - return error_response(f"'{zone.strip()}' is not a time zone SQL Server knows. Choose one from the list.", 400) + results = [row.get('Result') for row in (zone_row, start_row) if row is not None] + result = 'Updated' if 'Updated' in results else 'Unchanged' + detail = {'result': result} + response = {'Result': result} + messages = [] + if zone_row is not None: + zone = zone_row.get('TimeZone') + detail.update({'from': zone_row.get('PreviousTimeZone'), 'to': zone}) + response['TimeZone'] = zone + messages.append(f"Schedules are now read in {zone}." if zone_row.get('Result') == 'Updated' + else f"Schedules were already read in {zone}.") + if start_row is not None: + enabled = _optional_bool(start_row.get('StartOnDemandEnabled')) + detail['startOnDemand'] = {'from': _optional_bool(start_row.get('PreviousStartOnDemandEnabled')), 'to': enabled} + detail['maxPendingStarts'] = {'from': start_row.get('PreviousMaxPendingStarts'), 'to': start_row.get('MaxPendingStarts')} + response.update({ + 'StartOnDemandEnabled': enabled, + 'MaxPendingStarts': start_row.get('MaxPendingStarts'), + 'ZeroMinimumCount': start_row.get('ZeroMinimumCount'), + }) + messages.append(_start_on_demand_message(start_row)) - g.audit_detail = {'from': row.get('PreviousTimeZone'), 'to': row.get('TimeZone'), 'result': result} - return jsonify({'TimeZone': row.get('TimeZone'), 'Result': result, - 'message': f"Schedules are now read in {row.get('TimeZone')}." if result == 'Updated' - else f"Schedules were already read in {row.get('TimeZone')}."}), 200 + g.audit_detail = detail + response['message'] = ' '.join(messages) + return jsonify(serialize_for_json(response)), 200 except DatabaseUnavailable as e: logger.error("Database connection failed while updating the scaling policy.") @@ -4100,8 +4452,11 @@ def preview_scaling(): 'Serviceable': row.get('Serviceable'), 'InUse': row.get('InUse'), 'Draining': row.get('Draining'), + # Users waiting for a host to start (4.1); counted as demand. + 'Waiting': row.get('Waiting'), 'Utilization': float(utilization) if utilization is not None else None, }, + 'StartOnDemandEnabled': _optional_bool(row.get('StartOnDemandEnabled')), 'TimeZone': row.get('TimeZone'), 'LocalTime': row.get('LocalTime'), 'AtUtc': row.get('AtUtc'), @@ -4837,6 +5192,9 @@ def get_host_health(): ATTENTION_MAX_HOSTNAMES = 10 CHECKOUT_STAT_COUNTS = ( 'Total', 'Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Errors', 'DeniedLastHour', 'HostStarts', + # Start on demand (154): Starting answers, the users who waited, those a host was then + # given to, and those waiting now. Starting answers are not in Total. + 'Starting', 'Waits', 'WaitsServed', 'WaitingNow', ) @@ -4875,6 +5233,7 @@ def utilization_point(row): 'Checkouts': int(row.get('Checkouts') or 0), 'Denied': int(row.get('Denied') or 0), 'Failed': int(row.get('Failed') or 0), + 'Waited': int(row.get('Waited') or 0), } @@ -4885,6 +5244,8 @@ def checkout_stats(row): 'P95Ms': _optional_int(row.get('P95Ms')), 'StartP50Seconds': _optional_int(row.get('StartP50Seconds')), 'StartP95Seconds': _optional_int(row.get('StartP95Seconds')), + 'WaitP50Seconds': _optional_int(row.get('WaitP50Seconds')), + 'WaitP95Seconds': _optional_int(row.get('WaitP95Seconds')), 'LastDeniedUtc': row.get('LastDeniedUtc'), 'DeniedPercent': round(stats['NoneAvailable'] * 100.0 / stats['Total'], 1) if stats['Total'] else None, }) diff --git a/api/config.py b/api/config.py index e4ea3bb..fefa051 100644 --- a/api/config.py +++ b/api/config.py @@ -197,6 +197,28 @@ def env_int(name, default, minimum=None, maximum=None): ATTENTION_NOT_CONNECTED_MINUTES = 30 ATTENTION_DENIED_MINUTES = 60 +# =============================== +# Start on demand +# +# When a checkout finds no ready host and start on demand is on, dbo.ReserveVmForStart starts a +# stopped host for the user, and the API answers 202 with how long to wait before asking again. +# A waiting user's next request first probes the hosts starting for it on the port the scheduled +# task's reachability probe uses, so a host that is up is handed out without waiting for the +# task's next run. The probes run in parallel and each waits at most the timeout. +START_ON_DEMAND_PROBE_PORT = 22 +START_ON_DEMAND_PROBE_TIMEOUT_SECONDS = 2 +START_ON_DEMAND_PROBE_CONCURRENCY = 10 +# The wait the API asks for when it could not decide: scaling held the lock, or the host that +# became ready went to another user. dbo.ReserveVmForStart asks for 30 to 120 seconds otherwise. +START_ON_DEMAND_RETRY_SECONDS = 30 +# The AVD host script reports its version with each checkout. It is recorded with the checkout +# event, so the scaling policy page can list the AVD hosts whose script cannot wait for a host. +CLIENT_VERSION_MAX_CHARS = 32 +# The version avd_host/broker/Connect-LinuxBroker.ps1 declares as $ScriptVersion. Bump it with +# any change to that script; api/tests checks they agree. The scaling policy page shows the AVD +# hosts that report an older one. +AVD_HOST_SCRIPT_VERSION = '2.0.0' + # =============================== # Linux host agent # diff --git a/api/tests/test_lifecycle_scaling_settings.py b/api/tests/test_lifecycle_scaling_settings.py index 1cb6705..f34b310 100644 --- a/api/tests/test_lifecycle_scaling_settings.py +++ b/api/tests/test_lifecycle_scaling_settings.py @@ -403,7 +403,7 @@ def test_a_refused_power_operation_restores_the_recorded_state(app_module, clien @pytest.mark.parametrize("override,field", [ - ({"minvms": 0}, "minvms"), + ({"minvms": -1}, "minvms"), ({"maxvms": 2}, "maxvms"), ({"scaleupratio": 101}, "scaleupratio"), ({"scaleupratio": 20}, "scaleupratio"), @@ -419,6 +419,33 @@ def test_rule_creation_names_the_invalid_field(client, fake_db, override, field) assert fake_db.calls == [] +def test_a_rule_scales_to_zero_only_while_start_on_demand_is_on(client, fake_db): + zero = {**VALID_RULE, "minvms": 0} + + # Off, and on a database that predates start on demand, a pool with no host cannot recover. + for policy in ({"TimeZone": "UTC"}, {"TimeZone": "UTC", "StartOnDemandEnabled": False}): + fake_db.fetchone_rows["GetScalingPolicy"] = policy + response = client.post("/api/scaling/rules/create", json=zero) + assert response.status_code == 400 + assert response.get_json()["error"] == "minvms can be 0 only while start on demand is on." + fake_db.fetchone_rows.pop("GetScalingPolicy") + fake_db.raise_on_execute["GetScalingPolicy"] = "(2812, b\"Could not find stored procedure 'GetScalingPolicy'.\")" + assert client.post("/api/scaling/rules/create", json=zero).status_code == 400 + assert "CreateScalingRule" not in [call["proc"] for call in fake_db.calls] + + del fake_db.raise_on_execute["GetScalingPolicy"] + fake_db.fetchone_rows["GetScalingPolicy"] = {"TimeZone": "UTC", "StartOnDemandEnabled": True} + fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 4, "ActiveRuleID": 4, "Message": None} + assert client.post("/api/scaling/rules/create", json=zero).status_code == 201 + assert fake_db.latest_call("CreateScalingRule")["params"][:2] == (0, 10) + + +def test_a_rule_above_zero_never_reads_the_policy(client, fake_db): + fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 5, "ActiveRuleID": 5, "Message": None} + assert client.post("/api/scaling/rules/create", json=VALID_RULE).status_code == 201 + assert "GetScalingPolicy" not in [call["proc"] for call in fake_db.calls] + + def test_rule_creation_passes_the_stop_mode(client, fake_db): fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 3, "ActiveRuleID": 3, "Message": None} response = client.post("/api/scaling/rules/create", json={**VALID_RULE, "stopmode": "deallocate"}) diff --git a/api/tests/test_scaling_policy.py b/api/tests/test_scaling_policy.py index 2271196..2dac20f 100644 --- a/api/tests/test_scaling_policy.py +++ b/api/tests/test_scaling_policy.py @@ -124,7 +124,8 @@ def test_a_window_is_saved_after_validation(auth_client, fake_db, audit_entries) ({"days": []}, "Choose at least one day."), ({"start": "8:00"}, "start must be a time as HH:MM, from 00:00 to 23:59."), ({"end": "08:00"}, "start and end must differ."), - ({"minvms": 0}, "minvms must be at least 1."), + ({"minvms": -1}, "minvms must be at least 0."), + ({"minvms": 0}, "minvms can be 0 only while start on demand is on."), ({"maxvms": 3}, "maxvms must be greater than minvms."), ({"scaleupratio": 10}, "scaleupratio must be greater than scaledownratio."), ({"stopmode": "Hibernate"}, "stopmode must be PowerOff or Deallocate."), diff --git a/api/tests/test_start_on_demand.py b/api/tests/test_start_on_demand.py new file mode 100644 index 0000000..703c0c7 --- /dev/null +++ b/api/tests/test_start_on_demand.py @@ -0,0 +1,576 @@ +"""4.1 start on demand and scale to zero: a checkout that finds no ready host starts one and +answers 202, a waiting user's next request probes the hosts starting for it, the AVD host +script's version is recorded, and the policy, preview and metrics carry the new settings.""" + +import json +import re +import types +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +LEASE_ID = "8ff6eb09-90ca-4efa-8ea1-695761f950f7" +MISSING = "(2812, b\"Could not find stored procedure '{}'.\")" +NO_HOST = [{"Message": "No available VM found"}] + + +def checkout_row(**values): + row = {"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5", "LeaseId": LEASE_ID, "CheckoutType": "Assigned", + "ProfileResetRequested": False} + row.update(values) + return row + + +def reservation(result, **values): + row = {"Result": result, "VMID": None, "VMName": None, "PreviousNetworkStatus": None, "ActivityID": None, + "RetryAfterSeconds": None, "BootingHostsJson": "[]", "Waiting": 1, "Booting": 0} + row.update(values) + return row + + +def started(**values): + row = {"VMID": 7, "VMName": "lnx-07", "PreviousNetworkStatus": "Unreachable", "ActivityID": 41, + "RetryAfterSeconds": 75, "Booting": 1} + row.update(values) + return reservation("Started", **row) + + +def procs(fake_db): + return [call["proc"] for call in fake_db.calls] + + +def events(fake_db): + return [call["params"] for call in fake_db.calls if call["proc"] == "RecordCheckoutEvent"] + + +def checkout(client, **extra): + return client.post("/api/vms/checkout", json={"username": "a.lice", "avdhost": "avd-01", **extra}) + + +@pytest.fixture +def provisioned(app_module, monkeypatch): + calls = [] + monkeypatch.setattr(app_module, "create_or_update_remote_user", lambda *args: calls.append(args) or True) + monkeypatch.setattr(app_module, "release_vm_assignment", lambda vmid, lease_id: None) + return calls + + +@pytest.fixture +def azure(app_module, fake_db, monkeypatch): + state = {"starts": [], "refuse": False, "commits": None} + + def begin_start(resource_group, name): + state["commits"] = fake_db.commits + if state["refuse"]: + raise RuntimeError("Operation 'start' is not allowed on VM 'lnx-07' in region xyz.") + state["starts"].append((resource_group, name)) + + client = types.SimpleNamespace(virtual_machines=types.SimpleNamespace(begin_start=begin_start)) + monkeypatch.setattr(app_module, "get_compute_client", lambda: client) + return state + + +@pytest.fixture +def probes(app_module, monkeypatch): + state = {"up": set(), "asked": []} + + def probe(ip_address): + state["asked"].append(ip_address) + return ip_address in state["up"] + + monkeypatch.setattr(app_module, "probe_ssh", probe) + return state + + +def booting(*hosts): + return json.dumps([{"VMID": vmid, "Hostname": name, "IPAddress": ip} for vmid, name, ip in hosts]) + + +# ------------------------------------------------------------------ starting a host + + +def test_no_ready_host_starts_one_and_asks_the_avd_host_to_wait(client, fake_db, provisioned, azure, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + + response = checkout(client, clientVersion="2.0.0") + + assert response.status_code == 202 + assert response.headers["Retry-After"] == "75" + assert response.get_json() == { + "status": "Starting", "reason": "Started", "retryAfterSeconds": 75, + "message": "A Linux host is starting for you. Ask again in 75 seconds.", + } + assert azure["starts"] == [("resource-group", "lnx-07")] + assert fake_db.latest_call("ReserveVmForStart")["params"] == ("alice", "avd-01") + assert provisioned == [] + + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.start_on_demand"] + assert (entry["targetType"], entry["targetId"], entry["outcome"]) == ("vm", "lnx-07", "success") + assert json.loads(entry["detailJson"]) == {"username": "alice", "avdhost": "avd-01", "clientVersion": "2.0.0", + "activityId": 41, "waiting": 1, "starting": 1} + + [params] = events(fake_db) + assert params[:3] == ("alice", "avd-01", "Starting") and params[4:] == ("lnx-07", "2.0.0") + # The reservation holds the scaling lock, so it is committed before Azure is asked: + # the checkout's commit, then the reservation's. + assert azure["commits"] == 2 + + +def test_azure_refusing_the_start_puts_the_host_back_and_refuses_the_checkout(client, fake_db, provisioned, azure, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + azure["refuse"] = True + + response = checkout(client) + + assert response.status_code == 409 + assert response.get_json()["error"] == "No available VM found. Please try again." + assert "region xyz" not in response.get_data(as_text=True) + assert fake_db.latest_call("UpdateVmAttributes")["params"] == (7, "Off", "Unreachable", None) + assert fake_db.latest_call("AppendScalingActivityNote")["params"] == (41, "Starting lnx-07 failed, so it was recorded as off again.") + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.start_on_demand"] + assert entry["outcome"] == "failure" + assert json.loads(entry["detailJson"])["error"] == "The Azure start operation could not be requested." + assert events(fake_db)[0][2] == "NoneAvailable" + + +def test_a_missing_subscription_is_treated_as_a_refused_start(client, fake_db, provisioned, azure, app_module, monkeypatch): + monkeypatch.setattr(app_module, "VM_RESOURCE_GROUP", "") + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + + assert checkout(client).status_code == 409 + assert azure["starts"] == [] + assert fake_db.latest_call("UpdateVmAttributes")["params"][:2] == (7, "Off") + + +@pytest.mark.parametrize("seconds,expected", [(None, 30), (1, 5), (500, 300), ("abc", 30), (45, 45)]) +def test_the_wait_is_kept_between_five_seconds_and_five_minutes(client, fake_db, provisioned, azure, seconds, expected): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started(RetryAfterSeconds=seconds) + response = checkout(client) + assert response.headers["Retry-After"] == str(expected) and response.get_json()["retryAfterSeconds"] == expected + + +# ------------------------------------------------------------------ hosts already starting + + +def test_a_waiting_user_is_given_a_starting_host_as_soon_as_it_answers(client, fake_db, provisioned, azure, probes): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [checkout_row(VMID=7, Hostname="lnx-07", IPAddress="10.0.0.7")]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=40, Booting=2, + BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"), (8, "lnx-08", "10.0.0.8"))) + probes["up"] = {"10.0.0.7"} + + response = checkout(client) + + assert response.status_code == 200 + assert response.get_json()["Hostname"] == "lnx-07" + assert sorted(probes["asked"]) == ["10.0.0.7", "10.0.0.8"] + marked = [call["params"] for call in fake_db.calls if call["proc"] == "SetVmNetworkStatus"] + assert marked == [(7, "Reachable")] + assert procs(fake_db).count("CheckoutVm") == 2 + assert azure["starts"] == [] + assert events(fake_db)[0][2:5:2] == ("Assigned", "lnx-07") + + +def test_a_waiting_user_waits_again_while_no_starting_host_answers(client, fake_db, provisioned, probes): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=40, BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"))) + + response = checkout(client) + + assert response.status_code == 202 and response.headers["Retry-After"] == "40" + assert response.get_json()["message"] == "A Linux host is starting. Ask again in 40 seconds." + assert probes["asked"] == ["10.0.0.7"] + assert "SetVmNetworkStatus" not in procs(fake_db) + assert procs(fake_db).count("CheckoutVm") == 1 + [params] = events(fake_db) + assert params[2] == "Starting" and params[4] is None + + +def test_a_host_that_answered_but_went_to_someone_else_means_waiting_again(client, fake_db, provisioned, probes): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=30, BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"))) + probes["up"] = {"10.0.0.7"} + + assert checkout(client).status_code == 202 + assert procs(fake_db).count("CheckoutVm") == 2 + + +def test_probing_skips_unusable_entries_and_survives_a_database_failure(app_module, fake_db, probes): + hosts = [{"VMID": 7, "Hostname": "lnx-07", "IPAddress": "10.0.0.7"}, {"VMID": None, "IPAddress": "10.0.0.9"}, + {"VMID": 8, "Hostname": "lnx-08"}, "junk"] + probes["up"] = {"10.0.0.7"} + assert app_module.mark_started_hosts_reachable(hosts) == 1 + assert probes["asked"] == ["10.0.0.7"] + assert app_module.mark_started_hosts_reachable(None) == 0 + + fake_db.raise_on_execute["SetVmNetworkStatus"] = "connection reset" + assert app_module.mark_started_hosts_reachable(hosts) == 0 + + +def test_the_probe_connects_to_the_ssh_port_with_a_timeout(app_module, monkeypatch): + attempts = [] + + class Connection: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def connect(address, timeout): + attempts.append((address, timeout)) + if address[0] == "10.0.0.9": + raise OSError("timed out") + return Connection() + + monkeypatch.setattr(app_module.socket, "create_connection", connect) + assert app_module.probe_ssh("10.0.0.7") is True + assert app_module.probe_ssh("10.0.0.9") is False + assert attempts == [(("10.0.0.7", 22), 2), (("10.0.0.9", 22), 2)] + + +# ------------------------------------------------------------------ the other decisions + + +def test_a_host_that_became_ready_is_checked_out_at_once(client, fake_db, provisioned): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [checkout_row()]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + response = checkout(client) + + assert response.status_code == 200 and response.get_json()["Hostname"] == "lnx-05" + assert len(provisioned) == 1 + + +def test_a_ready_host_taken_by_someone_else_means_waiting(client, fake_db, provisioned): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + response = checkout(client) + + assert response.status_code == 202 and response.headers["Retry-After"] == "30" + assert response.get_json()["reason"] == "ReadyNow" + assert response.get_json()["message"] == "No Linux host is ready yet. Ask again in 30 seconds." + + +def test_busy_scaling_means_asking_again_shortly(client, fake_db, provisioned): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("Busy", RetryAfterSeconds=30) + + response = checkout(client) + assert response.status_code == 202 and response.get_json()["reason"] == "Busy" + assert procs(fake_db).count("CheckoutVm") == 1 + + +@pytest.mark.parametrize("row", [reservation("Disabled"), reservation("AtMaximum"), reservation("NoCandidate"), + reservation("Unexpected"), None]) +def test_no_host_to_start_refuses_the_checkout_as_before(client, fake_db, provisioned, azure, row): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + if row is not None: + fake_db.fetchone_rows["ReserveVmForStart"] = row + + response = checkout(client) + + assert response.status_code == 409 + assert response.get_json()["error"] == "No available VM found. Please try again." + assert azure["starts"] == [] and events(fake_db)[0][2] == "NoneAvailable" + + +def test_a_database_without_start_on_demand_is_logged_once(client, fake_db, provisioned, caplog): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.raise_on_execute["ReserveVmForStart"] = MISSING.format("ReserveVmForStart") + + assert checkout(client).status_code == 409 + assert checkout(client).status_code == 409 + assert caplog.text.count("ReserveVmForStart is not deployed yet") == 1 + + fake_db.raise_on_execute["ReserveVmForStart"] = "connection reset" + checkout(client) + checkout(client) + assert caplog.text.count("Could not decide whether to start a host for alice") == 2 + + +def test_an_unreachable_database_while_deciding_is_not_recorded(client, fake_db, provisioned, app_module, monkeypatch): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + connections = {"count": 0} + real_connect = fake_db.connect + + def connect(): + connections["count"] += 1 + return real_connect() if connections["count"] == 1 else None + + monkeypatch.setattr(app_module, "get_db_connection", connect) + response = checkout(client) + assert response.get_json()["error"] == "Database connection failed." + assert connections["count"] == 2 + + +def test_a_failing_second_checkout_is_a_server_error(client, fake_db, provisioned): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [{"ErrorNumber": 1205, "Message": "Deadlock"}]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + assert checkout(client).status_code == 500 + assert events(fake_db)[0][2] == "Error" + + +# ------------------------------------------------------------------ the script's version + + +@pytest.mark.parametrize("value,recorded", [ + ("2.0.0", "2.0.0"), (" 2.1.0-beta+7 ", "2.1.0-beta+7"), ("", None), ("x" * 33, None), ("2.0; DROP", None), + (200, None), (None, None), ("-2", None), +]) +def test_only_a_plain_client_version_is_recorded(client, fake_db, provisioned, value, recorded): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + + assert checkout(client, clientVersion=value).status_code == 200 + + [call] = [call for call in fake_db.calls if call["proc"] == "RecordCheckoutEvent"] + if recorded is None: + assert len(call["params"]) == 5 and "@ClientVersion" not in call["sql"] + else: + assert call["params"][5] == recorded and "@ClientVersion = %s" in call["sql"] + + +def test_an_admin_checkout_audit_names_the_client_version(auth_client, fake_db, provisioned, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + client = auth_client({"roles": ["FullAccess"], "scp": "access_as_user"}) + + response = client.post("/api/vms/checkout", json={"username": "alice", "avdhost": "avd-01", "clientVersion": "2.0.0"}, + headers=auth_client.headers) + + assert response.status_code == 200 + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.checkout"] + assert json.loads(entry["detailJson"])["clientVersion"] == "2.0.0" + + +def test_a_database_that_predates_the_client_version_still_records_the_event(client, fake_db, provisioned, app_module, monkeypatch, caplog): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + rejected = [] + + class Cursor: + def __init__(self, inner): + self.inner = inner + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def execute(self, sql, params=None): + if "@ClientVersion" in sql: + rejected.append(params) + raise RuntimeError("(8145, b'@ClientVersion is not a parameter for procedure RecordCheckoutEvent.')") + return self.inner.execute(sql, params) + + def __getattr__(self, name): + return getattr(self.inner, name) + + real_connect = fake_db.connect + + def connect(): + connection = real_connect() + real_cursor = connection.cursor + connection.cursor = lambda as_dict=False: Cursor(real_cursor(as_dict=as_dict)) + return connection + + monkeypatch.setattr(app_module, "get_db_connection", connect) + + assert checkout(client, clientVersion="2.0.0").status_code == 200 + assert checkout(client, clientVersion="2.0.0").status_code == 200 + + # The first event is written again without the version; the next leaves it out at once. + assert len(rejected) == 1 + recorded = events(fake_db) + assert len(recorded) == 2 and all(len(params) == 5 for params in recorded) + assert caplog.text.count("does not take @ClientVersion yet") == 1 + + +def test_the_current_script_version_is_the_one_the_avd_host_script_sends(app_module): + script = (REPO_ROOT / "avd_host" / "broker" / "Connect-LinuxBroker.ps1").read_text(encoding="ascii") + + [version] = re.findall(r"^\$ScriptVersion = '([^']+)'", script, re.MULTILINE) + + assert version == app_module.AVD_HOST_SCRIPT_VERSION + assert app_module.CLIENT_VERSION_RE.match(version) and len(version) <= app_module.CLIENT_VERSION_MAX_CHARS + + +# ------------------------------------------------------------------ policy + + +def policy_row(**values): + row = {"TimeZone": "UTC", "LocalTime": "2026-09-24T10:15:00", "StartOnDemandEnabled": True, "MaxPendingStarts": 2, + "ZeroMinimumCount": 1, "AvdHostsSeen": 4, "AvdHostsOutdated": 1, + "OutdatedAvdHostsJson": json.dumps([{"AvdHost": "avd-03"}]), + "AvdClientVersionsJson": json.dumps([{"ClientVersion": "1.9.0", "AvdHosts": 1}, {"ClientVersion": "2.0.0", "AvdHosts": 2}])} + row.update(values) + return row + + +def test_the_policy_reports_start_on_demand_and_the_avd_host_scripts(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row() + + policy = client.get("/api/scaling/policy").get_json() + + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["ZeroMinimumCount"]) == (True, 2, 1) + assert policy["AvdHostScripts"] == { + "Seen": 4, "Outdated": 1, "OutdatedHostnames": ["avd-03"], "CurrentVersion": "2.0.0", + "Versions": [{"ClientVersion": "1.9.0", "AvdHosts": 1, "Current": False}, + {"ClientVersion": "2.0.0", "AvdHosts": 2, "Current": True}], + } + + +def test_the_policy_of_an_older_database_has_no_start_on_demand(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = {"TimeZone": "UTC"} + policy = client.get("/api/scaling/policy").get_json() + assert policy["StartOnDemandEnabled"] is None and policy["MaxPendingStarts"] is None + assert policy["AvdHostScripts"] is None + + +def test_no_avd_host_asked_means_an_empty_summary(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row(AvdHostsSeen=0, AvdHostsOutdated=0, OutdatedAvdHostsJson=None, + AvdClientVersionsJson=None) + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + assert scripts["Seen"] == 0 and scripts["OutdatedHostnames"] == [] and scripts["Versions"] == [] + + +def start_on_demand_row(**values): + row = {"Result": "Updated", "Message": None, "StartOnDemandEnabled": True, "MaxPendingStarts": 4, + "PreviousStartOnDemandEnabled": False, "PreviousMaxPendingStarts": 2, "ZeroMinimumCount": 0} + row.update(values) + return row + + +ADMIN_USER = {"roles": ["FullAccess"], "scp": "access_as_user", "oid": "oid-alice", "preferred_username": "alice@contoso.com"} + + +def test_start_on_demand_is_set_and_audited(auth_client, fake_db, audit_entries): + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row() + client = auth_client(ADMIN_USER) + + response = client.post("/api/scaling/policy/update", json={"startondemandenabled": True, "maxpendingstarts": 4}, + headers=auth_client.headers) + + assert response.status_code == 200 + assert response.get_json() == { + "Result": "Updated", "StartOnDemandEnabled": True, "MaxPendingStarts": 4, "ZeroMinimumCount": 0, + "message": "Start on demand is on. Up to 4 hosts may start at once for waiting users.", + } + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"] == (True, 4, "alice@contoso.com") + assert "SetScalingPolicyTimeZone" not in procs(fake_db) and fake_db.commits == 1 + [entry] = [entry for entry in audit_entries if entry["action"] == "scaling.policy_update"] + assert json.loads(entry["detailJson"]) == {"result": "Updated", "startOnDemand": {"from": False, "to": True}, + "maxPendingStarts": {"from": 2, "to": 4}, "status": 200} + + +def test_one_setting_at_a_time_leaves_the_other_alone(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row(MaxPendingStarts=3) + client.post("/api/scaling/policy/update", json={"maxpendingstarts": "3"}) + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"][:2] == (None, 3) + + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + StartOnDemandEnabled=False, PreviousStartOnDemandEnabled=True, ZeroMinimumCount=2) + body = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"][:2] == (False, None) + assert body["message"] == ("Start on demand is off. 2 scaling rules and windows have a minimum of 0, so scaling keeps " + "one host running for them.") + + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + StartOnDemandEnabled=False, ZeroMinimumCount=0, Result="Unchanged") + body = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert body["Result"] == "Unchanged" + assert body["message"] == "Start on demand is off. A user who finds no ready host is refused." + + +def test_the_zone_and_start_on_demand_are_applied_together(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "Unchanged", "TimeZone": "UTC", "PreviousTimeZone": "UTC"} + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row(MaxPendingStarts=1) + + body = client.post("/api/scaling/policy/update", + json={"timezone": "UTC", "startondemandenabled": True, "maxpendingstarts": 1}).get_json() + + assert body["Result"] == "Updated" and body["TimeZone"] == "UTC" + assert body["message"] == "Schedules were already read in UTC. Start on demand is on. Up to 1 host may start at once for waiting users." + assert fake_db.commits == 1 and len(fake_db.connections) == 1 + + +@pytest.mark.parametrize("payload,message", [ + ({}, "Provide timezone, startondemandenabled, maxpendingstarts, or any of them."), + ({"startondemandenabled": "yes"}, "startondemandenabled must be true or false."), + ({"maxpendingstarts": 0}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": 21}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": True}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": 2.5}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": "\u00b2"}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": "9" * 5000}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"timezone": " ", "startondemandenabled": True}, "Provide the timezone as a name from /api/scaling/timezones."), +]) +def test_invalid_policy_settings_are_refused_before_the_database(client, fake_db, payload, message): + response = client.post("/api/scaling/policy/update", json=payload) + assert response.status_code == 400 and response.get_json()["error"] == message + assert fake_db.calls == [] + + +def test_nothing_is_applied_when_any_part_is_refused(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "InvalidTimeZone", "TimeZone": "UTC"} + response = client.post("/api/scaling/policy/update", json={"timezone": "Mars", "startondemandenabled": True}) + assert response.status_code == 400 and "Mars" in response.get_json()["error"] + assert "SetScalingPolicyStartOnDemand" not in procs(fake_db) and fake_db.commits == 0 + + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "Updated", "TimeZone": "UTC", "PreviousTimeZone": "Tokyo Standard Time"} + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + Result="Invalid", Message="maxpendingstarts must be between 1 and 20.") + response = client.post("/api/scaling/policy/update", json={"timezone": "UTC", "maxpendingstarts": 5}) + assert response.status_code == 400 and response.get_json()["error"] == "maxpendingstarts must be between 1 and 20." + assert fake_db.commits == 0 + + +def test_start_on_demand_needs_the_upgraded_database(client, fake_db): + fake_db.raise_on_execute["SetScalingPolicyStartOnDemand"] = MISSING.format("SetScalingPolicyStartOnDemand") + response = client.post("/api/scaling/policy/update", json={"startondemandenabled": True}) + assert response.status_code == 404 + assert response.get_json()["error"] == "Start on demand is not available until the database is upgraded." + assert fake_db.commits == 0 + + +# ------------------------------------------------------------------ schedules and the preview + + +SCHEDULE = {"name": "Nights", "days": ["mon"], "start": "20:00", "end": "23:00", "minvms": 0, "maxvms": 4, + "scaleupratio": 60, "scaleupincrement": 1, "scaledownratio": 20, "scaledownincrement": 1} + + +def test_a_window_scales_to_zero_while_start_on_demand_is_on(client, fake_db): + fake_db.fetchall_rows["GetScalingSchedules"] = [] + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row() + fake_db.fetchone_rows["SaveScalingSchedule"] = {"Result": "Created", "ScheduleID": 3} + + assert client.post("/api/scaling/schedules/create", json=SCHEDULE).status_code == 201 + assert fake_db.latest_call("SaveScalingSchedule")["params"][6] == 0 + + +def test_the_preview_counts_waiting_users_and_checks_a_zero_minimum(client, fake_db): + fake_db.fetchone_rows["TriggerScalingLogic"] = { + "Action": "PowerOn", "RequestCount": 1, "CandidatesJson": json.dumps([{"Hostname": "lnx-09"}]), + "Reason": "Serviceable hosts are below the minimum. 1 user is waiting for a host to start.", "PhaseSource": "Rule", + "MinVMs": 0, "MaxVMs": 4, "PoweredOn": 0, "Serviceable": 0, "InUse": 0, "Draining": 0, "Utilization": 100, + "Waiting": 1, "StartOnDemandEnabled": True, + } + + body = client.get("/api/scaling/preview").get_json() + assert body["Counts"]["Waiting"] == 1 and body["StartOnDemandEnabled"] is True + + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row(StartOnDemandEnabled=False) + proposed = {key: value for key, value in SCHEDULE.items() if key not in ("days", "start", "end")} + response = client.post("/api/scaling/preview", json={"rule": proposed}) + assert response.status_code == 400 + assert response.get_json()["error"] == "minvms can be 0 only while start on demand is on." diff --git a/api/tests/test_trends.py b/api/tests/test_trends.py index 3aab1c6..1a84fd3 100644 --- a/api/tests/test_trends.py +++ b/api/tests/test_trends.py @@ -130,7 +130,8 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app monkeypatch.setattr(app_module, "utilization_window", lambda hours: (start, end, 15)) fake_db.fetchall_rows["GetUtilizationSeries"] = [ {"BucketStartUtc": "2026-03-03T10:45:00Z", "Runs": 3, "PoweredOn": Decimal("4.0"), "InUse": Decimal("2.3"), - "Serviceable": Decimal("3.7"), "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0}, + "Serviceable": Decimal("3.7"), "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0, + "Waited": 2}, {"BucketStartUtc": "2026-03-03T11:00:00Z", "Runs": 0, "PoweredOn": None, "InUse": None, "Serviceable": None, "PeakInUse": None, "MinVMs": None, "MaxVMs": None, "Checkouts": 0, "Denied": 0, "Failed": 0}, ] @@ -138,6 +139,7 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app "Total": 40, "Assigned": 20, "Reused": 15, "NoneAvailable": 3, "ProvisionFailed": 1, "Errors": 1, "P50Ms": 2100, "P95Ms": 8400, "DeniedLastHour": 1, "LastDeniedUtc": "2026-03-04T10:02:11.120Z", "HostStarts": 4, "StartP50Seconds": 95, "StartP95Seconds": 180, + "Starting": 6, "Waits": 3, "WaitsServed": 2, "WaitP50Seconds": 84, "WaitP95Seconds": 150, "WaitingNow": 1, } response = client.get("/api/metrics/utilization") @@ -146,9 +148,13 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app body = response.get_json() assert (body["Hours"], body["BucketMinutes"], body["FromUtc"], body["ToUtc"]) == (24, 15, "2026-03-03T10:45:00Z", "2026-03-04T10:45:00Z") assert body["Series"][0] == {"BucketStartUtc": "2026-03-03T10:45:00Z", "Runs": 3, "PoweredOn": 4.0, "InUse": 2.3, - "Serviceable": 3.7, "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0} - assert body["Series"][1]["PoweredOn"] is None and body["Series"][1]["Runs"] == 0 + "Serviceable": 3.7, "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0, + "Waited": 2} + assert body["Series"][1]["PoweredOn"] is None and body["Series"][1]["Runs"] == 0 and body["Series"][1]["Waited"] == 0 assert body["Checkouts"]["DeniedPercent"] == 7.5 and body["Checkouts"]["StartP95Seconds"] == 180 + # Starting answers are not checkouts: the denial rate is still NoneAvailable over Total. + assert {key: body["Checkouts"][key] for key in ("Starting", "Waits", "WaitsServed", "WaitP50Seconds", "WaitP95Seconds", "WaitingNow")} == { + "Starting": 6, "Waits": 3, "WaitsServed": 2, "WaitP50Seconds": 84, "WaitP95Seconds": 150, "WaitingNow": 1} assert fake_db.latest_call("GetUtilizationSeries")["params"] == (start, end, 15) assert fake_db.latest_call("GetCheckoutStats")["params"] == (start, end) @@ -158,6 +164,8 @@ def test_utilization_with_no_checkouts_has_no_denial_rate(client, fake_db): body = client.get("/api/metrics/utilization?hours=168").get_json() assert body["BucketMinutes"] == 60 and body["Series"] == [] assert body["Checkouts"]["DeniedPercent"] is None and body["Checkouts"]["P50Ms"] is None + # A database that predates start on demand (154) reports no waits. + assert body["Checkouts"]["Waits"] == 0 and body["Checkouts"]["WaitP50Seconds"] is None @pytest.mark.parametrize("hours", ["12", "abc", "0", "720"]) diff --git a/api/tests_integration/conftest.py b/api/tests_integration/conftest.py index 302c099..c64a308 100644 --- a/api/tests_integration/conftest.py +++ b/api/tests_integration/conftest.py @@ -147,7 +147,7 @@ def db(sql): "DELETE FROM dbo.VmScalingActivityLog", "DELETE FROM dbo.VmScalingRules", "DELETE FROM dbo.ScalingSchedules", - "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL", + "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL, StartOnDemandEnabled=1, MaxPendingStarts=2", "UPDATE dbo.LinuxHostSettings SET GracePeriodSeconds=1200, ReconcileIntervalSeconds=60, " "IdleTimeoutSeconds=0, IdleWarningSeconds=120, ScreenLockEnabled=0, DisableLockScreen=1, " "PreserveSessionsOnDisconnect=0 WHERE SettingsScope='Global'", diff --git a/api/tests_integration/test_api_against_sql.py b/api/tests_integration/test_api_against_sql.py index b233cfb..63cd07c 100644 --- a/api/tests_integration/test_api_against_sql.py +++ b/api/tests_integration/test_api_against_sql.py @@ -14,9 +14,13 @@ def _checkout(client, username="alice", avdhost="avd-01"): def test_checkout_with_no_ready_host_is_a_conflict_not_a_server_error(client, db, remote): + # With start on demand on, the stopped host would be started instead: see + # test_start_on_demand_against_sql.py. + db.run("UPDATE dbo.ScalingPolicy SET StartOnDemandEnabled = 0") db.add_vm("lnxhost-01", power="Off", network="Unreachable") response = _checkout(client) assert response.status_code == 409, response.get_json() + assert db.vm("lnxhost-01")["PowerState"] == "Off" def test_checkout_provisions_the_user_and_records_the_assignment(client, db, remote): diff --git a/api/tests_integration/test_start_on_demand_against_sql.py b/api/tests_integration/test_start_on_demand_against_sql.py new file mode 100644 index 0000000..7ca2311 --- /dev/null +++ b/api/tests_integration/test_start_on_demand_against_sql.py @@ -0,0 +1,261 @@ +"""4.1 start on demand and scale to zero against the real stored procedures and driver.""" + +import json +import types + +import pytest + + +RULE = {"minvms": 2, "maxvms": 6, "scaleupratio": 70, "scaleupincrement": 1, "scaledownratio": 30, "scaledownincrement": 1} +EVERY_DAY = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + + +class Compute: + def __init__(self, states=None, fail=()): + self.operations = [] + self.states = states or {} + self.fail = set(fail) + compute = self + + class VirtualMachines: + def _operate(self, operation, name): + if operation in compute.fail: + raise RuntimeError("refused") + compute.operations.append((operation, name)) + + def instance_view(self, resource_group, name): + return types.SimpleNamespace(statuses=[types.SimpleNamespace(code=compute.states.get(name, "PowerState/running"))]) + + def begin_start(self, resource_group, name): + self._operate("start", name) + + def begin_power_off(self, resource_group, name): + self._operate("power_off", name) + + def begin_deallocate(self, resource_group, name): + self._operate("deallocate", name) + + self.virtual_machines = VirtualMachines() + + +@pytest.fixture +def compute(app_module, monkeypatch): + fake = Compute() + monkeypatch.setattr(app_module, "ComputeManagementClient", lambda *a, **k: fake) + return fake + + +@pytest.fixture +def probes(app_module, monkeypatch): + """The hosts that answer the start on demand probe, by IP address.""" + answering = set() + monkeypatch.setattr(app_module, "probe_ssh", lambda ip: ip in answering) + return answering + + +def _checkout(client, username, avdhost="avd-01", **extra): + return client.post("/api/vms/checkout", json={"username": username, "avdhost": avdhost, **extra}) + + +def _events(db): + return [(row["Username"], row["Outcome"], row["Hostname"], row["ClientVersion"]) + for row in db.run("SELECT Username, Outcome, Hostname, ClientVersion FROM dbo.CheckoutEvents ORDER BY EventID")] + + +def _audits(db, action): + return db.run("SELECT TargetId, Outcome, DetailJson FROM dbo.AuditLog WHERE Action = %s ORDER BY AuditId", (action,)) + + +def test_a_checkout_that_finds_no_ready_host_starts_one_and_is_told_to_wait(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + + response = _checkout(client, "alice", clientVersion="2.0.0") + + assert response.status_code == 202, response.get_json() + body = response.get_json() + # With no start history the procedure expects a start to take 60 seconds. + assert body == {"status": "Starting", "reason": "Started", "retryAfterSeconds": 60, + "message": "A Linux host is starting for you. Ask again in 60 seconds."} + assert response.headers["Retry-After"] == "60" + assert compute.operations == [("start", "lnxhost-01")] + vm = db.vm("lnxhost-01") + assert (vm["PowerState"], vm["NetworkStatus"], vm["VmStatus"], vm["Username"]) == ("On", "Unreachable", "Available", None) + assert vm["StartRequestedAt"] is not None + + log = db.one("SELECT TOP 1 * FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC") + assert log["ActionTaken"] == "Start On Demand" and log["VMsPoweredOn"] == 1 + assert "alice" in log["Notes"] and "avd-01" in log["Notes"] + assert _events(db) == [("alice", "Starting", "lnxhost-01", "2.0.0")] + [entry] = _audits(db, "vm.start_on_demand") + detail = json.loads(entry["DetailJson"]) + assert (entry["TargetId"], entry["Outcome"]) == ("lnxhost-01", "success") + assert (detail["username"], detail["avdhost"], detail["clientVersion"], detail["activityId"]) == ( + "alice", "avd-01", "2.0.0", log["ActivityID"]) + assert remote.calls == [], "no user is created until a host is ready" + + +def test_the_waiting_user_is_given_the_host_as_soon_as_it_answers(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + assert _checkout(client, "alice", clientVersion="2.0.0").status_code == 202 + # The host was asked to start 90 seconds ago, and has not answered yet. + db.run("UPDATE dbo.VirtualMachines SET StartRequestedAt = DATEADD(SECOND, -90, SYSUTCDATETIME()), " + "PowerStateChangedDate = DATEADD(SECOND, -90, GETDATE())") + db.run("UPDATE dbo.CheckoutEvents SET OccurredAt = DATEADD(SECOND, -90, OccurredAt)") + + still_starting = _checkout(client, "alice", clientVersion="2.0.0") + assert still_starting.status_code == 202, still_starting.get_json() + assert still_starting.get_json()["reason"] == "AlreadyStarting" + assert still_starting.get_json()["retryAfterSeconds"] == 30, "a start expected in 60 seconds is already overdue" + assert compute.operations == [("start", "lnxhost-01")], "a user already waiting does not start a second host" + + probes.add("10.0.0.4") + ready = _checkout(client, "alice", clientVersion="2.0.0") + + assert ready.status_code == 200, ready.get_json() + assert ready.get_json()["Hostname"] == "lnxhost-01" + vm = db.vm("lnxhost-01") + assert (vm["NetworkStatus"], vm["VmStatus"], vm["Username"]) == ("Reachable", "CheckedOut", "alice") + assert [kind for _, kind in remote.calls] == ["create"] + start = db.one("SELECT Hostname, Seconds FROM dbo.HostStartEvents") + assert start["Hostname"] == "lnxhost-01" and 85 <= start["Seconds"] <= 120 + + stats = client.get("/api/metrics/utilization").get_json()["Checkouts"] + assert (stats["Total"], stats["Assigned"], stats["Starting"]) == (1, 1, 2) + assert (stats["Waits"], stats["WaitsServed"], stats["WaitingNow"]) == (1, 1, 0) + assert 85 <= stats["WaitP50Seconds"] <= 120 + series = client.get("/api/metrics/utilization").get_json()["Series"] + assert sum(point["Waited"] for point in series) == 1 + + +def test_starts_for_waiting_users_stop_at_the_pending_limit(client, db, remote, compute, probes): + for index in range(1, 4): + db.add_vm(f"lnxhost-0{index}", power="Off", network="Unreachable") + + first = _checkout(client, "alice") + second = _checkout(client, "bob") + third = _checkout(client, "carol") + + assert [r.get_json()["reason"] for r in (first, second, third)] == ["Started", "Started", "AlreadyStarting"] + assert compute.operations == [("start", "lnxhost-01"), ("start", "lnxhost-02")] + assert db.vm("lnxhost-03")["PowerState"] == "Off" + assert client.get("/api/metrics/utilization").get_json()["Checkouts"]["WaitingNow"] == 3 + + preview = client.get("/api/scaling/preview").get_json() + assert preview["Counts"]["Waiting"] == 3 and preview["StartOnDemandEnabled"] is True + + +def test_a_refused_start_puts_the_host_back_off_and_refuses_the_checkout(app_module, client, db, remote, monkeypatch): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + monkeypatch.setattr(app_module, "ComputeManagementClient", lambda *a, **k: Compute(fail={"start"})) + + response = _checkout(client, "alice", clientVersion="2.0.0") + + assert response.status_code == 409, response.get_json() + vm = db.vm("lnxhost-01") + assert (vm["PowerState"], vm["NetworkStatus"]) == ("Off", "Unreachable") + log = db.one("SELECT TOP 1 Notes FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC") + assert "Starting lnxhost-01 failed, so it was recorded as off again." in log["Notes"] + [entry] = _audits(db, "vm.start_on_demand") + assert entry["Outcome"] == "failure" + assert json.loads(entry["DetailJson"])["error"] == "The Azure start operation could not be requested." + assert _events(db) == [("alice", "NoneAvailable", None, "2.0.0")] + + +def test_the_policy_update_sets_start_on_demand_with_the_time_zone_or_not_at_all(app_module, client, db, monkeypatch): + updated = client.post("/api/scaling/policy/update", json={"startondemandenabled": False, "maxpendingstarts": "5"}) + assert updated.status_code == 200, updated.get_json() + assert updated.get_json() == { + "Result": "Updated", "StartOnDemandEnabled": False, "MaxPendingStarts": 5, "ZeroMinimumCount": 0, + "message": "Start on demand is off. A user who finds no ready host is refused.", + } + policy = client.get("/api/scaling/policy").get_json() + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["ZeroMinimumCount"]) == (False, 5, 0) + + again = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert again["Result"] == "Unchanged" + + both = client.post("/api/scaling/policy/update", json={"timezone": "Tokyo Standard Time", "startondemandenabled": True}) + assert both.status_code == 200 and both.get_json()["Result"] == "Updated" + assert both.get_json()["message"] == ("Schedules are now read in Tokyo Standard Time. " + "Start on demand is on. Up to 5 hosts may start at once for waiting users.") + + refused = client.post("/api/scaling/policy/update", json={"timezone": "Nowhere", "startondemandenabled": False}) + assert refused.status_code == 400 + assert client.post("/api/scaling/policy/update", json={"maxpendingstarts": 21}).status_code == 400 + policy = client.get("/api/scaling/policy").get_json() + assert (policy["TimeZone"], policy["StartOnDemandEnabled"], policy["MaxPendingStarts"]) == ("Tokyo Standard Time", True, 5) + + # The database refusing the start on demand half rolls back the time zone set before it. + monkeypatch.setattr(app_module, "_policy_update_fields", lambda body: {"timezone": "UTC", "maxpendingstarts": 99}) + invalid = client.post("/api/scaling/policy/update", json={}) + assert invalid.status_code == 400 and invalid.get_json()["error"] == "maxpendingstarts must be between 1 and 20." + assert client.get("/api/scaling/policy").get_json()["TimeZone"] == "Tokyo Standard Time" + + +def test_a_minimum_of_zero_needs_start_on_demand(client, db, compute): + rule_id = client.get("/api/scaling/rules").get_json()[0]["RuleID"] + assert client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0}).status_code == 200 + assert client.get("/api/scaling/policy").get_json()["ZeroMinimumCount"] == 1 + + off = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert off["message"] == ("Start on demand is off. 1 scaling rule or window has a minimum of 0, " + "so scaling keeps one host running for them.") + preview = client.get("/api/scaling/preview").get_json() + assert preview["Phase"]["MinVMs"] == 1 and preview["StartOnDemandEnabled"] is False + + window = {"name": "Nights", "days": EVERY_DAY, "start": "20:00", "end": "06:00", **RULE, "minvms": 0} + refused = client.post("/api/scaling/schedules/create", json=window) + assert refused.status_code == 400 and refused.get_json()["error"] == "minvms can be 0 only while start on demand is on." + refused_rule = client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0, "maxvms": 8}) + assert refused_rule.status_code == 400 + + assert client.post("/api/scaling/policy/update", json={"startondemandenabled": True}).status_code == 200 + assert client.post("/api/scaling/schedules/create", json=window).status_code == 201 + assert client.get("/api/scaling/policy").get_json()["ZeroMinimumCount"] == 2 + + +def test_scale_to_zero_stops_the_last_idle_host_and_the_next_user_starts_it(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", PowerStateChangedDate="2000-01-01") + rule_id = client.get("/api/scaling/rules").get_json()[0]["RuleID"] + assert client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0}).status_code == 200 + + scaled = client.post("/api/scaling/trigger", json={}) + + assert scaled.status_code == 200, scaled.get_json() + assert scaled.get_json()["PoweredOffVMs"] == ["lnxhost-01"] + assert compute.operations == [("power_off", "lnxhost-01")] + assert db.vm("lnxhost-01")["PowerState"] == "Off" + + compute.states["lnxhost-01"] = "PowerState/stopped" + response = _checkout(client, "alice") + assert response.status_code == 202 and response.get_json()["reason"] == "Started" + assert compute.operations[-1] == ("start", "lnxhost-01") + + # While alice waits, scaling keeps the host she is waiting for. + compute.states["lnxhost-01"] = "PowerState/running" + again = client.post("/api/scaling/trigger", json={}) + assert again.status_code == 200 and again.get_json()["PoweredOffVMs"] == [] + assert db.vm("lnxhost-01")["PowerState"] == "On" + + +def test_the_policy_reports_which_broker_script_each_avd_host_runs(client, db, remote): + db.run("UPDATE dbo.ScalingPolicy SET StartOnDemandEnabled = 0") + assert _checkout(client, "alice", "avd-01", clientVersion="2.0.0").status_code == 409 + assert _checkout(client, "bob", "avd-02").status_code == 409 + assert _checkout(client, "carol", "avd-03", clientVersion="1.9.0").status_code == 409 + assert _checkout(client, "dave", "avd-04", clientVersion="not a version!").status_code == 409 + + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + + assert scripts == { + "Seen": 4, "Outdated": 2, "OutdatedHostnames": ["avd-02", "avd-04"], + "Versions": [{"ClientVersion": "1.9.0", "AvdHosts": 1, "Current": False}, + {"ClientVersion": "2.0.0", "AvdHosts": 1, "Current": True}], + "CurrentVersion": "2.0.0", + } + + # Only each AVD host's latest checkout counts, so an updated host stops being outdated. + assert _checkout(client, "bob", "avd-02", clientVersion="2.0.0").status_code == 409 + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + assert scripts["Outdated"] == 1 and scripts["OutdatedHostnames"] == ["avd-04"] + assert {"ClientVersion": "2.0.0", "AvdHosts": 2, "Current": True} in scripts["Versions"] diff --git a/avd_host/broker/Connect-LinuxBroker.ps1 b/avd_host/broker/Connect-LinuxBroker.ps1 index b191745..11e7e6c 100644 --- a/avd_host/broker/Connect-LinuxBroker.ps1 +++ b/avd_host/broker/Connect-LinuxBroker.ps1 @@ -1,26 +1,63 @@ +<# +.SYNOPSIS + Checks out a Linux host from the Linux Broker and opens a Remote Desktop session to it. + +.DESCRIPTION + The Linux Desktop RemoteApp runs this script on the AVD session host, in the user's session. + The session host's managed identity authenticates it to the broker API. + + When no host is ready, the broker can start one and answer 202. The script then shows a + window saying the desktop is starting, with Cancel, and asks again when the broker says to, + until a host is ready, the user cancels, or MaxWaitSeconds pass. + + Configure-AVD-Host.ps1 installs the script at C:\Temp\Connect-LinuxBroker.ps1 when the + session host is provisioned, and deploy/Update-AvdHostBrokerScript.ps1 replaces it on + existing session hosts. Both fill in the broker API's URL and client ID. +#> param ( [Parameter(Mandatory = $false, HelpMessage = "Only 'desktop' is supported, which opens a Remote Desktop session to a Linux host. Any other value opens the desktop too.")] - [string]$Mode = "desktop" + [string]$Mode = "desktop", + + [Parameter(Mandatory = $false, HelpMessage = "How long to wait, in seconds, for a Linux host that the broker starts because none is ready. 0 does not wait.")] + [ValidateRange(0, 3600)] + [int]$MaxWaitSeconds = 600 ) -$ProgressPreference = 'SilentlyContinue' +# Sent with every checkout, so the scaling policy page can list the session hosts whose script +# cannot wait for a host to start. AVD_HOST_SCRIPT_VERSION in api/config.py must match: bump +# both with any change to this script. +$ScriptVersion = '2.0.0' -# Automatically collect local Windows hostname and username -$localHostname = $env:COMPUTERNAME -$localUsername = $env:USERNAME -replace '[^a-zA-Z0-9_]', '' +$ProgressPreference = 'SilentlyContinue' -# Define the API endpoints +# Filled in when the script is installed on the session host. $apiBaseUrl = "https://your_linuxbroker_api_base_url/api" -$checkoutVmUrl = "$apiBaseUrl/vms/checkout" - -# Define the maximum number of update attempts -$maxAttempts = 3 -$attemptCount = 0 -$hasExistingCheckedInVM = $false +$apiAppIdUri = "api://your_linuxbroker_api_client_id" $sourceName = "LinuxBrokerScript" # The source name for your event log. $logName = "Application" # The log where your source will write events. Commonly "Application". +# A checkout that provisions the user on the Linux host can take a while. +$RequestTimeoutSeconds = 120 +# A request that gets no answer, 408, 429 or a 5xx is tried again, until this many fail in a row. +$MaxTransientFailures = 3 +$TransientRetrySeconds = 5 +# The broker counts a user as waiting for five minutes after its last 202, so the script always +# asks again well within that. +$MinRetrySeconds = 5 +$MaxRetrySeconds = 120 +# Used when a 202 does not say how long to wait: 30, 30, then 60 seconds from then on. +$FallbackRetrySeconds = @(30, 30, 60) + +$OutcomeMessages = @{ + NoHost = @{ Text = "No Linux host is available right now. Try again in a few minutes."; Icon = "Warning" } + Starting = @{ Text = "A Linux host is starting for you. Try again in a few minutes."; Icon = "Information" } + TimedOut = @{ Text = "Your Linux desktop is taking longer than usual to start. Try again in a few minutes."; Icon = "Warning" } + AuthFailed = @{ Text = "The Linux Broker could not authenticate this session host. Contact your administrator."; Icon = "Error" } + Unavailable = @{ Text = "The Linux Broker is not responding. Try again in a few minutes, or contact your administrator."; Icon = "Error" } + Rejected = @{ Text = "The Linux Broker could not give you a Linux host. Contact your administrator."; Icon = "Error" } +} + function Write-Log { param ( [string]$Message, @@ -36,10 +73,11 @@ function Write-Log { } try { - Write-EventLog -LogName $logName -Source $sourceName -EntryType $entryType -EventId 1 -Message $Message + Write-EventLog -LogName $logName -Source $sourceName -EntryType $entryType -EventId 1 -Message $Message -ErrorAction Stop } catch { # Logging must never stop the connection attempt. + $null = $_ } } @@ -91,75 +129,492 @@ function Get-AccessToken { } } -# Earlier releases kept every other value for starting a single application through xpra, -# which was never implemented and has been removed. A RemoteApp that still passes one gets the -# desktop rather than nothing. -if ($Mode -ine "desktop") { - Write-Log "Mode '$Mode' is not supported, so the desktop is opened instead." "WARNING" +# Seconds on a clock that is not moved by changes to the system time. +function Get-MonotonicTime { + return [System.Diagnostics.Stopwatch]::GetTimestamp() / [double][System.Diagnostics.Stopwatch]::Frequency } -# Define the API's Application ID URI (use the updated valid URL) -$apiAppIdUri = "api://your_linuxbroker_api_client_id" # Replace with your API's actual Application ID URI +# The whole number of seconds in a Retry-After value, or $null when it holds none. +function ConvertTo-RetryAfter { + param ( + $Value + ) -# Obtain the access token using Managed Identity -$accessToken = Get-AccessToken -Resource $apiAppIdUri + if ($Value -is [array]) { + $Value = $Value | Select-Object -First 1 + } + if ($null -eq $Value) { + return $null + } -if (-not $accessToken) { - Write-Log "Unable to obtain access token. Exiting script." "ERROR" - Show-UserMessage "The Linux Broker could not authenticate this session host. Contact your administrator." "Error" - exit 1 + $seconds = 0 + if ([int]::TryParse(([string]$Value).Trim(), [ref]$seconds)) { + return $seconds + } + return $null } -# Prepare the Authorization header -$authHeader = @{ - "Authorization" = "Bearer $accessToken" +# How long to wait before asking again: what the broker asked for, within limits, or the +# fallback schedule when it asked for nothing. +function Get-RetryDelay { + param ( + $RetryAfter, + [int]$WaitCount = 0 + ) + + if ($null -ne $RetryAfter) { + return [int][Math]::Min($MaxRetrySeconds, [Math]::Max($MinRetrySeconds, [int]$RetryAfter)) + } + + $index = [Math]::Min([Math]::Max($WaitCount, 0), $FallbackRetrySeconds.Count - 1) + return [int]$FallbackRetrySeconds[$index] } -# Attempt to get an available VM or a checked-out VM -while ($attemptCount -lt $maxAttempts) { - $attemptCount++ - Write-Log "Attempt $attemptCount of $maxAttempts Checking for an available or already checked-out VM..." "INFO" +function Test-TransientFailure { + param ( + [int]$StatusCode + ) + return ($StatusCode -eq 0 -or $StatusCode -eq 408 -or $StatusCode -eq 429 -or ($StatusCode -ge 500 -and $StatusCode -le 599)) +} + +function New-BrokerHttpClient { + Add-Type -AssemblyName System.Net.Http + $client = New-Object System.Net.Http.HttpClient + $client.Timeout = [TimeSpan]::FromSeconds($RequestTimeoutSeconds) + $client.DefaultRequestHeaders.ExpectContinue = $false + return $client +} + +# A small window that tells the user their desktop is starting, with Cancel. It is modeless: +# the script keeps working and pumps its messages. Returns $null when it cannot be shown, and +# the wait goes on without it. +function Open-WaitWindow { try { - # Prepare the payload - $checkoutPayload = @{ - "username" = $localUsername - "avdhost" = $localHostname + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + [System.Windows.Forms.Application]::EnableVisualStyles() + + $state = [pscustomobject]@{ + Form = $null + Cancelled = $false + Closing = $false } - # Invoke the API to checkout a VM with authentication - Write-Log "Attempting to checkout a VM via API with Managed Identity authentication..." "INFO" - $checkoutResponse = Invoke-RestMethod -Uri $checkoutVmUrl -Method POST ` - -ContentType "application/json" ` - -Body ($checkoutPayload | ConvertTo-Json) ` - -Headers $authHeader - - if ($checkoutResponse.VMID) { - $hasExistingCheckedInVM = $true - Write-Log "Successfully checked out or retrieved an existing VM (VMID: $($checkoutResponse.VMID), Hostname: $($checkoutResponse.Hostname))." "INFO" - break + $form = New-Object System.Windows.Forms.Form + $form.Text = "Linux Desktop" + $form.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::FixedDialog + $form.StartPosition = [System.Windows.Forms.FormStartPosition]::CenterScreen + $form.MaximizeBox = $false + $form.ShowInTaskbar = $true + $form.AutoSize = $true + $form.AutoSizeMode = [System.Windows.Forms.AutoSizeMode]::GrowAndShrink + $form.Font = [System.Drawing.SystemFonts]::MessageBoxFont + + $panel = New-Object System.Windows.Forms.FlowLayoutPanel + $panel.FlowDirection = [System.Windows.Forms.FlowDirection]::TopDown + $panel.WrapContents = $false + $panel.AutoSize = $true + $panel.AutoSizeMode = [System.Windows.Forms.AutoSizeMode]::GrowAndShrink + $panel.Padding = New-Object System.Windows.Forms.Padding -ArgumentList 16 + + $heading = New-Object System.Windows.Forms.Label + $heading.AutoSize = $true + $heading.MaximumSize = New-Object System.Drawing.Size -ArgumentList 360, 0 + $heading.Font = New-Object System.Drawing.Font -ArgumentList $form.Font, ([System.Drawing.FontStyle]::Bold) + $heading.Text = "Your Linux desktop is starting" + [char]0x2026 + + $detail = New-Object System.Windows.Forms.Label + $detail.AutoSize = $true + $detail.MaximumSize = New-Object System.Drawing.Size -ArgumentList 360, 0 + $detail.Margin = New-Object System.Windows.Forms.Padding -ArgumentList 3, 8, 3, 12 + $detail.Text = "This can take a few minutes. It opens by itself when it is ready." + + $progress = New-Object System.Windows.Forms.ProgressBar + $progress.Style = [System.Windows.Forms.ProgressBarStyle]::Marquee + $progress.MarqueeAnimationSpeed = 30 + $progress.Width = 360 + $progress.AccessibleName = "Waiting for your Linux desktop" + + $cancel = New-Object System.Windows.Forms.Button + $cancel.Text = "Cancel" + $cancel.AutoSize = $true + $cancel.Anchor = [System.Windows.Forms.AnchorStyles]::Right + $cancel.Margin = New-Object System.Windows.Forms.Padding -ArgumentList 3, 12, 3, 3 + $cancel.Add_Click({ $state.Cancelled = $true; $state.Form.Close() }.GetNewClosure()) + + $panel.Controls.AddRange(@($heading, $detail, $progress, $cancel)) + $form.Controls.Add($panel) + $form.CancelButton = $cancel + # Closing the window any way but from the script cancels the wait too. + $form.Add_FormClosing({ if (-not $state.Closing) { $state.Cancelled = $true } }.GetNewClosure()) + + $state.Form = $form + $form.Show() + $form.Activate() + [System.Windows.Forms.Application]::DoEvents() + return $state + } + catch { + Write-Log "The wait window could not be shown: $_" "WARNING" + return $null + } +} + +function Close-WaitWindow { + param ( + $WaitWindow + ) + + if ($null -eq $WaitWindow -or $null -eq $WaitWindow.Form) { + return + } + + try { + $WaitWindow.Closing = $true + if (-not $WaitWindow.Form.IsDisposed) { + $WaitWindow.Form.Close() + $WaitWindow.Form.Dispose() } - else { - Write-Log "Failed to retrieve VM information from API response." "WARNING" + } + catch { + Write-Log "The wait window could not be closed: $_" "WARNING" + } +} + +# Handles the wait window's pending messages, so it repaints and Cancel works. +function Invoke-WindowMessagePump { + param ( + $WaitWindow + ) + + if ($null -eq $WaitWindow) { + return + } + + try { + [System.Windows.Forms.Application]::DoEvents() + } + catch { + Write-Log "The wait window stopped responding: $_" "WARNING" + } +} + +# Waits the given number of seconds, keeping the wait window responsive. Returns $false as soon +# as the user cancels. +function Wait-CheckoutInterval { + param ( + [double]$Seconds, + $WaitWindow + ) + + $until = (Get-MonotonicTime) + $Seconds + while ((Get-MonotonicTime) -lt $until) { + if ($null -ne $WaitWindow -and $WaitWindow.Cancelled) { + return $false } + Invoke-WindowMessagePump -WaitWindow $WaitWindow + Start-Sleep -Milliseconds 100 + } + + return -not ($null -ne $WaitWindow -and $WaitWindow.Cancelled) +} + +# Posts one checkout request. Returns its status code (0 when no answer came), the parsed body, +# the wait a Retry-After asks for, and the error. While the wait window is open it stays +# responsive, and Cancel abandons the request. +function Invoke-BrokerCheckout { + param ( + [Parameter(Mandatory = $true)] + $Client, + + [Parameter(Mandatory = $true)] + [string]$Url, + + [Parameter(Mandatory = $true)] + [string]$AccessToken, + + [Parameter(Mandatory = $true)] + [hashtable]$Body, + + $WaitWindow + ) + + $result = [pscustomobject]@{ + StatusCode = 0 + Body = $null + RetryAfter = $null + Error = $null + Cancelled = $false + } + + $request = $null + $response = $null + $cancellation = New-Object System.Threading.CancellationTokenSource + try { + $request = New-Object System.Net.Http.HttpRequestMessage -ArgumentList ([System.Net.Http.HttpMethod]::Post), $Url + $request.Headers.Authorization = New-Object System.Net.Http.Headers.AuthenticationHeaderValue -ArgumentList "Bearer", $AccessToken + $request.Content = New-Object System.Net.Http.StringContent -ArgumentList ($Body | ConvertTo-Json -Compress), ([System.Text.Encoding]::UTF8), "application/json" + + $task = $Client.SendAsync($request, $cancellation.Token) + while (-not $task.IsCompleted) { + if ($null -ne $WaitWindow -and $WaitWindow.Cancelled) { + $cancellation.Cancel() + $result.Cancelled = $true + return $result + } + Invoke-WindowMessagePump -WaitWindow $WaitWindow + try { + [void]$task.Wait(100) + } + catch { + # A failed request is reported from the task below. + $null = $_ + } + } + + if ($task.IsCanceled) { + $result.Error = "The broker did not answer within $RequestTimeoutSeconds seconds." + return $result + } + if ($task.IsFaulted) { + $result.Error = $task.Exception.GetBaseException().Message + return $result + } + + $response = $task.Result + $result.StatusCode = [int]$response.StatusCode + $retryHeader = $response.Headers.RetryAfter + if ($null -ne $retryHeader -and $null -ne $retryHeader.Delta) { + $result.RetryAfter = [int][Math]::Ceiling($retryHeader.Delta.TotalSeconds) + } + + $text = $response.Content.ReadAsStringAsync().Result + if (-not [string]::IsNullOrWhiteSpace($text)) { + try { + $result.Body = $text | ConvertFrom-Json + } + catch { + $result.Error = "The broker's answer was not JSON." + } + } + + if ($null -ne $result.Body) { + $bodyRetryAfter = ConvertTo-RetryAfter $result.Body.retryAfterSeconds + if ($null -ne $bodyRetryAfter) { + $result.RetryAfter = $bodyRetryAfter + } + if ($result.Body.error) { + $result.Error = [string]$result.Body.error + } + } + + return $result } catch { - Write-Log "API request failed: $_" "ERROR" + $result.Error = $_.Exception.Message + return $result + } + finally { + if ($null -ne $response) { + $response.Dispose() + } + if ($null -ne $request) { + $request.Dispose() + } + $cancellation.Dispose() + } +} + +function New-CheckoutResult { + param ( + [Parameter(Mandatory = $true)] + [ValidateSet("Assigned", "NoHost", "Starting", "TimedOut", "Cancelled", "AuthFailed", "Unavailable", "Rejected")] + [string]$Outcome, + + $Checkout = $null + ) + + return [pscustomobject]@{ + Outcome = $Outcome + Checkout = $Checkout + } +} + +# Checks out a Linux host for the user. While the broker answers 202 because a host is starting, +# shows the wait window and asks again when the broker says to, until a host is ready, the user +# cancels, or MaxWaitSeconds pass. Returns the outcome, with the checkout when a host was assigned. +function Request-LinuxHost { + param ( + [Parameter(Mandatory = $true)] + [string]$CheckoutUrl, + + [Parameter(Mandatory = $true)] + [string]$Resource, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$AvdHost, + + [int]$MaxWaitSeconds = 600 + ) + + $accessToken = Get-AccessToken -Resource $Resource + if (-not $accessToken) { + Write-Log "Unable to obtain access token. Exiting script." "ERROR" + return New-CheckoutResult -Outcome AuthFailed + } + + $body = @{ + username = $Username + avdhost = $AvdHost + clientVersion = $ScriptVersion + } + + $client = New-BrokerHttpClient + $waitWindow = $null + $deadline = $null + $waits = 0 + $failures = 0 + $tokenRenewed = $false + $attempt = 0 + + try { + while ($true) { + $attempt++ + Write-Log "Checkout attempt $attempt for $Username from $AvdHost." "INFO" + $response = Invoke-BrokerCheckout -Client $client -Url $CheckoutUrl -AccessToken $accessToken -Body $body -WaitWindow $waitWindow + + if ($response.Cancelled) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + + $status = $response.StatusCode + $checkout = $response.Body + + if ($status -eq 200) { + if ($null -ne $checkout -and $checkout.VMID -and $checkout.IPAddress) { + Write-Log "Successfully checked out or retrieved an existing VM (VMID: $($checkout.VMID), Hostname: $($checkout.Hostname))." "INFO" + return New-CheckoutResult -Outcome Assigned -Checkout $checkout + } + Write-Log "The checkout answered without a VMID or IP address." "ERROR" + return New-CheckoutResult -Outcome Rejected + } + + if ($status -eq 202) { + $failures = 0 + Write-Log "No Linux host is ready yet ($($checkout.reason)): $($checkout.message)" "INFO" + if ($MaxWaitSeconds -le 0) { + return New-CheckoutResult -Outcome Starting + } + + $now = Get-MonotonicTime + if ($null -eq $deadline) { + $deadline = $now + $MaxWaitSeconds + } + $remaining = $deadline - $now + if ($remaining -le 0) { + Write-Log "No Linux host was ready within $MaxWaitSeconds seconds." "WARNING" + return New-CheckoutResult -Outcome TimedOut + } + + $delay = [Math]::Min((Get-RetryDelay -RetryAfter $response.RetryAfter -WaitCount $waits), [Math]::Ceiling($remaining)) + $waits++ + if ($null -eq $waitWindow) { + $waitWindow = Open-WaitWindow + } + Write-Log "Asking again in $delay seconds." "INFO" + if (-not (Wait-CheckoutInterval -Seconds $delay -WaitWindow $waitWindow)) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + continue + } + + if ($status -eq 409) { + Write-Log "No available or checked-out VM found: $($response.Error)" "WARNING" + return New-CheckoutResult -Outcome NoHost + } + + # A token can expire while the user waits, so a refused one is renewed once. + if ($status -eq 401 -and -not $tokenRenewed) { + $tokenRenewed = $true + Write-Log "The broker did not accept the access token, so a new one is requested." "WARNING" + $accessToken = Get-AccessToken -Resource $Resource + if ($accessToken) { + continue + } + return New-CheckoutResult -Outcome AuthFailed + } + + if ($status -eq 401 -or $status -eq 403) { + Write-Log "The broker refused this session host ($status): $($response.Error)" "ERROR" + return New-CheckoutResult -Outcome AuthFailed + } + + if (Test-TransientFailure -StatusCode $status) { + $failures++ + Write-Log "API request failed ($status): $($response.Error)" "ERROR" + if ($failures -ge $MaxTransientFailures) { + return New-CheckoutResult -Outcome Unavailable + } + + $delay = $TransientRetrySeconds * $failures + if ($null -ne $response.RetryAfter) { + $delay = Get-RetryDelay -RetryAfter $response.RetryAfter + } + if ($null -ne $deadline) { + $remaining = $deadline - (Get-MonotonicTime) + if ($remaining -le 0) { + Write-Log "No Linux host was ready within $MaxWaitSeconds seconds." "WARNING" + return New-CheckoutResult -Outcome TimedOut + } + $delay = [Math]::Min($delay, [Math]::Ceiling($remaining)) + } + if (-not (Wait-CheckoutInterval -Seconds $delay -WaitWindow $waitWindow)) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + continue + } + + Write-Log "The broker refused the checkout ($status): $($response.Error)" "ERROR" + return New-CheckoutResult -Outcome Rejected + } + } + finally { + Close-WaitWindow -WaitWindow $waitWindow + if ($client -is [System.IDisposable]) { + $client.Dispose() + } } } -# If a VM was checked out or found, connect to it -if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { - $hostname = $checkoutResponse.Hostname - $ipAddress = $checkoutResponse.IPAddress +function Save-LinuxHostCredential { + param ( + [Parameter(Mandatory = $true)] + $Checkout, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username + ) + + $hostname = $Checkout.Hostname + $ipAddress = $Checkout.IPAddress # Store or update credentials in Credential Manager try { Write-Log "Updating Windows Credential Manager with credentials for $hostname..." "INFO" - + # Delete all existing credentials in Credential Manager Write-Log "Deleting all existing credentials in Credential Manager..." "INFO" - + cmdkey /list | ForEach-Object { if ($_ -match "Target: (.+)") { $target = $matches[1] @@ -168,20 +623,30 @@ if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { } } - New-StoredCredential -Target $hostname -UserName $localUsername -Password $checkoutResponse.password -Persist LocalMachine | Out-Null - New-StoredCredential -Target $ipAddress -UserName $localUsername -Password $checkoutResponse.password -Persist LocalMachine | Out-Null + New-StoredCredential -Target $hostname -UserName $Username -Password $Checkout.Password -Persist LocalMachine | Out-Null + New-StoredCredential -Target $ipAddress -UserName $Username -Password $Checkout.Password -Persist LocalMachine | Out-Null # mstsc only reuses a saved credential whose target carries the TERMSRV/ prefix. - New-StoredCredential -Target "TERMSRV/$ipAddress" -UserName $localUsername -Password $checkoutResponse.password -Type Generic -Persist LocalMachine | Out-Null - New-StoredCredential -Target "TERMSRV/$hostname" -UserName $localUsername -Password $checkoutResponse.password -Type Generic -Persist LocalMachine | Out-Null + New-StoredCredential -Target "TERMSRV/$ipAddress" -UserName $Username -Password $Checkout.Password -Type Generic -Persist LocalMachine | Out-Null + New-StoredCredential -Target "TERMSRV/$hostname" -UserName $Username -Password $Checkout.Password -Type Generic -Persist LocalMachine | Out-Null Write-Log "Credentials for $hostname updated successfully in Credential Manager." "INFO" } catch { Write-Log "Failed to update credentials in Credential Manager: $_" "ERROR" } +} + +function Open-RemoteDesktop { + param ( + [Parameter(Mandatory = $true)] + [string]$Hostname, + + [Parameter(Mandatory = $true)] + [string]$IPAddress + ) - Write-Log "Connecting to $hostname (IP: $ipAddress) using Remote Desktop Connection..." "INFO" + Write-Log "Connecting to $Hostname (IP: $IPAddress) using Remote Desktop Connection..." "INFO" try { # xrdp presents a self-signed certificate, so skip the server authentication warning for this user. $rdpClientKey = "HKCU:\Software\Microsoft\Terminal Server Client" @@ -190,17 +655,71 @@ if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { } New-ItemProperty -Path $rdpClientKey -Name "AuthenticationLevelOverride" -PropertyType DWord -Value 0 -Force | Out-Null - # Launch mstsc with the hostname or IP address - Start-Process mstsc.exe -ArgumentList "/v:$ipAddress" + # Launch mstsc with the IP address + Start-Process mstsc.exe -ArgumentList "/v:$IPAddress" -ErrorAction Stop - Write-Log "Successfully connected to $hostname (IP: $ipAddress) using Remote Desktop Connection." "INFO" + Write-Log "Successfully connected to $Hostname (IP: $IPAddress) using Remote Desktop Connection." "INFO" + return $true } catch { - Write-Log "Failed to connect to $hostname (IP: $ipAddress) using Remote Desktop Connection: $_" "ERROR" - Show-UserMessage "Remote Desktop Connection could not be started for $hostname. Try again, or contact your administrator." "Error" + Write-Log "Failed to connect to $Hostname (IP: $IPAddress) using Remote Desktop Connection: $_" "ERROR" + Show-UserMessage "Remote Desktop Connection could not be started for $Hostname. Try again, or contact your administrator." "Error" + return $false + } +} + +# Returns the exit code: 0 when the desktop opened or the user cancelled, 1 otherwise. +function Invoke-ConnectLinuxBroker { + param ( + [string]$Mode = "desktop", + + [int]$MaxWaitSeconds = 600, + + [Parameter(Mandatory = $true)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $true)] + [string]$Resource, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$AvdHost + ) + + # Earlier releases kept every other value for starting a single application through xpra, + # which was never implemented and has been removed. A RemoteApp that still passes one gets the + # desktop rather than nothing. + if ($Mode -ine "desktop") { + Write-Log "Mode '$Mode' is not supported, so the desktop is opened instead." "WARNING" } + + Write-Log "Connect-LinuxBroker.ps1 $ScriptVersion is checking out a Linux host for $Username." "INFO" + $result = Request-LinuxHost -CheckoutUrl "$ApiBaseUrl/vms/checkout" -Resource $Resource -Username $Username -AvdHost $AvdHost -MaxWaitSeconds $MaxWaitSeconds + + if ($result.Outcome -eq "Assigned") { + Save-LinuxHostCredential -Checkout $result.Checkout -Username $Username + if (Open-RemoteDesktop -Hostname $result.Checkout.Hostname -IPAddress $result.Checkout.IPAddress) { + return 0 + } + return 1 + } + + if ($result.Outcome -eq "Cancelled") { + return 0 + } + + $message = $OutcomeMessages[$result.Outcome] + Show-UserMessage $message.Text $message.Icon + return 1 } -else { - Write-Log "No available or checked-out VM found. Exiting script." "WARNING" - Show-UserMessage "No Linux host is available right now. Try again in a few minutes." "Warning" + +# Dot-sourcing the script, as its tests do, defines the functions without connecting. +if ($MyInvocation.InvocationName -ne '.') { + $exitCode = Invoke-ConnectLinuxBroker -Mode $Mode -MaxWaitSeconds $MaxWaitSeconds -ApiBaseUrl $apiBaseUrl -Resource $apiAppIdUri ` + -Username ($env:USERNAME -replace '[^a-zA-Z0-9_]', '') -AvdHost $env:COMPUTERNAME + exit ([int]($exitCode | Select-Object -Last 1)) } diff --git a/avd_host/tests/Connect-LinuxBroker.Tests.ps1 b/avd_host/tests/Connect-LinuxBroker.Tests.ps1 new file mode 100644 index 0000000..9a9a3a9 --- /dev/null +++ b/avd_host/tests/Connect-LinuxBroker.Tests.ps1 @@ -0,0 +1,776 @@ +# Pester 5 tests for avd_host/broker/Connect-LinuxBroker.ps1, which runs in Windows PowerShell 5.1 +# on the AVD session hosts. From the repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:BrokerScriptPath = (Resolve-Path (Join-Path $PSScriptRoot '..\broker\Connect-LinuxBroker.ps1')).Path + . $script:BrokerScriptPath + + Add-Type -AssemblyName System.Net.Http + if (-not ('LinuxBrokerTests.StubHandler' -as [type])) { + Add-Type -ReferencedAssemblies System.Net.Http -TypeDefinition @' +namespace LinuxBrokerTests +{ + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + + // Answers every request HttpClient sends with the response it is set up with. + public class StubHandler : HttpMessageHandler + { + public HttpStatusCode StatusCode = HttpStatusCode.OK; + public string ResponseBody = ""; + public string RetryAfter; + public string Failure; + public bool Cancel; + public bool Hang; + public int DelayMilliseconds; + public int Calls; + public HttpRequestMessage LastRequest; + public string LastBody; + public string LastContentType; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Calls++; + LastRequest = request; + if (request.Content != null) + { + LastBody = request.Content.ReadAsStringAsync().Result; + LastContentType = request.Content.Headers.ContentType.MediaType; + } + + var source = new TaskCompletionSource(); + if (Hang) + { + return source.Task; + } + if (Failure != null) + { + source.SetException(new HttpRequestException(Failure)); + return source.Task; + } + if (Cancel) + { + source.SetCanceled(); + return source.Task; + } + + var response = new HttpResponseMessage(StatusCode); + response.Content = new StringContent(ResponseBody ?? "", System.Text.Encoding.UTF8, "application/json"); + if (RetryAfter != null) + { + response.Headers.TryAddWithoutValidation("Retry-After", RetryAfter); + } + response.RequestMessage = request; + if (DelayMilliseconds > 0) + { + return Task.Delay(DelayMilliseconds).ContinueWith(delay => response); + } + source.SetResult(response); + return source.Task; + } + } +} +'@ + } + + function New-TestResponse { + param( + [int]$StatusCode, + $Body = $null, + $RetryAfter = $null, + [string]$Message = $null, + [switch]$Cancelled + ) + + [pscustomobject]@{ + StatusCode = $StatusCode + Body = $Body + RetryAfter = $RetryAfter + Error = $Message + Cancelled = [bool]$Cancelled + } + } + + function New-TestCheckout { + [pscustomobject]@{ VMID = 5; Hostname = 'lnx-05'; IPAddress = '10.0.0.5'; LeaseId = 'lease'; Password = 'from-the-broker' } + } + + function New-TestStarting { + param($RetryAfter = 60, [string]$Reason = 'Started') + + New-TestResponse -StatusCode 202 -RetryAfter $RetryAfter -Body ([pscustomobject]@{ + status = 'Starting' + reason = $Reason + retryAfterSeconds = $RetryAfter + message = "A Linux host is starting for you. Ask again in $RetryAfter seconds." + }) + } +} + +Describe 'Connect-LinuxBroker.ps1' { + BeforeAll { + $script:Content = [System.IO.File]::ReadAllText($script:BrokerScriptPath) + } + + It 'is plain ASCII, because it is installed without a byte order mark and Windows PowerShell reads it as ANSI' { + $bytes = [System.IO.File]::ReadAllBytes($script:BrokerScriptPath) + @($bytes | Where-Object { $_ -gt 127 }).Count | Should -Be 0 + } + + It 'parses' { + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseFile($script:BrokerScriptPath, [ref]$null, [ref]$errors) + $errors | Should -BeNullOrEmpty + } + + # Configure-AVD-Host.ps1 replaces every occurrence of each, so any other mention would be + # rewritten too. + It 'names each value filled in at installation exactly once' { + [regex]::Matches($script:Content, [regex]::Escape('https://your_linuxbroker_api_base_url/api')).Count | Should -Be 1 + [regex]::Matches($script:Content, [regex]::Escape('your_linuxbroker_api_client_id')).Count | Should -Be 1 + } + + It 'takes the API URL and client ID the way Configure-AVD-Host.ps1 fills them in' { + $installed = ($script:Content -replace 'https://your_linuxbroker_api_base_url/api', 'https://broker.example/api').Replace( + 'your_linuxbroker_api_client_id', '11111111-2222-3333-4444-555555555555') + + $installed | Should -Match ([regex]::Escape('$apiBaseUrl = "https://broker.example/api"')) + $installed | Should -Match ([regex]::Escape('$apiAppIdUri = "api://11111111-2222-3333-4444-555555555555"')) + $installed | Should -Not -Match 'your_linuxbroker' + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($installed, [ref]$null, [ref]$errors) + $errors | Should -BeNullOrEmpty + } + + # The same rule as CLIENT_VERSION_RE and CLIENT_VERSION_MAX_CHARS in api/app.py. + It 'reports a version the broker records' { + $ScriptVersion | Should -Match '^[A-Za-z0-9][A-Za-z0-9._+-]*$' + $ScriptVersion.Length | Should -BeLessOrEqual 32 + } + + It 'waits ten minutes at most by default' { + $MaxWaitSeconds | Should -Be 600 + } +} + +Describe 'ConvertTo-RetryAfter' { + It 'reads as ' -TestCases @( + @{ Value = 45; Expected = 45 } + @{ Value = '45'; Expected = 45 } + @{ Value = ' 30 '; Expected = 30 } + @{ Value = @('75', '10'); Expected = 75 } + ) { + ConvertTo-RetryAfter $Value | Should -Be $Expected + } + + It 'returns nothing for ' -TestCases @( + @{ Value = $null } + @{ Value = '' } + @{ Value = 'soon' } + @{ Value = '4.5' } + @{ Value = @() } + ) { + ConvertTo-RetryAfter $Value | Should -BeNullOrEmpty + } +} + +Describe 'Get-RetryDelay' { + It 'waits what the broker asks for, from 5 to 120 seconds: gives ' -TestCases @( + @{ RetryAfter = 60; Expected = 60 } + @{ RetryAfter = 1; Expected = 5 } + @{ RetryAfter = 0; Expected = 5 } + @{ RetryAfter = 600; Expected = 120 } + ) { + Get-RetryDelay -RetryAfter $RetryAfter -WaitCount 3 | Should -Be $Expected + } + + It 'waits 30, 30, then 60 seconds when the broker does not say' { + 0..5 | ForEach-Object { Get-RetryDelay -RetryAfter $null -WaitCount $_ } | Should -Be @(30, 30, 60, 60, 60, 60) + } +} + +Describe 'Test-TransientFailure' { + It 'retries ' -TestCases @( + @{ StatusCode = 0 }, @{ StatusCode = 408 }, @{ StatusCode = 429 }, @{ StatusCode = 500 }, @{ StatusCode = 502 }, @{ StatusCode = 503 } + ) { + Test-TransientFailure -StatusCode $StatusCode | Should -BeTrue + } + + It 'does not retry ' -TestCases @( + @{ StatusCode = 200 }, @{ StatusCode = 202 }, @{ StatusCode = 400 }, @{ StatusCode = 401 }, @{ StatusCode = 403 }, @{ StatusCode = 404 }, @{ StatusCode = 409 } + ) { + Test-TransientFailure -StatusCode $StatusCode | Should -BeFalse + } +} + +Describe 'Invoke-BrokerCheckout' { + BeforeEach { + Mock Write-Log {} + Mock Invoke-WindowMessagePump {} + $handler = New-Object LinuxBrokerTests.StubHandler + $client = New-Object System.Net.Http.HttpClient -ArgumentList $handler + $checkoutRequest = @{ + Client = $client + Url = 'https://broker.example/api/vms/checkout' + AccessToken = 'token-1' + Body = @{ username = 'alice'; avdhost = 'avd-0'; clientVersion = '2.0.0' } + } + } + + AfterEach { + $client.Dispose() + } + + It 'posts the checkout as JSON with the bearer token' { + $handler.ResponseBody = '{"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5", "LeaseId": "lease"}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 200 + $response.Body.Hostname | Should -Be 'lnx-05' + $response.Body.IPAddress | Should -Be '10.0.0.5' + $response.Error | Should -BeNullOrEmpty + $response.Cancelled | Should -BeFalse + $handler.Calls | Should -Be 1 + $handler.LastRequest.Method.Method | Should -Be 'POST' + $handler.LastRequest.RequestUri.AbsoluteUri | Should -Be 'https://broker.example/api/vms/checkout' + $handler.LastRequest.Headers.Authorization.Scheme | Should -Be 'Bearer' + $handler.LastRequest.Headers.Authorization.Parameter | Should -Be 'token-1' + $handler.LastContentType | Should -Be 'application/json' + $sent = $handler.LastBody | ConvertFrom-Json + $sent.username | Should -Be 'alice' + $sent.avdhost | Should -Be 'avd-0' + $sent.clientVersion | Should -Be '2.0.0' + } + + It 'reads the wait from a 202 body' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Accepted + $handler.RetryAfter = '90' + $handler.ResponseBody = '{"status": "Starting", "reason": "Started", "retryAfterSeconds": 75, "message": "A Linux host is starting for you."}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 202 + $response.RetryAfter | Should -Be 75 + $response.Body.reason | Should -Be 'Started' + } + + It 'reads the wait from the Retry-After header when the body has none' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Accepted + $handler.RetryAfter = '40' + $handler.ResponseBody = '{"status": "Starting"}' + + (Invoke-BrokerCheckout @checkoutRequest).RetryAfter | Should -Be 40 + } + + It 'reports the error the broker gives' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Conflict + $handler.ResponseBody = '{"error": "No available VM found. Please try again."}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 409 + $response.Error | Should -Be 'No available VM found. Please try again.' + $response.RetryAfter | Should -BeNullOrEmpty + } + + It 'reports an answer that is not JSON' { + $handler.StatusCode = [System.Net.HttpStatusCode]::BadGateway + $handler.ResponseBody = 'Bad gateway' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 502 + $response.Body | Should -BeNullOrEmpty + $response.Error | Should -Match 'not JSON' + } + + It 'reports no status when the request fails' { + $handler.Failure = 'No such host is known.' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 0 + $response.Error | Should -Match 'No such host is known' + } + + It 'reports no status when the broker does not answer in time' { + $handler.Cancel = $true + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 0 + $response.Error | Should -Match 'did not answer within 120 seconds' + $response.Cancelled | Should -BeFalse + } + + It 'keeps the wait window responsive while the request runs' { + $handler.DelayMilliseconds = 400 + $handler.ResponseBody = '{"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5"}' + + $response = Invoke-BrokerCheckout @checkoutRequest -WaitWindow ([pscustomobject]@{ Cancelled = $false }) + + $response.StatusCode | Should -Be 200 + Should -Invoke Invoke-WindowMessagePump -Scope It + } + + It 'abandons the request when the user cancels' { + $handler.Hang = $true + + $response = Invoke-BrokerCheckout @checkoutRequest -WaitWindow ([pscustomobject]@{ Cancelled = $true }) + + $response.Cancelled | Should -BeTrue + $response.StatusCode | Should -Be 0 + } +} + +Describe 'Wait-CheckoutInterval' { + BeforeEach { + $script:Clock = 100.0 + Mock Get-MonotonicTime { $script:Clock } + Mock Start-Sleep { $script:Clock += $Milliseconds / 1000.0 } + Mock Invoke-WindowMessagePump {} + } + + It 'waits the whole interval, keeping the window responsive' { + Wait-CheckoutInterval -Seconds 2 -WaitWindow ([pscustomobject]@{ Cancelled = $false }) | Should -BeTrue + $script:Clock | Should -BeGreaterOrEqual 102 + $script:Clock | Should -BeLessThan 102.2 + Should -Invoke Invoke-WindowMessagePump -Scope It -Times 19 + } + + It 'stops as soon as the user cancels' { + $script:Window = [pscustomobject]@{ Cancelled = $false } + Mock Start-Sleep { + $script:Clock += $Milliseconds / 1000.0 + if ($script:Clock -ge 100.5) { $script:Window.Cancelled = $true } + } + + Wait-CheckoutInterval -Seconds 60 -WaitWindow $script:Window | Should -BeFalse + $script:Clock | Should -BeLessThan 101 + } + + It 'waits without a window' { + Wait-CheckoutInterval -Seconds 1 | Should -BeTrue + $script:Clock | Should -BeGreaterOrEqual 101 + } +} + +Describe 'Request-LinuxHost' { + BeforeEach { + $script:Responses = New-Object System.Collections.Queue + $script:Clock = 0.0 + $script:Waited = New-Object System.Collections.ArrayList + $script:Tokens = New-Object System.Collections.Queue + $script:Tokens.Enqueue('token-1') + $script:Tokens.Enqueue('token-2') + $script:Window = [pscustomobject]@{ Form = $null; Cancelled = $false; Closing = $false } + + Mock Write-Log {} + Mock Get-AccessToken { if ($script:Tokens.Count -gt 0) { $script:Tokens.Dequeue() } } + Mock New-BrokerHttpClient { [pscustomobject]@{ Name = 'client' } } + Mock Invoke-BrokerCheckout { $script:Responses.Dequeue() } + Mock Open-WaitWindow { $script:Window } + Mock Close-WaitWindow {} + Mock Get-MonotonicTime { $script:Clock } + Mock Wait-CheckoutInterval { + [void]$script:Waited.Add([double]$Seconds) + $script:Clock += $Seconds + $true + } + + $requestArguments = @{ + CheckoutUrl = 'https://broker.example/api/vms/checkout' + Resource = 'api://client' + Username = 'alice' + AvdHost = 'avd-0' + } + } + + It 'returns the host from a 200 without showing the wait window' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + $result = Request-LinuxHost @requestArguments + + $result.Outcome | Should -Be 'Assigned' + $result.Checkout.Hostname | Should -Be 'lnx-05' + Should -Invoke Get-AccessToken -Scope It -Times 1 -Exactly -ParameterFilter { $Resource -eq 'api://client' } + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly -ParameterFilter { + $Url -eq 'https://broker.example/api/vms/checkout' -and $AccessToken -eq 'token-1' -and + $Body.username -eq 'alice' -and $Body.avdhost -eq 'avd-0' -and $Body.clientVersion -eq $ScriptVersion + } + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + Should -Invoke Wait-CheckoutInterval -Scope It -Times 0 -Exactly + } + + It 'does not ask the broker without an access token' { + $script:Tokens.Clear() + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 0 -Exactly + } + + It 'shows the wait window and asks again when the broker says, until the host is ready' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 75)) + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30 -Reason 'AlreadyStarting')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + $result = Request-LinuxHost @requestArguments + + $result.Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(75, 30) + Should -Invoke Open-WaitWindow -Scope It -Times 1 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 3 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 2 -Exactly -ParameterFilter { $null -ne $WaitWindow } + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'waits 30, 30, then 60 seconds when the broker does not say how long' { + 1..5 | ForEach-Object { $script:Responses.Enqueue((New-TestResponse -StatusCode 202 -Body ([pscustomobject]@{ status = 'Starting' }))) } + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(30, 30, 60, 60, 60) + } + + It 'gives up after MaxWaitSeconds, asking one last time at the deadline' { + 1..10 | ForEach-Object { $script:Responses.Enqueue((New-TestStarting -RetryAfter 120)) } + + $result = Request-LinuxHost @requestArguments -MaxWaitSeconds 300 + + $result.Outcome | Should -Be 'TimedOut' + $script:Waited | Should -Be @(120, 120, 60) + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 4 -Exactly + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'asks again within two minutes, however long the broker asks it to wait' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 900)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(120) + } + + It 'does not wait when MaxWaitSeconds is 0' { + $script:Responses.Enqueue((New-TestStarting)) + + (Request-LinuxHost @requestArguments -MaxWaitSeconds 0).Outcome | Should -Be 'Starting' + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + Should -Invoke Wait-CheckoutInterval -Scope It -Times 0 -Exactly + } + + It 'stops when the user cancels the wait' { + $script:Responses.Enqueue((New-TestStarting)) + Mock Wait-CheckoutInterval { $false } + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Cancelled' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'stops when the user cancels during a request' { + $script:Responses.Enqueue((New-TestStarting)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 0 -Cancelled)) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Cancelled' + } + + It 'waits with no window when it cannot be shown' { + Mock Open-WaitWindow { $null } + $script:Responses.Enqueue((New-TestStarting -RetryAfter 45)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(45) + } + + It 'reports that no host is available on a 409' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 409 -Message 'No available VM found. Please try again.')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'NoHost' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } + + It 'renews a refused access token once' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 401 -Message 'Token is invalid.')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + Should -Invoke Get-AccessToken -Scope It -Times 2 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly -ParameterFilter { $AccessToken -eq 'token-2' } + } + + It 'gives up when the renewed token is refused too' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 401)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 401)) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 2 -Exactly + } + + It 'gives up when the session host is not allowed to check out' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 403 -Message 'Forbidden')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Get-AccessToken -Scope It -Times 1 -Exactly + } + + It 'retries a failed request after 5, then 10 seconds' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 500 -Message 'Unable to check out a virtual machine.')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 0 -Message 'Unable to connect to the remote server')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(5, 10) + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + } + + It 'reports the broker as unavailable after three failures in a row' { + 1..3 | ForEach-Object { $script:Responses.Enqueue((New-TestResponse -StatusCode 503 -Message 'The broker is busy. Please retry shortly.')) } + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Unavailable' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 3 -Exactly + $script:Waited | Should -Be @(5, 10) + } + + It 'honours a Retry-After on a failure' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 429 -RetryAfter 20)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(20) + } + + It 'counts failures in a row only' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(5, 10, 30, 5, 10) + } + + It 'does not retry failures past the wait deadline' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + + (Request-LinuxHost @requestArguments -MaxWaitSeconds 30).Outcome | Should -Be 'TimedOut' + $script:Waited | Should -Be @(30) + } + + It 'refuses a checkout that names no address' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body ([pscustomobject]@{ VMID = 5; Hostname = 'lnx-05'; IPAddress = $null }))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Rejected' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } + + It 'gives up on a request the broker rejects: ' -TestCases @( + @{ StatusCode = 400 }, @{ StatusCode = 404 } + ) { + $script:Responses.Enqueue((New-TestResponse -StatusCode $StatusCode -Message 'Rejected')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Rejected' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } +} + +Describe 'Invoke-ConnectLinuxBroker' { + BeforeEach { + Mock Write-Log {} + Mock Show-UserMessage {} + Mock Save-LinuxHostCredential {} + Mock Open-RemoteDesktop { $true } + $connectArguments = @{ + ApiBaseUrl = 'https://broker.example/api' + Resource = 'api://client' + Username = 'alice' + AvdHost = 'avd-0' + } + } + + It 'saves the credentials and opens the desktop on the assigned host' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 0 + Should -Invoke Request-LinuxHost -Scope It -Times 1 -Exactly -ParameterFilter { + $CheckoutUrl -eq 'https://broker.example/api/vms/checkout' -and $Resource -eq 'api://client' -and + $Username -eq 'alice' -and $AvdHost -eq 'avd-0' -and $MaxWaitSeconds -eq 600 + } + Should -Invoke Save-LinuxHostCredential -Scope It -Times 1 -Exactly -ParameterFilter { $Username -eq 'alice' -and $Checkout.Hostname -eq 'lnx-05' } + Should -Invoke Open-RemoteDesktop -Scope It -Times 1 -Exactly -ParameterFilter { $Hostname -eq 'lnx-05' -and $IPAddress -eq '10.0.0.5' } + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + } + + It 'passes MaxWaitSeconds on' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Cancelled } + + Invoke-ConnectLinuxBroker @connectArguments -MaxWaitSeconds 120 | Should -Be 0 + Should -Invoke Request-LinuxHost -Scope It -Times 1 -Exactly -ParameterFilter { $MaxWaitSeconds -eq 120 } + } + + It 'fails when Remote Desktop Connection cannot be started' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + Mock Open-RemoteDesktop { $false } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 1 + } + + It 'closes quietly when the user cancels' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Cancelled } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 0 + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + Should -Invoke Open-RemoteDesktop -Scope It -Times 0 -Exactly + } + + It 'tells the user what happened: ' -TestCases @( + @{ Outcome = 'NoHost'; ExpectedText = 'No Linux host is available right now. Try again in a few minutes.'; ExpectedIcon = 'Warning' } + @{ Outcome = 'Starting'; ExpectedText = 'A Linux host is starting for you. Try again in a few minutes.'; ExpectedIcon = 'Information' } + @{ Outcome = 'TimedOut'; ExpectedText = 'Your Linux desktop is taking longer than usual to start. Try again in a few minutes.'; ExpectedIcon = 'Warning' } + @{ Outcome = 'AuthFailed'; ExpectedText = 'The Linux Broker could not authenticate this session host. Contact your administrator.'; ExpectedIcon = 'Error' } + @{ Outcome = 'Unavailable'; ExpectedText = 'The Linux Broker is not responding. Try again in a few minutes, or contact your administrator.'; ExpectedIcon = 'Error' } + @{ Outcome = 'Rejected'; ExpectedText = 'The Linux Broker could not give you a Linux host. Contact your administrator.'; ExpectedIcon = 'Error' } + ) { + $script:Outcome = $Outcome + Mock Request-LinuxHost { New-CheckoutResult -Outcome $script:Outcome } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 1 + Should -Invoke Show-UserMessage -Scope It -Times 1 -Exactly -ParameterFilter { $Message -eq $ExpectedText -and $Icon -eq $ExpectedIcon } + Should -Invoke Open-RemoteDesktop -Scope It -Times 0 -Exactly + } + + It 'opens the desktop for any other mode, with a warning' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + + Invoke-ConnectLinuxBroker @connectArguments -Mode 'xpra' | Should -Be 0 + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Level -eq 'WARNING' -and $Message -like "Mode 'xpra' is not supported*" } + Should -Invoke Open-RemoteDesktop -Scope It -Times 1 -Exactly + } +} + +Describe 'Save-LinuxHostCredential' { + BeforeAll { + # The CredentialManager module is installed on the session hosts only. + if (-not (Get-Command New-StoredCredential -ErrorAction SilentlyContinue)) { + function script:New-StoredCredential { + param($Target, $UserName, $Password, $Type, $Persist) + } + } + } + + BeforeEach { + Mock Write-Log {} + Mock cmdkey { 'Currently stored credentials:', '', ' Target: LegacyGeneric:target=lnx-01', ' Type: Generic' } + Mock New-StoredCredential {} + } + + It 'replaces every saved credential with the ones for the assigned host' { + Save-LinuxHostCredential -Checkout (New-TestCheckout) -Username 'alice' + + Should -Invoke cmdkey -Scope It -Times 1 -Exactly -ParameterFilter { $args[0] -eq '/delete:LegacyGeneric:target=lnx-01' } + foreach ($expected in 'lnx-05', '10.0.0.5') { + Should -Invoke New-StoredCredential -Scope It -Times 1 -Exactly -ParameterFilter { + $Target -eq $expected -and $UserName -eq 'alice' -and $Password -eq 'from-the-broker' -and $Persist -eq 'LocalMachine' -and -not $Type + } + } + foreach ($expected in 'TERMSRV/lnx-05', 'TERMSRV/10.0.0.5') { + Should -Invoke New-StoredCredential -Scope It -Times 1 -Exactly -ParameterFilter { + $Target -eq $expected -and $UserName -eq 'alice' -and $Password -eq 'from-the-broker' -and $Persist -eq 'LocalMachine' -and $Type -eq 'Generic' + } + } + } + + It 'still opens the desktop when the credentials cannot be saved' { + Mock New-StoredCredential { throw 'Access is denied.' } + + { Save-LinuxHostCredential -Checkout (New-TestCheckout) -Username 'alice' } | Should -Not -Throw + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Level -eq 'ERROR' } + } +} + +Describe 'Open-RemoteDesktop' { + BeforeEach { + Mock Write-Log {} + Mock Show-UserMessage {} + Mock Test-Path { $true } + Mock New-Item {} + Mock New-ItemProperty {} + Mock Start-Process {} + } + + It 'turns off the server authentication warning and starts mstsc for the host' { + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeTrue + + Should -Invoke New-ItemProperty -Scope It -Times 1 -Exactly -ParameterFilter { + $Path -eq 'HKCU:\Software\Microsoft\Terminal Server Client' -and $Name -eq 'AuthenticationLevelOverride' -and $Value -eq 0 + } + Should -Invoke Start-Process -Scope It -Times 1 -Exactly -ParameterFilter { $FilePath -eq 'mstsc.exe' -and (@($ArgumentList) -join ' ') -eq '/v:10.0.0.5' } + Should -Invoke New-Item -Scope It -Times 0 -Exactly + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + } + + It 'creates the Remote Desktop client key when it is missing' { + Mock Test-Path { $false } + + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeTrue + Should -Invoke New-Item -Scope It -Times 1 -Exactly -ParameterFilter { $Path -eq 'HKCU:\Software\Microsoft\Terminal Server Client' } + } + + It 'tells the user when mstsc cannot be started' { + Mock Start-Process { throw 'The system cannot find the file specified.' } + + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeFalse + Should -Invoke Show-UserMessage -Scope It -Times 1 -Exactly -ParameterFilter { + $Message -eq 'Remote Desktop Connection could not be started for lnx-05. Try again, or contact your administrator.' -and $Icon -eq 'Error' + } + } +} + +# Needs a desktop to show the window on. +Describe 'Open-WaitWindow' -Skip:(-not [Environment]::UserInteractive) { + BeforeEach { + Mock Write-Log {} + } + + It 'says the desktop is starting, and Cancel cancels the wait' { + $window = Open-WaitWindow + try { + $window | Should -Not -BeNullOrEmpty + $window.Form.Visible | Should -BeTrue + $window.Form.Text | Should -Be 'Linux Desktop' + $window.Cancelled | Should -BeFalse + + $window.Form.CancelButton.PerformClick() + + $window.Cancelled | Should -BeTrue + $window.Form.IsDisposed | Should -BeTrue + } + finally { + Close-WaitWindow -WaitWindow $window + } + } + + It 'counts closing the window as a cancel' { + $window = Open-WaitWindow + $window.Form.Close() + + $window.Cancelled | Should -BeTrue + } + + It 'does not count closing it from the script as a cancel' { + $window = Open-WaitWindow + Close-WaitWindow -WaitWindow $window + + $window.Cancelled | Should -BeFalse + $window.Form.IsDisposed | Should -BeTrue + { Close-WaitWindow -WaitWindow $window } | Should -Not -Throw + } +} diff --git a/avd_host/tests/PSScriptAnalyzerSettings.psd1 b/avd_host/tests/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000..811ec66 --- /dev/null +++ b/avd_host/tests/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,13 @@ +# PSScriptAnalyzer settings for the AVD session host script and the script that updates it, +# used by the avd-scripts job in .github/workflows/front-end-tests.yml. +@{ + Severity = @('Error', 'Warning') + ExcludeRules = @( + # The flagged New-* functions only build objects in memory; there is nothing for -WhatIf + # to skip. + 'PSUseShouldProcessForStateChangingFunctions', + # Update-AvdHostBrokerScript.ps1 reports its progress to the operator who runs it, as the + # other deployment scripts do. + 'PSAvoidUsingWriteHost' + ) +} diff --git a/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 b/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 new file mode 100644 index 0000000..fb96c27 --- /dev/null +++ b/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 @@ -0,0 +1,1009 @@ +# Pester 5 tests for deploy/Update-AvdHostBrokerScript.ps1, which replaces Connect-LinuxBroker.ps1 +# on the existing AVD session hosts through Run Command, and for how +# deploy/Migrate-ExistingEnvironment.ps1 runs it. From the repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path + $script:UpdateScriptPath = Join-Path $script:RepoRoot 'deploy\Update-AvdHostBrokerScript.ps1' + $script:BrokerScriptPath = Join-Path $script:RepoRoot 'avd_host\broker\Connect-LinuxBroker.ps1' + . $script:UpdateScriptPath + + # Mock can only replace a command that exists, and the machine running the tests may not have + # the Azure CLIs. + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { + function script:az { } + } + if (-not (Get-Command azd -ErrorAction SilentlyContinue)) { + function script:azd { } + } + + $script:ResourceGroup = 'rg-linuxbroker-dev' + $script:SubscriptionId = '6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f' + $script:ApiUrl = 'https://app-linuxbroker-api.azurewebsites.net/api' + $script:ClientId = '0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' + $script:DefaultSourceUrl = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main/avd_host/broker/Connect-LinuxBroker.ps1' + $script:BrokerScriptVersion = [regex]::Match([System.IO.File]::ReadAllText($script:BrokerScriptPath), '(?m)^\$ScriptVersion = ''([^'']+)''').Groups[1].Value + $script:WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $script:Utf8NoBom = New-Object System.Text.UTF8Encoding -ArgumentList $false + $script:AzureVariables = @('AZURE_ENV_NAME', 'AZURE_ENVIRONMENT_NAME', 'AZURE_SUBSCRIPTION_ID') + + function New-TestVm { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + + $Tags = @{ 'broker-role' = 'avd-host' }, + + [string]$PowerState = 'VM running', + + [switch]$NoPowerState + ) + + $vm = [ordered]@{ name = $Name; resourceGroup = $script:ResourceGroup; tags = $Tags } + if (-not $NoPowerState) { + $vm['powerState'] = $PowerState + } + return [pscustomobject]$vm + } + + function Set-TestVm { + param([object[]]$Vm) + + $script:VmListJson = ConvertTo-Json -InputObject @($Vm) -Depth 5 + } + + function ConvertTo-RunCommandJson { + param( + [AllowEmptyString()] + [string]$StdOut = '', + + [AllowEmptyString()] + [string]$StdErr = '' + ) + + return ConvertTo-Json -Depth 5 -InputObject ([ordered]@{ + value = @( + [ordered]@{ code = 'ComponentStatus/StdOut/succeeded'; displayStatus = 'Provisioning succeeded'; level = 'Info'; message = $StdOut }, + [ordered]@{ code = 'ComponentStatus/StdErr/succeeded'; displayStatus = 'Provisioning succeeded'; level = 'Info'; message = $StdErr } + ) + }) + } + + # Queues what Run Command answers on a VM, one entry per call: @{ StdOut; StdErr } for a run, + # @{ Error } for az failing, or @{ Raw } for output that is not JSON. A VM with nothing queued + # reports a completed update. + function Add-RunCommandOutcome { + param( + [Parameter(Mandatory = $true)] + [string]$Vm, + + [Parameter(Mandatory = $true)] + [hashtable[]]$Outcome + ) + + if (-not $script:RunCommandOutcomes.ContainsKey($Vm)) { + $script:RunCommandOutcomes[$Vm] = New-Object System.Collections.Queue + } + foreach ($entry in $Outcome) { + $script:RunCommandOutcomes[$Vm].Enqueue($entry) + } + } + + function Get-ArgumentValue { + param( + [string[]]$Arguments, + + [string]$Name + ) + + $index = [array]::IndexOf($Arguments, $Name) + if ($index -lt 0 -or $index + 1 -ge $Arguments.Count) { + return $null + } + return $Arguments[$index + 1] + } + + # The value a top-level assignment in the session host's update script gives a variable. + function Get-PayloadValue { + param( + [Parameter(Mandatory = $true)] + [string]$Payload, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + $parseErrors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseInput($Payload, [ref]$null, [ref]$parseErrors) + $assignment = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.AssignmentStatementAst] -and + $node.Left -is [System.Management.Automation.Language.VariableExpressionAst] -and + $node.Left.VariablePath.UserPath -eq $Name + }, $false) + if ($null -eq $assignment) { + return $null + } + return $assignment.Right.Expression.Value + } + + function Test-Utf8Bom { + param([byte[]]$Bytes) + + return ($Bytes.Length -ge 3 -and $Bytes[0] -eq 0xEF -and $Bytes[1] -eq 0xBB -and $Bytes[2] -eq 0xBF) + } + + function Invoke-ChildProcess { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + + [Parameter(Mandatory = $true)] + [string]$ScriptPath + ) + + $folder = Split-Path -Parent $ScriptPath + $stdoutPath = Join-Path $folder 'stdout.txt' + $stderrPath = Join-Path $folder 'stderr.txt' + $process = Start-Process -FilePath $FilePath ` + -ArgumentList @('-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', ('"{0}"' -f $ScriptPath)) ` + -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -Wait -PassThru -NoNewWindow + + return [pscustomobject]@{ + ExitCode = $process.ExitCode + StdOut = [System.IO.File]::ReadAllText($stdoutPath) + StdErr = [System.IO.File]::ReadAllText($stderrPath) + } + } + + # Runs the script Run Command would run on a session host, in Windows PowerShell as Run Command + # does, with the destination in the test drive. The stubs keep it from registering an event + # source on the machine running the tests and from waiting between attempts, and a failure is + # reported on one line, as Run Command returns it. + function Invoke-HostUpdate { + param( + [Parameter(Mandatory = $true)] + [string]$SourceUrl, + + [Parameter(Mandatory = $true)] + [string]$Destination + ) + + $folder = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $folder | Out-Null + $payloadPath = Join-Path $folder 'payload.ps1' + $harnessPath = Join-Path $folder 'harness.ps1' + + $payload = Get-AvdHostUpdateScript -SourceUrl $SourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId -Destination $Destination + [System.IO.File]::WriteAllText($payloadPath, $payload, $script:Utf8NoBom) + $harness = @' +function New-EventLog { param($LogName, $Source) Write-Output "TEST: New-EventLog $LogName $Source" } +function Start-Sleep { param($Seconds) Write-Output "TEST: Start-Sleep $Seconds" } +try { + . __PAYLOAD__ +} +catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} +'@ + [System.IO.File]::WriteAllText($harnessPath, $harness.Replace('__PAYLOAD__', (ConvertTo-PowerShellLiteral -Value $payloadPath)), $script:Utf8NoBom) + + return Invoke-ChildProcess -FilePath $script:WindowsPowerShell -ScriptPath $harnessPath + } + + function New-InstalledScript { + param([string]$Content = "# Connect-LinuxBroker.ps1 from an earlier release`r`n") + + $destination = Join-Path $TestDrive ('{0}\Connect-LinuxBroker.ps1' -f [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path (Split-Path -Parent $destination) | Out-Null + [System.IO.File]::WriteAllText($destination, $Content) + return $destination + } + + function Get-FolderEntry { + param([string]$Path) + + return @(Get-ChildItem -LiteralPath (Split-Path -Parent $Path) -Force | ForEach-Object { $_.Name }) + } + + function Save-AzureVariable { + $saved = @{} + foreach ($name in $script:AzureVariables) { + $saved[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') + [Environment]::SetEnvironmentVariable($name, $null, 'Process') + } + return $saved + } + + function Restore-AzureVariable { + param([hashtable]$Saved) + + foreach ($name in $Saved.Keys) { + [Environment]::SetEnvironmentVariable($name, $Saved[$name], 'Process') + } + } +} + +Describe 'ConvertTo-PowerShellLiteral' { + It 'doubles single quotes, including the curly ones PowerShell also ends a string at' { + ConvertTo-PowerShellLiteral -Value "it's" | Should -BeExactly "'it''s'" + $curly = [string][char]0x2019 + ConvertTo-PowerShellLiteral -Value "it$($curly)s" | Should -BeExactly "'it$curly$($curly)s'" + ConvertTo-PowerShellLiteral -Value '' | Should -BeExactly "''" + } +} + +Describe 'Test-SafeHttpsUrl' { + It 'accepts ' -TestCases @( + @{ Value = 'https://app-linuxbroker-api.azurewebsites.net/api' } + @{ Value = 'https://app-linuxbroker-api.azurewebsites.us:8443/api' } + @{ Value = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main' } + @{ Value = 'https://mirror.contoso.us/linux%20broker/~main/' } + @{ Value = 'https://10.0.0.4' } + ) { + Test-SafeHttpsUrl -Value $Value | Should -BeTrue + } + + It 'refuses a URL that ' -TestCases @( + @{ Description = 'is empty'; Value = '' } + @{ Description = 'is not https'; Value = 'http://app/api' } + @{ Description = 'names no host'; Value = 'https:///api' } + @{ Description = 'holds a double quote'; Value = 'https://app/api"' } + @{ Description = 'holds a single quote'; Value = "https://app/it's" } + @{ Description = 'holds a typographic double quote'; Value = ('https://app/api{0}' -f [char]0x201D) } + @{ Description = 'holds a typographic single quote'; Value = ('https://app/it{0}s' -f [char]0x2019) } + @{ Description = 'holds a letter outside ASCII'; Value = ('https://app/{0}/api' -f [char]0x00D1) } + @{ Description = 'holds the Kelvin sign, which a match without regard to case takes for a k'; Value = ('https://app/{0}/api' -f [char]0x212A) } + @{ Description = 'holds a subexpression'; Value = 'https://app/$(Get-Process)/api' } + @{ Description = 'holds a backtick'; Value = 'https://app/`api' } + @{ Description = 'holds a space'; Value = 'https://app /api' } + @{ Description = 'ends with a line break'; Value = "https://app/api`n" } + @{ Description = 'has a query string'; Value = 'https://app/api?code=1' } + @{ Description = 'has a fragment'; Value = 'https://app/api#top' } + @{ Description = 'names a user'; Value = 'https://user@app/api' } + @{ Description = 'has a port that is not a number'; Value = 'https://app:port/api' } + @{ Description = 'has a port above 65535'; Value = 'https://app:99999/api' } + ) { + Test-SafeHttpsUrl -Value $Value | Should -BeFalse + } +} + +Describe 'Get-AzdEnvValue' { + BeforeEach { + Set-StrictMode -Version Latest + $script:AzdCalls = New-Object System.Collections.Generic.List[string] + $script:AzdExitCode = 0 + $script:AzdOutput = @('rg-linuxbroker-dev', '') + Mock azd { + $script:AzdCalls.Add(($args -join ' ')) + $global:LASTEXITCODE = $script:AzdExitCode + $script:AzdOutput + } + } + + It 'reads a value from the azd environment' { + Get-AzdEnvValue -EnvironmentName 'linuxbroker-dev' -Key 'resourceGroupName' | Should -BeExactly 'rg-linuxbroker-dev' + $script:AzdCalls | Should -Be @('env get-value resourceGroupName --environment linuxbroker-dev') + } + + It 'answers with nothing when azd has no such value' { + $script:AzdExitCode = 1 + $script:AzdOutput = 'ERROR: key ''apiUrl'' not found in the environment values' + + Get-AzdEnvValue -EnvironmentName 'linuxbroker-dev' -Key 'apiUrl' | Should -BeExactly '' + } + + It 'does not ask azd without an environment' { + Get-AzdEnvValue -EnvironmentName '' -Key 'apiUrl' | Should -BeExactly '' + $script:AzdCalls.Count | Should -Be 0 + } +} + +Describe 'Get-AvdHostUpdateScript' { + BeforeEach { + Set-StrictMode -Version Latest + } + + It 'fills in exactly, as a string PowerShell 5.1 parses' -TestCases @( + @{ Description = 'ordinary values'; Value = 'https://app-linuxbroker-api.azurewebsites.net/api' } + @{ Description = 'a single quote'; Value = "https://example.com/it's" } + @{ Description = 'a curly quote'; Value = ('https://example.com/it{0}s' -f [char]0x2019) } + @{ Description = 'what a double-quoted string would expand'; Value = 'https://example.com/$(Get-Process)/`t/$env:COMPUTERNAME"' } + ) { + $payload = Get-AvdHostUpdateScript -SourceUrl $Value -ApiBaseUrl $Value -ApiClientId $Value -Destination $Value + + $parseErrors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($payload, [ref]$null, [ref]$parseErrors) + @($parseErrors).Count | Should -Be 0 + foreach ($name in 'sourceUrl', 'apiBaseUrl', 'apiClientId', 'destination') { + Get-PayloadValue -Payload $payload -Name $name | Should -BeExactly $Value + } + $payload | Should -Not -Match '__(SOURCE_URL|API_BASE_URL|API_CLIENT_ID|DESTINATION)__' + } + + It 'fills in every value in one pass, so a value that holds a token keeps it' { + $sourceUrl = 'https://example.com/__DESTINATION__/__API_CLIENT_ID__/Connect-LinuxBroker.ps1' + + $payload = Get-AvdHostUpdateScript -SourceUrl $sourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId + + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly $sourceUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + Get-PayloadValue -Payload $payload -Name 'destination' | Should -BeExactly 'C:\Temp\Connect-LinuxBroker.ps1' + } + + It 'installs the script where Configure-AVD-Host.ps1 does and the RemoteApp runs it from' { + $default = Get-PayloadValue -Payload (Get-AvdHostUpdateScript -SourceUrl $script:DefaultSourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId) -Name 'destination' + + $bicep = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'deploy\bicep\modules\AVD\main.bicep')) + $remoteAppPath = [regex]::Match($bicep, "commandLineArguments: '[^']*-File (\S+?)'").Groups[1].Value.Replace('\\', '\') + $remoteAppPath | Should -BeExactly $default + $configure = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'custom_script_extensions\Configure-AVD-Host.ps1')) + $configure | Should -Match ([regex]::Escape('$folderPath = "C:\Temp"')) + $configure | Should -Match ([regex]::Escape('$outputPath = "$folderPath\Connect-LinuxBroker.ps1"')) + } +} + +Describe 'Updating the session hosts through Run Command' { + BeforeEach { + Set-StrictMode -Version Latest + $global:LASTEXITCODE = 0 + $script:AzCalls = New-Object System.Collections.Generic.List[string] + $script:RunCommands = New-Object System.Collections.Generic.List[object] + $script:RunCommandOutcomes = @{} + $script:AccountSetExitCode = 0 + $script:VmListExitCode = 0 + $script:VmListJson = '[]' + $script:HostLines = New-Object System.Collections.Generic.List[string] + $script:Warnings = New-Object System.Collections.Generic.List[string] + $script:Slept = New-Object System.Collections.Generic.List[int] + $script:AzdValues = @{} + $script:AzdEnvironments = New-Object System.Collections.Generic.List[string] + + Mock az { + $arguments = [string[]]@($args | ForEach-Object { [string]$_ }) + $commandLine = $arguments -join ' ' + $script:AzCalls.Add($commandLine) + $global:LASTEXITCODE = 0 + + if ($commandLine -like 'account set *') { + $global:LASTEXITCODE = $script:AccountSetExitCode + return + } + + if ($commandLine -like 'vm list *') { + $global:LASTEXITCODE = $script:VmListExitCode + if ($script:VmListExitCode -ne 0) { + return + } + return $script:VmListJson + } + + if ($commandLine -like 'vm run-command invoke *') { + $vmName = Get-ArgumentValue -Arguments $arguments -Name '--name' + $scriptFile = (Get-ArgumentValue -Arguments $arguments -Name '--scripts').Substring(1) + $script:RunCommands.Add([pscustomobject]@{ + Vm = $vmName + ResourceGroup = Get-ArgumentValue -Arguments $arguments -Name '--resource-group' + Arguments = $arguments + File = $scriptFile + Script = [System.IO.File]::ReadAllText($scriptFile) + Bytes = [System.IO.File]::ReadAllBytes($scriptFile) + }) + + $outcome = @{ StdOut = "Updated Connect-LinuxBroker.ps1 on $vmName from 1.0.0 to 2.0.0." } + if ($script:RunCommandOutcomes.ContainsKey($vmName) -and $script:RunCommandOutcomes[$vmName].Count -gt 0) { + $outcome = $script:RunCommandOutcomes[$vmName].Dequeue() + } + + if ($outcome.ContainsKey('Error')) { + $global:LASTEXITCODE = 1 + return (New-Object System.Management.Automation.ErrorRecord -ArgumentList @( + (New-Object System.Exception -ArgumentList $outcome['Error']), + 'NativeCommandError', + [System.Management.Automation.ErrorCategory]::NotSpecified, + $null)) + } + if ($outcome.ContainsKey('Raw')) { + return $outcome['Raw'] + } + return (ConvertTo-RunCommandJson -StdOut ([string]$outcome['StdOut']) -StdErr ([string]$outcome['StdErr'])) + } + + throw "Unexpected az call: $commandLine" + } + Mock azd { throw "Unexpected azd call: $($args -join ' ')" } + Mock Get-AzdEnvValue { + $script:AzdEnvironments.Add($EnvironmentName) + [string]$script:AzdValues[$Key] + } + Mock Write-Host { $script:HostLines.Add([string]$Object) } + Mock Write-Warning { $script:Warnings.Add([string]$Message) } + Mock Start-Sleep { $script:Slept.Add([int]$Seconds) } + + $savedVariables = Save-AzureVariable + $update = @{ + ResourceGroupName = $script:ResourceGroup + ApiBaseUrl = $script:ApiUrl + ApiClientId = $script:ClientId + SubscriptionId = $script:SubscriptionId + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + } + + Context 'Invoke-RunCommandWithRetry' { + BeforeEach { + $testScript = "Write-Output 'first'`r`nWrite-Output 'second'`r`n" + } + + It 'returns what the script wrote to standard output and standard error' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = "first`nsecond`n"; StdErr = 'WARNING: The CredentialManager module is not installed.' } + + $result = Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript + + $result.StdOut | Should -BeExactly "first`nsecond" + $result.StdErr | Should -BeExactly 'WARNING: The CredentialManager module is not installed.' + $script:RunCommands.Count | Should -Be 1 + $script:Slept.Count | Should -Be 0 + } + + It 'hands az the script as a file, without a byte order mark, and deletes the file afterwards' { + Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript | Out-Null + + $call = $script:RunCommands[0] + $call.Script | Should -BeExactly $testScript + Test-Utf8Bom -Bytes $call.Bytes | Should -BeFalse + Test-Path -LiteralPath $call.File | Should -BeFalse + Get-ArgumentValue -Arguments $call.Arguments -Name '--command-id' | Should -BeExactly 'RunPowerShellScript' + $call.ResourceGroup | Should -BeExactly $script:ResourceGroup + } + + It 'retries a Run Command that fails, waiting longer each time' { + $conflict = 'ERROR: (Conflict) Run command extension execution is in progress. Please wait for completion before invoking a run command.' + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = $conflict }, @{ Error = $conflict }, @{ StdOut = 'done' } + + $result = Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript + + $result.StdOut | Should -BeExactly 'done' + $script:RunCommands.Count | Should -Be 3 + $script:Slept | Should -Be @(5, 10) + $script:Warnings | Should -Contain "Run Command failed on 'avd-0' attempt 1 of 3. Retrying in 5 seconds." + $script:Warnings | Should -Contain $conflict + } + + It 'gives up after the last attempt, reporting the last error, and deletes the file' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = 'ERROR: first' }, @{ Error = 'ERROR: second' }, @{ Error = 'ERROR: third' } + + { Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript } | + Should -Throw "Run Command failed on 'avd-0'. Last error: ERROR: third" + + $script:RunCommands.Count | Should -Be 3 + $script:Slept | Should -Be @(5, 10) + Test-Path -LiteralPath $script:RunCommands[0].File | Should -BeFalse + } + + It 'waits at most 30 seconds between attempts' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ StdOut = 'done' } + + Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript -MaxAttempts 5 -InitialDelaySeconds 20 | Out-Null + + $script:Slept | Should -Be @(20, 30, 30, 30) + } + + It 'refuses output that is not JSON' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Raw = 'This command is in preview and under development.' } + + { Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript } | + Should -Throw "Run Command on 'avd-0' returned output that is not JSON*" + $script:RunCommands.Count | Should -Be 1 + } + } + + Context 'Invoke-AvdHostBrokerScriptUpdate' { + It 'updates every running AVD session host in the resource group and no other VM' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }), + (New-TestVm -Name 'jumpbox' -Tags @{ environment = 'dev' }), + (New-TestVm -Name 'untagged' -Tags $null), + (New-TestVm -Name 'avd-1'), + (New-TestVm -Name 'avd-2' -NoPowerState) + ) + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0', 'avd-1', 'avd-2') + @($script:RunCommands | ForEach-Object { $_.ResourceGroup } | Select-Object -Unique) | Should -Be @($script:ResourceGroup) + $script:AzCalls[0] | Should -BeExactly "account set --subscription $script:SubscriptionId" + $script:AzCalls[1] | Should -BeExactly "vm list --resource-group $script:ResourceGroup --show-details --output json --only-show-errors" + $script:HostLines | Should -Contain 'Connect-LinuxBroker.ps1 is up to date on 3 AVD session host(s).' + $script:Warnings.Count | Should -Be 0 + } + + It 'runs the same update on every session host, filled in with this environment''s values' { + Set-TestVm -Vm @((New-TestVm -Name 'avd-0'), (New-TestVm -Name 'avd-1')) + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + $script:RunCommands[1].Script | Should -BeExactly $payload + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly $script:DefaultSourceUrl + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + Get-PayloadValue -Payload $payload -Name 'destination' | Should -BeExactly 'C:\Temp\Connect-LinuxBroker.ps1' + } + + It 'downloads Connect-LinuxBroker.ps1 from ScriptSourceRoot and drops a trailing slash from the API URL' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.ScriptSourceRoot = 'https://mirror.contoso.com/linuxbroker/' + $update.ApiBaseUrl = " $script:ApiUrl/ " + $update.ApiClientId = " $script:ClientId " + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly 'https://mirror.contoso.com/linuxbroker/avd_host/broker/Connect-LinuxBroker.ps1' + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + } + + It 'skips the session hosts that are not running and names them at the end' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'avd-1' -PowerState 'VM deallocated'), + (New-TestVm -Name 'avd-2' -PowerState 'VM stopped') + ) + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0') + $script:Warnings | Should -Contain "Skipping AVD session host 'avd-1' because it is not running (VM deallocated). Start it and run this script again with -AvdHostNames avd-1." + $script:Warnings | Should -Contain 'Not updated because they are not running: avd-1, avd-2.' + $script:HostLines | Should -Contain 'Connect-LinuxBroker.ps1 is up to date on 1 AVD session host(s).' + } + + It 'tries every session host and then names the ones where the update failed' { + Set-TestVm -Vm @((New-TestVm -Name 'avd-0'), (New-TestVm -Name 'avd-1'), (New-TestVm -Name 'avd-2')) + $downloadFailure = "Could not download $script:DefaultSourceUrl after 3 attempts: The remote server returned an error: (404) Not Found." + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = ''; StdErr = $downloadFailure } + $agentFailure = 'ERROR: (VMAgentStatusCommunicationError) The VM agent is not reporting.' + Add-RunCommandOutcome -Vm 'avd-1' -Outcome @{ Error = $agentFailure }, @{ Error = $agentFailure }, @{ Error = $agentFailure } + + { Invoke-AvdHostBrokerScriptUpdate @update } | + Should -Throw 'The Connect-LinuxBroker.ps1 update failed on: avd-0, avd-1. Rerun with -AvdHostNames to retry them.' + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0', 'avd-1', 'avd-1', 'avd-1', 'avd-2') + $script:Warnings | Should -Contain "The update did not run to completion on 'avd-0'. $downloadFailure" + $script:Warnings | Should -Contain "Run Command failed on 'avd-1'. Last error: $agentFailure" + @($script:HostLines | Where-Object { $_ -like '*is up to date*' }).Count | Should -Be 0 + } + + It 'treats a run that did not reach its last line as failed, even without an error' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = 'WARNING: The LinuxBrokerScript event source could not be created.' } + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw 'The Connect-LinuxBroker.ps1 update failed on: avd-0.*' + $script:Warnings | Should -Contain "The update did not run to completion on 'avd-0'." + } + + It 'reads the values it is not given from the azd environment' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $script:AzdValues = @{ + resourceGroupName = $script:ResourceGroup + AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + apiUrl = $script:ApiUrl + apiClientId = $script:ClientId + } + + Invoke-AvdHostBrokerScriptUpdate -EnvironmentName 'linuxbroker-dev' + + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('linuxbroker-dev') + $script:AzCalls | Should -Contain "account set --subscription $script:SubscriptionId" + $script:AzCalls | Should -Contain "vm list --resource-group $script:ResourceGroup --show-details --output json --only-show-errors" + Get-PayloadValue -Payload $script:RunCommands[0].Script -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $script:RunCommands[0].Script -Name 'apiClientId' | Should -BeExactly $script:ClientId + } + + It 'uses the azd environment in when none is given' -TestCases @( + @{ Variable = 'AZURE_ENV_NAME' } + @{ Variable = 'AZURE_ENVIRONMENT_NAME' } + ) { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + [Environment]::SetEnvironmentVariable($Variable, 'linuxbroker-hook', 'Process') + $update.Remove('ResourceGroupName') + $script:AzdValues = @{ resourceGroupName = $script:ResourceGroup } + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('linuxbroker-hook') + $script:RunCommands.Count | Should -Be 1 + } + + It 'selects the subscription in AZURE_SUBSCRIPTION_ID when azd has none' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.Remove('SubscriptionId') + $env:AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + + Invoke-AvdHostBrokerScriptUpdate @update + + $script:AzCalls | Should -Contain "account set --subscription $script:SubscriptionId" + } + + It 'keeps the current subscription when none is known' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.Remove('SubscriptionId') + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:AzCalls | Where-Object { $_ -like 'account set*' }).Count | Should -Be 0 + $script:RunCommands.Count | Should -Be 1 + } + + It 'stops before it calls Azure when is not known' -TestCases @( + @{ Missing = 'ResourceGroupName' } + @{ Missing = 'ApiBaseUrl' } + @{ Missing = 'ApiClientId' } + ) { + $update.Remove($Missing) + + { Invoke-AvdHostBrokerScriptUpdate @update } | + Should -Throw 'The AVD session host update needs ResourceGroupName, ApiBaseUrl and ApiClientId, as parameters or azd environment values.' + $script:AzCalls.Count | Should -Be 0 + } + + It 'refuses an API URL that ' -TestCases @( + @{ Description = 'is not https'; Url = 'http://app-linuxbroker-api.azurewebsites.net/api' } + @{ Description = 'holds a double quote'; Url = 'https://app/api";Remove-Item("C:\Temp");"' } + @{ Description = 'holds a typographic double quote'; Url = ('https://app/api{0};Remove-Item("C:\Temp");{0}' -f [char]0x201D) } + @{ Description = 'holds a letter outside ASCII'; Url = ('https://app/{0}/api' -f [char]0x00D1) } + @{ Description = 'holds a single quote'; Url = "https://app/api'" } + @{ Description = 'holds a subexpression'; Url = 'https://app/$(Get-Process)/api' } + @{ Description = 'holds a backtick'; Url = 'https://app/`api' } + @{ Description = 'holds a space'; Url = 'https://app /api' } + ) { + $update.ApiBaseUrl = $Url + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not *" + $script:AzCalls.Count | Should -Be 0 + } + + It 'refuses a ScriptSourceRoot that ' -TestCases @( + @{ Description = 'is not https'; Root = 'http://mirror.contoso.com/linuxbroker' } + @{ Description = 'holds a single quote'; Root = "https://mirror.contoso.com/it's" } + @{ Description = 'holds a typographic single quote'; Root = ('https://mirror.contoso.com/it{0}s' -f [char]0x2019) } + @{ Description = 'holds a letter outside ASCII'; Root = ('https://mirror.contoso.com/{0}' -f [char]0x00D1) } + @{ Description = 'is empty'; Root = '' } + ) { + $update.ScriptSourceRoot = $Root + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw 'ScriptSourceRoot must be the https:// URL of the repository or of a mirror of it, such as https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main, not *' + $script:AzCalls.Count | Should -Be 0 + } + + It 'accepts URLs with a port, percent-encoding and a tilde' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.ScriptSourceRoot = 'https://mirror.contoso.us:8443/linux%20broker/~main' + $update.ApiBaseUrl = 'https://app-linuxbroker-api.azurewebsites.us:8443/api' + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly 'https://mirror.contoso.us:8443/linux%20broker/~main/avd_host/broker/Connect-LinuxBroker.ps1' + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly 'https://app-linuxbroker-api.azurewebsites.us:8443/api' + } + + It 'refuses a client ID that is not a GUID in its usual form: ' -TestCases @( + @{ ClientId = 'api://0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' } + @{ ClientId = '{0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21}' } + @{ ClientId = '0b5a9c3e6f1d4a2b9c8e7d6f5e4a3b21' } + @{ ClientId = 'linuxbroker-api' } + ) { + $update.ApiClientId = $ClientId + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "ApiClientId must be the broker API app registration's client ID, a GUID, not *" + $script:AzCalls.Count | Should -Be 0 + } + + It 'updates only the session hosts named with -AvdHostNames, in any case, and warns about the rest' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'avd-1'), + (New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }) + ) + + Invoke-AvdHostBrokerScriptUpdate @update -AvdHostNames 'AVD-1', ' avd-9 ', 'lnx-0', '' + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-1') + $script:Warnings | Should -Contain "Requested AVD session host 'avd-9' was not found in resource group '$script:ResourceGroup'." + $script:Warnings | Should -Contain "Requested AVD session host 'lnx-0' was not found in resource group '$script:ResourceGroup'." + $script:Warnings.Count | Should -Be 2 + } + + It 'does nothing when no session host has a requested name' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + + Invoke-AvdHostBrokerScriptUpdate @update -AvdHostNames 'avd-9' + + $script:RunCommands.Count | Should -Be 0 + $script:HostLines | Should -Contain 'No AVD session hosts matched the requested update scope.' + } + + It 'does nothing in a resource group with ' -TestCases @( + @{ Description = 'no VMs'; Linux = $false } + @{ Description = 'only Linux hosts'; Linux = $true } + ) { + if ($Linux) { + Set-TestVm -Vm @(New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }) + } + + Invoke-AvdHostBrokerScriptUpdate @update + + $script:RunCommands.Count | Should -Be 0 + $script:HostLines | Should -Contain "No AVD session host VMs found in resource group '$script:ResourceGroup'." + } + + It 'stops when the VMs cannot be listed' { + $script:VmListExitCode = 1 + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "Could not list the VMs in resource group '$script:ResourceGroup'." + $script:RunCommands.Count | Should -Be 0 + } + + It 'stops when the subscription cannot be selected' { + $script:AccountSetExitCode = 1 + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "Could not select subscription '$script:SubscriptionId'." + @($script:AzCalls | Where-Object { $_ -like 'vm *' }).Count | Should -Be 0 + } + } +} + +Describe 'The update on a session host' { + BeforeAll { + $script:SourcePath = Join-Path $TestDrive 'source\Connect-LinuxBroker.ps1' + New-Item -ItemType Directory -Path (Split-Path -Parent $script:SourcePath) | Out-Null + Copy-Item -LiteralPath $script:BrokerScriptPath -Destination $script:SourcePath + $script:SourceUrl = ([Uri]$script:SourcePath).AbsoluteUri + $script:ExpectedScript = [System.IO.File]::ReadAllText($script:BrokerScriptPath).Replace('https://your_linuxbroker_api_base_url/api', $script:ApiUrl).Replace('your_linuxbroker_api_client_id', $script:ClientId) + } + + It 'installs the current script where there is none, with the API URL and client ID filled in' { + $destination = Join-Path $TestDrive ('{0}\Temp\Connect-LinuxBroker.ps1' -f [guid]::NewGuid().ToString('N')) + + $result = Invoke-HostUpdate -SourceUrl $script:SourceUrl -Destination $destination + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + $script:BrokerScriptVersion | Should -Not -BeNullOrEmpty + $result.StdOut | Should -Match ([regex]::Escape("Updated Connect-LinuxBroker.ps1 on $env:COMPUTERNAME from no script to $script:BrokerScriptVersion.")) + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $script:ExpectedScript + $script:ExpectedScript | Should -Match ([regex]::Escape("`$apiBaseUrl = `"$script:ApiUrl`"")) + $script:ExpectedScript | Should -Match ([regex]::Escape("api://$script:ClientId")) + Test-Utf8Bom -Bytes ([System.IO.File]::ReadAllBytes($destination)) | Should -BeFalse + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'replaces in place' -TestCases @( + @{ Description = 'a script from before versions'; Content = "# Connect-LinuxBroker.ps1`r`n`$apiBaseUrl = `"https://app-linuxbroker-api.azurewebsites.net/api`"`r`n"; Previous = 'a script without a version' } + @{ Description = 'an earlier version'; Content = "`$ScriptVersion = '1.9.0'`r`n"; Previous = '1.9.0' } + ) { + $destination = New-InstalledScript -Content $Content + + $result = Invoke-HostUpdate -SourceUrl $script:SourceUrl -Destination $destination + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + $result.StdOut | Should -Match ([regex]::Escape("from $Previous to $script:BrokerScriptVersion.")) + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $script:ExpectedScript + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'leaves the installed script alone when the download fails' { + $destination = New-InstalledScript + $before = [System.IO.File]::ReadAllText($destination) + $missingUrl = ([Uri](Join-Path $TestDrive 'missing\Connect-LinuxBroker.ps1')).AbsoluteUri + + $result = Invoke-HostUpdate -SourceUrl $missingUrl -Destination $destination + + $result.ExitCode | Should -Be 1 + $result.StdErr | Should -Match ('Could not download ' + [regex]::Escape($missingUrl) + ' after 3 attempts') + $result.StdOut | Should -Match 'TEST: Start-Sleep 2' + $result.StdOut | Should -Match 'TEST: Start-Sleep 4' + $result.StdOut | Should -Not -Match 'Updated Connect-LinuxBroker.ps1' + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $before + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'refuses and leaves the installed script alone' -TestCases @( + @{ + Description = 'a file that is not a Connect-LinuxBroker.ps1' + Content = "Write-Output 'hello'`r`n" + Message = 'is not a Connect-LinuxBroker.ps1 that this update can install.' + } + @{ + Description = 'a script without the placeholders to fill in' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://app-linuxbroker-api.azurewebsites.net/api`"`r`n" + Message = 'is not a Connect-LinuxBroker.ps1 that this update can install.' + } + @{ + Description = 'a script that does not parse' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://your_linuxbroker_api_base_url/api`"`r`n`$apiAppIdUri = `"api://your_linuxbroker_api_client_id`"`r`nfunction {`r`n" + Message = 'Connect-LinuxBroker.ps1 9.9.9 does not parse once the API URL and client ID are filled in' + } + @{ + Description = 'a script with characters outside ASCII' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://your_linuxbroker_api_base_url/api`"`r`n`$apiAppIdUri = `"api://your_linuxbroker_api_client_id`"`r`n# Caf$([char]0x00E9)`r`n" + Message = 'Connect-LinuxBroker.ps1 9.9.9 holds characters outside ASCII, which Windows PowerShell could misread.' + } + ) { + $sourcePath = Join-Path $TestDrive ('{0}.ps1' -f [guid]::NewGuid().ToString('N')) + [System.IO.File]::WriteAllText($sourcePath, $Content) + $destination = New-InstalledScript + $before = [System.IO.File]::ReadAllText($destination) + + $result = Invoke-HostUpdate -SourceUrl ([Uri]$sourcePath).AbsoluteUri -Destination $destination + + $result.ExitCode | Should -Be 1 + $result.StdErr | Should -Match ([regex]::Escape($Message)) + $result.StdOut | Should -Not -Match 'Updated Connect-LinuxBroker.ps1' + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $before + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } +} + +Describe 'Update-AvdHostBrokerScript.ps1' { + It 'runs the update with its parameters when it is run as a script in PowerShell 7' -Skip:(-not (Get-Command pwsh -ErrorAction SilentlyContinue)) { + $folder = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $folder | Out-Null + $harnessPath = Join-Path $folder 'harness.ps1' + $harness = @' +try { + & __SCRIPT__ -ResourceGroupName 'rg-linuxbroker-dev' -ApiBaseUrl 'http://app-linuxbroker-api.azurewebsites.net/api' -ApiClientId '0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' +} +catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} +'@ + [System.IO.File]::WriteAllText($harnessPath, $harness.Replace('__SCRIPT__', (ConvertTo-PowerShellLiteral -Value $script:UpdateScriptPath)), $script:Utf8NoBom) + + $result = Invoke-ChildProcess -FilePath (Get-Command pwsh).Source -ScriptPath $harnessPath + + $result.ExitCode | Should -Be 1 + $result.StdErr.Trim() | Should -BeExactly "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not 'http://app-linuxbroker-api.azurewebsites.net/api'." + } +} + +Describe 'Migrate-ExistingEnvironment.ps1' { + BeforeAll { + $script:MigrationFolder = Join-Path $TestDrive 'deploy' + New-Item -ItemType Directory -Path $script:MigrationFolder | Out-Null + Copy-Item -LiteralPath (Join-Path $script:RepoRoot 'deploy\Migrate-ExistingEnvironment.ps1') -Destination $script:MigrationFolder + $script:MigrationPath = Join-Path $script:MigrationFolder 'Migrate-ExistingEnvironment.ps1' + $script:StepLog = Join-Path $script:MigrationFolder 'steps.log' + + # Each step records how it was run, with an array argument as a,b, and fails when + # LINUXBROKER_TEST_FAILING_STEPS names it. + $stub = @' +$step = [System.IO.Path]::GetFileNameWithoutExtension($MyInvocation.MyCommand.Path) +$arguments = @($args | ForEach-Object { if ($_ -is [array]) { $_ -join ',' } else { [string]$_ } }) +Add-Content -LiteralPath (Join-Path $PSScriptRoot 'steps.log') -Value ('{0} {1}' -f $step, ($arguments -join ' ')) +if (@([string]$env:LINUXBROKER_TEST_FAILING_STEPS -split ',') -contains $step) { + throw "$step failed on host-1." +} +'@ + foreach ($step in 'Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') { + [System.IO.File]::WriteAllText((Join-Path $script:MigrationFolder "$step.ps1"), $stub) + } + + function Get-StepLog { + if (-not (Test-Path -LiteralPath $script:StepLog)) { + return @() + } + return @(Get-Content -LiteralPath $script:StepLog) + } + + function Get-StepName { + return @(Get-StepLog | ForEach-Object { ($_ -split ' ')[0] }) + } + } + + BeforeEach { + Remove-Item -LiteralPath $script:StepLog -ErrorAction SilentlyContinue + # Not $script: variables: in a mock that Migrate-ExistingEnvironment.ps1 calls, $script: is + # that script's scope. These are found through the scopes that called it. + $hostLines = New-Object System.Collections.Generic.List[string] + $warnings = New-Object System.Collections.Generic.List[string] + Mock Write-Host { $hostLines.Add([string]$Object) } + Mock Write-Warning { $warnings.Add([string]$Message) } + Mock az { throw "Unexpected az call: $($args -join ' ')" } + Mock azd { throw "Unexpected azd call: $($args -join ' ')" } + + $savedVariables = Save-AzureVariable + $savedFailingSteps = $env:LINUXBROKER_TEST_FAILING_STEPS + $env:LINUXBROKER_TEST_FAILING_STEPS = $null + $migration = @{ + ResourceGroupName = $script:ResourceGroup + ApiBaseUrl = $script:ApiUrl + ApiClientId = $script:ClientId + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + $env:LINUXBROKER_TEST_FAILING_STEPS = $savedFailingSteps + } + + It 'updates the AVD session hosts after the Linux hosts, with the same values' { + & $script:MigrationPath @migration -LinuxHostNames 'lnx-0' -AvdHostNames 'avd-0', 'avd-1' + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') + $steps = Get-StepLog + $steps[1] | Should -Match ([regex]::Escape('-LinuxHostNames lnx-0')) + $steps[2] | Should -Match ([regex]::Escape("-ResourceGroupName $script:ResourceGroup -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId")) + $steps[2] | Should -Match ([regex]::Escape('-ScriptSourceRoot https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main')) + $steps[2] | Should -Match ([regex]::Escape('-AvdHostNames avd-0,avd-1')) + $hostLines | Should -Contain 'Existing environment migration completed.' + } + + It 'still updates the AVD session hosts when the Linux host migration fails, and then fails' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Migrate-LinuxHostReleaseAgent' + + { & $script:MigrationPath @migration } | + Should -Throw 'Existing environment migration failed in the Linux host migration. The warnings above name the hosts to retry.' + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') + $warnings | Should -Contain 'Migrate-LinuxHostReleaseAgent failed on host-1.' + $hostLines | Should -Not -Contain 'Existing environment migration completed.' + } + + It 'names both host updates when both fail' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Migrate-LinuxHostReleaseAgent,Update-AvdHostBrokerScript' + + { & $script:MigrationPath @migration } | + Should -Throw 'Existing environment migration failed in the Linux host migration and the AVD session host update. The warnings above name the hosts to retry.' + $warnings | Should -Contain 'Update-AvdHostBrokerScript failed on host-1.' + } + + It 'stops at a failed post-provision step, before it updates any host' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Post-Provision' + + { & $script:MigrationPath @migration } | Should -Throw 'Post-Provision failed on host-1.' + Get-StepName | Should -Be @('Post-Provision') + } + + It 'leaves the AVD session hosts alone with -SkipAvdHostScriptUpdate' { + & $script:MigrationPath @migration -SkipAvdHostScriptUpdate + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent') + } + + It 'updates only the AVD session hosts with -SkipLinuxHostReleaseAgentMigration' { + & $script:MigrationPath @migration -SkipPostProvision -SkipLinuxHostReleaseAgentMigration + + Get-StepName | Should -Be @('Update-AvdHostBrokerScript') + } + + It 'stops before it updates any host when is not known' -TestCases @( + @{ Missing = 'ResourceGroupName' } + @{ Missing = 'ApiBaseUrl' } + @{ Missing = 'ApiClientId' } + ) { + $migration.Remove($Missing) + + { & $script:MigrationPath @migration } | + Should -Throw 'Updating the Linux hosts and the AVD session hosts requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' + Get-StepName | Should -Be @('Post-Provision') + } + + It 'needs none of them when both host updates are skipped' { + & $script:MigrationPath -SkipLinuxHostReleaseAgentMigration -SkipAvdHostScriptUpdate + + Get-StepName | Should -Be @('Post-Provision') + $hostLines | Should -Contain 'Existing environment migration completed.' + } +} diff --git a/deploy/DEPLOYMENT.md b/deploy/DEPLOYMENT.md index 41841b3..c99dbfc 100644 --- a/deploy/DEPLOYMENT.md +++ b/deploy/DEPLOYMENT.md @@ -430,7 +430,7 @@ If the build environment cannot reach `registry.npmjs.org`, point npm at an inte ## Migration For Existing Deployments -Use the migration flow when you already have a deployed customer environment and want to roll forward the current application, SQL, and Linux-host release-agent changes without treating that as part of the normal `azd up` lifecycle. +Use the migration flow when you already have a deployed customer environment and want to roll forward the current application, SQL, Linux host release-agent and AVD session host script changes without treating that as part of the normal `azd up` lifecycle. The migration entrypoint is [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1). @@ -439,10 +439,13 @@ That script intentionally stays separate from `azd up`: - `azd up` continues to express the desired greenfield deployment for new environments. - [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1) is the supported in-place process for existing environments. -By default, the migration script does two things: +By default, the migration script does three things: 1. Runs [Post-Provision.ps1](Post-Provision.ps1) so the existing environment gets the latest container images, SQL scripts, role assignments, VM group sync, and Linux host SQL registration. 2. Runs [Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1) so existing Linux hosts get the current release-agent files, one-minute reconciliation timer, and `systemd-logind` watcher. +3. Runs [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) so existing AVD session hosts get the current `Connect-LinuxBroker.ps1`, the script the **Linux Desktop** RemoteApp runs. + +A failed post-provision step stops the migration. The two host steps run even when the other fails, and the migration then fails, naming the steps that did; the warnings before it name the hosts to retry. Example full migration: @@ -460,6 +463,17 @@ Set-Location .\deploy -LinuxHostNames lnxhost-01,lnxhost-02 ``` +`-LinuxHostNames` limits only the Linux host step. `-AvdHostNames` limits the AVD session host step in the same way, and `-SkipLinuxHostReleaseAgentMigration` and `-SkipAvdHostScriptUpdate` leave out a step. Example update of only selected AVD session hosts: + +```powershell +Set-Location .\deploy +.\Migrate-ExistingEnvironment.ps1 ` + -EnvironmentName ` + -SkipPostProvision ` + -SkipLinuxHostReleaseAgentMigration ` + -AvdHostNames , +``` + Example host-only migration when you do not want to rerun the post-provision steps: ```powershell @@ -480,6 +494,8 @@ Set-Location .\deploy The host migration step updates only the release-agent-related files and services on existing Linux VMs. It does not reprovision infrastructure, replace the VM image, rerun the full Linux custom script extension, or attempt to reconcile every manual drift in an older environment. +The AVD session host step changes only `C:\Temp\Connect-LinuxBroker.ps1`, through Run Command, on the VMs in the resource group tagged `broker-role=avd-host`. On each running session host it downloads the script from `ScriptSourceRoot`, fills in the API URL and client ID as `Configure-AVD-Host.ps1` does, checks that the result is a broker script in plain ASCII that parses, and replaces the installed script in one step. Because the API URL and `ScriptSourceRoot` are written into PowerShell code on the session host, the update refuses either one unless it is an `https://` URL made only of ASCII letters, digits and the characters `.`, `-`, `_`, `~`, `%` and `/`, with an optional port. A download or check that fails leaves the installed script as it was, and sessions already open keep the script they started with. Session hosts that are not running are skipped and named at the end; update each one with `-AvdHostNames` once it is started. To update only the session hosts, you can also run [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) on its own, with the same `-EnvironmentName`, `-ScriptSourceRoot` and `-AvdHostNames` parameters. + The migration also rewrites `/etc/sudoers.d/avdadmin`. Older hosts were provisioned with a broad allowlist that included `cat`, `rm`, `chmod`, `chown`, `cp`, `mount`, and `umount`. The current policy grants only `userdel`, `groupadd`, `usermod`, `chpasswd`, `/usr/local/bin/create-user.sh`, `/usr/local/bin/manage-lease.sh`, `/usr/local/bin/apply-host-settings.sh`, `/usr/local/bin/session-control.sh`, and `/usr/local/bin/patch-host.sh`; all privileged file work now happens inside those root-owned scripts. The generated policy is validated with `visudo -c` and moved into place only if it passes. `apply-host-settings.sh` is the only way the broker API can change host configuration. It accepts a JSON settings document on stdin and nothing on argv, rejects unknown keys, and clamps every value to a supported range before writing anything, so a bad value cannot strand the fleet. @@ -617,6 +633,24 @@ This release changes which Linux distributions and desktops the deployment offer Every layer tolerates the others being one release behind during the rollout, and the database does not change. Hosts on agent 1.1.0 ignore the key the new API sends and keep working, flagged **Agent outdated**. A 1.2.0 host that meets the previous API, or an API without `KEYRING_VAULT_URL`, gets no key and starts the desktop as before. The previous API drops the launcher's version from the heartbeat. +## Upgrading To Start On Demand + +This release lets a checkout that finds no ready host start one, so a pool can scale to zero when nobody needs it (item 4.1 of the [roadmap](../docs/ROADMAP.md)). It needs no new Azure resources, role assignments or deployment parameters, and no Bicep change, but the AVD session hosts need the new `Connect-LinuxBroker.ps1`. + +- **Start on demand.** When a checkout finds no ready host, the API starts a stopped one for the user and answers `202` with how long to wait, instead of refusing. On the AVD session host, **Linux Desktop** shows *Your Linux desktop is starting…* with **Cancel**, asks again when the broker says to, and opens the desktop as soon as the host is reachable, for up to 10 minutes. The start is recorded as a scaling start is, in the scaling activity log as `Start On Demand` and in the audit log as `vm.start_on_demand`, and uses the API's existing **Desktop Virtualization Power On Off Contributor** role. It never takes the pool past the active rule's or window's `MaxVMs`, so a user who arrives when that many hosts are on and none is free is refused as before. Start on demand is on after the upgrade; an administrator can turn it off, and set how many hosts may start at once for waiting users (2 by default, up to 20), in the **Start on demand** card on the **Scaling** page. +- **Scale to zero.** While start on demand is on, the default rule and schedule windows may keep a minimum of 0 hosts, so idle hosts stop, for example overnight, and the first user to arrive waits a minute or two for one to start. Scaling counts waiting users as demand, and a scale-down keeps a host for each of them. A minimum of 0 cannot be saved while start on demand is off. If it is turned off afterwards, scaling keeps one host on for those rules and windows, and the card says so. +- **Connect-LinuxBroker.ps1 2.0.0.** Besides waiting for a host to start, the script tries a request again after no answer, a `408`, a `429` or a `5xx`, until three fail in a row, where the previous script tried three times at once. It requests a new token once when the broker refuses one, and tells the user in plain words what went wrong. Each attempt is logged under the **LinuxBrokerScript** source in the session host's Application event log. Session hosts keep the script they were deployed with, so the migration's new third step, [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1), replaces it on every running session host. An older script treats `202` as a failure: its user is told that no Linux host is available while a host starts for them, and gets that host at their next try once it is up. The script now reports its version with each checkout, and the **Start on demand** card shows, for the session hosts that asked for a host in the last seven days, how many run each version, and names those whose script cannot wait. +- **Waits on the dashboard.** Every time a user is told to wait, the checkout records a `Starting` event. The dashboard's checkout card adds how many users waited, how many of them got a host, the median and 95th percentile wait, and how many are waiting now, and the capacity chart shows the users who waited. **Attention** no longer reports that no host is ready for a pool scaled to zero that nobody is waiting on. + +### Recommended order + +1. Run [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1). It applies SQL scripts `144`–`156` before the new images start, restarts the apps in the order API, task, front end, migrates the Linux hosts, and then updates `Connect-LinuxBroker.ps1` on the running AVD session hosts. +2. Start any AVD session host the migration skipped because it was not running, and update it with `-SkipPostProvision -SkipLinuxHostReleaseAgentMigration -AvdHostNames `. +3. With a test user, open **Linux Desktop** while no host is free, for example with every idle host stopped, and confirm that the wait window appears and the desktop opens once the host is up. +4. Before you set any minimum to 0, confirm that the **Start on demand** card names no session host whose script cannot wait. The card goes by each session host's latest checkout in the last seven days, so a session host updated since then is listed until its next checkout. + +Every layer tolerates the others being one release behind during the rollout. Against a database without the new procedures, the new API refuses a checkout that finds no host with `409`, as before, and logs it once, and the portal says that the broker does not support start on demand yet. The previous API build keeps working against the new database: it never answers `202`, so nobody waits, and it refuses a minimum of 0 as before. Session hosts that still run an older script behave as described above. + ## Manual Steps After `azd up` ### Admin consent @@ -792,7 +826,17 @@ A device object left over from an earlier deployment that used the same VM name ### Linux Desktop opens but reports that no Linux host is available -`Connect-LinuxBroker.ps1` shows this when checkout does not return a host. Check the **LinuxBrokerScript** source in the session host's Application event log. A `403` from the API means the session host's managed identity is not yet in the AVD host group; rerun `azd hooks run postprovision`. Otherwise, confirm in the portal that at least one Linux host is available. +`Connect-LinuxBroker.ps1` shows this when the broker answers `409`: no host is ready and the broker cannot start one, because start on demand is off, the active rule or window already has `MaxVMs` hosts on, or no stopped host is free to start. Hosts in maintenance, draining or waiting for cleanup are never started. The session host's Application event log has the broker's answer to each attempt under the **LinuxBrokerScript** source, and the portal's scaling activity log shows each start on demand. Otherwise, confirm in the portal that at least one Linux host is available. + +A session host whose script is older than 2.0.0 shows the same message while a host starts for its user, after logging `Failed to retrieve VM information from API response.` three times. The **Start on demand** card on the **Scaling** page names such session hosts; update them as described in [Migration For Existing Deployments](#migration-for-existing-deployments). + +### Linux Desktop says the Linux desktop is taking longer than usual to start + +The user waited `MaxWaitSeconds` (10 minutes by default) and no host became ready for them. Usually the host the broker started did not become reachable on SSH: find it in the portal's scaling activity log under the `Start On Demand` action, then check its power state and reachability. The broker stops counting a host as starting 10 minutes after it started, so the next request can start another, if a stopped host is free and `MaxVMs` allows. A start that Azure refuses is logged by the API, and the user is then refused with `409` instead of waiting. + +### Linux Desktop reports that the Linux Broker could not authenticate this session host + +`Connect-LinuxBroker.ps1` shows this when the session host cannot get a token from its managed identity (`Failed to obtain access token` in the **LinuxBrokerScript** event log) or the broker refuses it (`The broker refused this session host (403)`). A `403` means the session host's managed identity is not yet in the AVD host group; rerun `azd hooks run postprovision`. The script renews a token the broker refuses with `401` once before it gives up. ### The portal shows "No access" after signing in diff --git a/deploy/Migrate-ExistingEnvironment.ps1 b/deploy/Migrate-ExistingEnvironment.ps1 index 58d926c..a4772e5 100644 --- a/deploy/Migrate-ExistingEnvironment.ps1 +++ b/deploy/Migrate-ExistingEnvironment.ps1 @@ -45,6 +45,9 @@ param( [Parameter(Mandatory = $false)] [string[]]$LinuxHostNames, + [Parameter(Mandatory = $false)] + [string[]]$AvdHostNames, + [Parameter(Mandatory = $false)] [ValidateRange(1, 300)] [int]$WatcherDebounceSeconds = 10, @@ -57,7 +60,10 @@ param( [switch]$SkipPostProvision, [Parameter(Mandatory = $false)] - [switch]$SkipLinuxHostReleaseAgentMigration + [switch]$SkipLinuxHostReleaseAgentMigration, + + [Parameter(Mandatory = $false)] + [switch]$SkipAvdHostScriptUpdate ) Set-StrictMode -Version Latest @@ -132,23 +138,58 @@ if (-not $SkipPostProvision) { -LinuxHostGroupId $LinuxHostGroupId } +if ((-not $SkipLinuxHostReleaseAgentMigration -or -not $SkipAvdHostScriptUpdate) -and + ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId))) { + throw 'Updating the Linux hosts and the AVD session hosts requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' +} + +# Neither host update stops the other, so a Linux host that fails does not leave the AVD session +# hosts on a Connect-LinuxBroker.ps1 that cannot wait for a host to start. Both scripts name the +# hosts to retry, and the failures end the migration together. +$failedSteps = New-Object System.Collections.Generic.List[string] + if (-not $SkipLinuxHostReleaseAgentMigration) { - if ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId)) { - throw 'Linux host release-agent migration requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' + Write-Host 'Migrating existing Linux hosts to the current release-agent layout.' + + try { + & "$PSScriptRoot/Migrate-LinuxHostReleaseAgent.ps1" ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -LinuxHostNames $LinuxHostNames ` + -WatcherDebounceSeconds $WatcherDebounceSeconds ` + -WatcherSettleSeconds $WatcherSettleSeconds + } + catch { + Write-Warning $_.Exception.Message + $failedSteps.Add('the Linux host migration') } +} - Write-Host 'Migrating existing Linux hosts to the current release-agent layout.' +if (-not $SkipAvdHostScriptUpdate) { + Write-Host 'Updating Connect-LinuxBroker.ps1 on the existing AVD session hosts.' + + try { + & "$PSScriptRoot/Update-AvdHostBrokerScript.ps1" ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -AvdHostNames $AvdHostNames + } + catch { + Write-Warning $_.Exception.Message + $failedSteps.Add('the AVD session host update') + } +} - & "$PSScriptRoot/Migrate-LinuxHostReleaseAgent.ps1" ` - -ResourceGroupName $ResourceGroupName ` - -ApiBaseUrl $ApiBaseUrl ` - -ApiClientId $ApiClientId ` - -SubscriptionId $SubscriptionId ` - -EnvironmentName $EnvironmentName ` - -ScriptSourceRoot $ScriptSourceRoot ` - -LinuxHostNames $LinuxHostNames ` - -WatcherDebounceSeconds $WatcherDebounceSeconds ` - -WatcherSettleSeconds $WatcherSettleSeconds +if ($failedSteps.Count -gt 0) { + throw "Existing environment migration failed in $($failedSteps -join ' and '). The warnings above name the hosts to retry." } Write-Host 'Existing environment migration completed.' \ No newline at end of file diff --git a/deploy/Update-AvdHostBrokerScript.ps1 b/deploy/Update-AvdHostBrokerScript.ps1 new file mode 100644 index 0000000..76ec450 --- /dev/null +++ b/deploy/Update-AvdHostBrokerScript.ps1 @@ -0,0 +1,556 @@ +<# +.SYNOPSIS + Replaces Connect-LinuxBroker.ps1 on the existing AVD session hosts with the current one. + +.DESCRIPTION + Configure-AVD-Host.ps1 installs Connect-LinuxBroker.ps1 only when a session host is + provisioned, so session hosts deployed by an earlier release keep the script they were + deployed with. A script older than 2.0.0 cannot wait while the broker starts a Linux host for + its user, and shows an error instead. + + Through Run Command, this script downloads Connect-LinuxBroker.ps1 from ScriptSourceRoot onto + every running AVD session host in the resource group (VMs tagged broker-role=avd-host), fills + in the broker API's URL and client ID as Configure-AVD-Host.ps1 does, and replaces + C:\Temp\Connect-LinuxBroker.ps1 in one step, so a user who connects meanwhile runs either the + old script or the new one. + + Session hosts that are not running are skipped and named at the end: update each one with + -AvdHostNames once it is started. Every running session host is attempted, and the ones where + the update failed are reported together at the end, with a non-zero exit code. + + Values that are not given are read from the azd environment. ScriptSourceRoot and the API URL + are written into PowerShell code, so both must be https:// URLs made only of ASCII letters, + digits and the characters . - _ ~ % and /, with an optional port. + +.EXAMPLE + .\Update-AvdHostBrokerScript.ps1 -EnvironmentName + +.EXAMPLE + .\Update-AvdHostBrokerScript.ps1 -EnvironmentName -AvdHostNames +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $false)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $false)] + [string]$ApiClientId, + + [Parameter(Mandatory = $false)] + [string]$SubscriptionId, + + [Parameter(Mandatory = $false)] + [string]$EnvironmentName, + + [Parameter(Mandatory = $false)] + [string]$ScriptSourceRoot = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main', + + [Parameter(Mandatory = $false)] + [string[]]$AvdHostNames +) + +$CompletionMarker = 'Updated Connect-LinuxBroker.ps1 on' + +function Get-AzdEnvValue { + param( + [AllowEmptyString()] + [string]$EnvironmentName, + + [Parameter(Mandatory = $true)] + [string]$Key + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + return '' + } + + $value = azd env get-value $Key --environment $EnvironmentName 2>$null + if ($LASTEXITCODE -ne 0) { + return '' + } + + return ($value | Out-String).Trim() +} + +# A property's value, or $null when the object or the property is missing, which StrictMode +# would otherwise refuse. +function Get-PropertyValue { + param( + $InputObject, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$Name] + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Get-VmTagValue { + param( + [Parameter(Mandatory = $true)] + $Vm, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [string](Get-PropertyValue -InputObject (Get-PropertyValue -InputObject $Vm -Name 'tags') -Name $Name) +} + +function ConvertTo-PowerShellLiteral { + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Value + ) + + return "'" + [System.Management.Automation.Language.CodeGeneration]::EscapeSingleQuotedStringContent($Value) + "'" +} + +# Whether a URL can be written into PowerShell code without ending the string it is in. Only plain +# ASCII URL characters pass: PowerShell also ends strings at typographic quotes, and Windows +# PowerShell reads a script without a byte order mark in the ANSI code page, where some accented +# letters turn into quotes. The match is case-sensitive, because matching without regard to case +# lets the Kelvin sign through as a k. +function Test-SafeHttpsUrl { + param( + [AllowEmptyString()] + [string]$Value + ) + + if ($Value -cnotmatch '^https://[A-Za-z0-9._-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~%/-]*)?\z') { + return $false + } + + # The pattern lets through a port above 65535, which a URI cannot have. + $uri = $null + return [Uri]::TryCreate($Value, [UriKind]::Absolute, [ref]$uri) +} + +# The script Run Command runs on each session host, in Windows PowerShell 5.1 as SYSTEM. +function Get-AvdHostUpdateScript { + param( + [Parameter(Mandatory = $true)] + [string]$SourceUrl, + + [Parameter(Mandatory = $true)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $true)] + [string]$ApiClientId, + + [Parameter(Mandatory = $false)] + [string]$Destination = 'C:\Temp\Connect-LinuxBroker.ps1' + ) + + $template = @' +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$sourceUrl = __SOURCE_URL__ +$apiBaseUrl = __API_BASE_URL__ +$apiClientId = __API_CLIENT_ID__ +$destination = __DESTINATION__ + +$baseUrlPlaceholder = 'https://your_linuxbroker_api_base_url/api' +$clientIdPlaceholder = 'your_linuxbroker_api_client_id' + +function Get-BrokerScriptVersion { + param([string]$Text) + + $match = [regex]::Match($Text, '(?m)^\$ScriptVersion = ''([^'']+)''') + if ($match.Success) { + return $match.Groups[1].Value + } + return $null +} + +$folder = Split-Path -Parent $destination +if (-not (Test-Path -LiteralPath $folder)) { + New-Item -ItemType Directory -Path $folder -Force | Out-Null +} + +$previous = 'no script' +if (Test-Path -LiteralPath $destination) { + $previous = Get-BrokerScriptVersion ([System.IO.File]::ReadAllText($destination)) + if (-not $previous) { + $previous = 'a script without a version' + } +} + +$staged = '{0}.{1}.download' -f $destination, [guid]::NewGuid().ToString('N') +try { + $attempt = 0 + while ($true) { + $attempt++ + try { + Invoke-WebRequest -Uri $sourceUrl -OutFile $staged -UseBasicParsing + break + } + catch { + if ($attempt -ge 3) { + throw "Could not download $sourceUrl after $attempt attempts: $($_.Exception.Message)" + } + Start-Sleep -Seconds (2 * $attempt) + } + } + + $text = [System.IO.File]::ReadAllText($staged) + $version = Get-BrokerScriptVersion $text + if (-not $version -or -not $text.Contains($baseUrlPlaceholder) -or -not $text.Contains($clientIdPlaceholder)) { + throw "$sourceUrl is not a Connect-LinuxBroker.ps1 that this update can install." + } + + $text = $text.Replace($baseUrlPlaceholder, $apiBaseUrl).Replace($clientIdPlaceholder, $apiClientId) + + # The script is written without a byte order mark, which Windows PowerShell reads in the ANSI + # code page, so any character outside ASCII could read differently, even as a quote. + if ($text -cmatch '[^\x00-\x7F]') { + throw "Connect-LinuxBroker.ps1 $version holds characters outside ASCII, which Windows PowerShell could misread." + } + + $parseErrors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($text, [ref]$null, [ref]$parseErrors) + if ($parseErrors) { + throw "Connect-LinuxBroker.ps1 $version does not parse once the API URL and client ID are filled in: $($parseErrors[0].Message)" + } + + # Without a byte order mark, as Configure-AVD-Host.ps1 writes it. + [System.IO.File]::WriteAllText($staged, $text, (New-Object System.Text.UTF8Encoding -ArgumentList $false)) + + # In one step, so a user who connects meanwhile runs either the old script or the new one. + $attempt = 0 + while ($true) { + $attempt++ + try { + if (Test-Path -LiteralPath $destination) { + [System.IO.File]::Replace($staged, $destination, [NullString]::Value) + } + else { + [System.IO.File]::Move($staged, $destination) + } + break + } + catch { + if ($attempt -ge 5) { + throw "Could not replace ${destination}: $($_.Exception.Message)" + } + Start-Sleep -Seconds 1 + } + } +} +finally { + Remove-Item -LiteralPath $staged -Force -ErrorAction SilentlyContinue +} + +Unblock-File -LiteralPath $destination +if ((Get-BrokerScriptVersion ([System.IO.File]::ReadAllText($destination))) -ne $version) { + throw "$destination does not hold Connect-LinuxBroker.ps1 $version after the update." +} + +# Configure-AVD-Host.ps1 creates the event source the script logs to, and installs the module it +# saves the Linux host's credentials with. +try { + if (-not [System.Diagnostics.EventLog]::SourceExists('LinuxBrokerScript')) { + New-EventLog -LogName Application -Source 'LinuxBrokerScript' + } +} +catch { + Write-Output "WARNING: The LinuxBrokerScript event source could not be created, so Connect-LinuxBroker.ps1 cannot log: $($_.Exception.Message)" +} + +if (-not (Get-Module -ListAvailable -Name CredentialManager)) { + Write-Output 'WARNING: The CredentialManager module is not installed, so Connect-LinuxBroker.ps1 cannot save the Linux host credentials. Install it with Install-Module CredentialManager -Scope AllUsers.' +} + +Write-Output ('Updated Connect-LinuxBroker.ps1 on {0} from {1} to {2}.' -f $env:COMPUTERNAME, $previous, $version) +'@ + + $values = @{ + SOURCE_URL = ConvertTo-PowerShellLiteral -Value $SourceUrl + API_BASE_URL = ConvertTo-PowerShellLiteral -Value $ApiBaseUrl + API_CLIENT_ID = ConvertTo-PowerShellLiteral -Value $ApiClientId + DESTINATION = ConvertTo-PowerShellLiteral -Value $Destination + } + + # In one pass, so a value that happens to contain a token is not replaced again. + return [regex]::Replace($template, '__(SOURCE_URL|API_BASE_URL|API_CLIENT_ID|DESTINATION)__', { + param($match) + $values[$match.Groups[1].Value] + }) +} + +# Returns what the script wrote to standard output and standard error on the session host. +function Invoke-RunCommandWithRetry { + param( + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$VmName, + + [Parameter(Mandatory = $true)] + [string]$Script, + + [Parameter(Mandatory = $false)] + [int]$MaxAttempts = 3, + + [Parameter(Mandatory = $false)] + [int]$InitialDelaySeconds = 5 + ) + + $lastError = '' + + # The script reaches az as @file. Passed inline, on Windows it goes through az.cmd, where + # cmd.exe ends the command at the first newline, and the session host runs only the first line. + $scriptFile = Join-Path ([System.IO.Path]::GetTempPath()) ('linuxbroker-avd-update-{0}.ps1' -f [guid]::NewGuid().ToString('N')) + [System.IO.File]::WriteAllText($scriptFile, $Script, (New-Object System.Text.UTF8Encoding -ArgumentList $false)) + + try { + for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { + $output = @(az vm run-command invoke ` + --resource-group $ResourceGroupName ` + --name $VmName ` + --command-id RunPowerShellScript ` + --scripts "@$scriptFile" ` + --output json ` + --only-show-errors 2>&1) + $exitCode = $LASTEXITCODE + $errorText = (@($output | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] } | ForEach-Object { $_.ToString() }) -join [Environment]::NewLine).Trim() + $json = (@($output | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] } | ForEach-Object { [string]$_ }) -join "`n").Trim() + + if ($exitCode -eq 0) { + try { + $result = ConvertFrom-Json -InputObject $json + } + catch { + throw "Run Command on '$VmName' returned output that is not JSON: $json" + } + + $standardOutput = New-Object System.Collections.Generic.List[string] + $standardError = New-Object System.Collections.Generic.List[string] + foreach ($status in @(Get-PropertyValue -InputObject $result -Name 'value')) { + $message = [string](Get-PropertyValue -InputObject $status -Name 'message') + if ([string](Get-PropertyValue -InputObject $status -Name 'code') -like '*StdErr*') { + $standardError.Add($message) + } + else { + $standardOutput.Add($message) + } + } + + return [pscustomobject]@{ + StdOut = ($standardOutput -join "`n").Trim() + StdErr = ($standardError -join "`n").Trim() + } + } + + $lastError = $errorText + if ($attempt -ge $MaxAttempts) { + break + } + + $delaySeconds = [Math]::Min($InitialDelaySeconds * [Math]::Pow(2, $attempt - 1), 30) + Write-Warning "Run Command failed on '$VmName' attempt $attempt of $MaxAttempts. Retrying in $([int]$delaySeconds) seconds." + if (-not [string]::IsNullOrWhiteSpace($lastError)) { + Write-Warning $lastError + } + + Start-Sleep -Seconds ([int]$delaySeconds) + } + } + finally { + Remove-Item -LiteralPath $scriptFile -Force -ErrorAction SilentlyContinue + } + + if ([string]::IsNullOrWhiteSpace($lastError)) { + throw "Run Command failed on '$VmName'." + } + + throw "Run Command failed on '$VmName'. Last error: $lastError" +} + +function Invoke-AvdHostBrokerScriptUpdate { + param( + [string]$ResourceGroupName, + [string]$ApiBaseUrl, + [string]$ApiClientId, + [string]$SubscriptionId, + [string]$EnvironmentName, + [string]$ScriptSourceRoot = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main', + [string[]]$AvdHostNames + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + $EnvironmentName = if (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENV_NAME)) { + $env:AZURE_ENV_NAME + } + elseif (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENVIRONMENT_NAME)) { + $env:AZURE_ENVIRONMENT_NAME + } + else { + '' + } + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName)) { + $ResourceGroupName = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'resourceGroupName' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'AZURE_SUBSCRIPTION_ID' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = $env:AZURE_SUBSCRIPTION_ID + } + + if ([string]::IsNullOrWhiteSpace($ApiBaseUrl)) { + $ApiBaseUrl = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'apiUrl' + } + + if ([string]::IsNullOrWhiteSpace($ApiClientId)) { + $ApiClientId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'apiClientId' + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId)) { + throw 'The AVD session host update needs ResourceGroupName, ApiBaseUrl and ApiClientId, as parameters or azd environment values.' + } + + # The API URL and client ID end up inside double-quoted strings in Connect-LinuxBroker.ps1, and + # the source URL in the script Run Command runs as SYSTEM. + $ApiBaseUrl = $ApiBaseUrl.Trim().TrimEnd('/') + if (-not (Test-SafeHttpsUrl -Value $ApiBaseUrl)) { + throw "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not '$ApiBaseUrl'." + } + + $ApiClientId = $ApiClientId.Trim() + $parsedClientId = [guid]::Empty + if (-not [guid]::TryParseExact($ApiClientId, 'D', [ref]$parsedClientId)) { + throw "ApiClientId must be the broker API app registration's client ID, a GUID, not '$ApiClientId'." + } + + $ScriptSourceRoot = ([string]$ScriptSourceRoot).Trim().TrimEnd('/') + if (-not (Test-SafeHttpsUrl -Value $ScriptSourceRoot)) { + throw "ScriptSourceRoot must be the https:// URL of the repository or of a mirror of it, such as https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main, not '$ScriptSourceRoot'." + } + + $sourceUrl = "$ScriptSourceRoot/avd_host/broker/Connect-LinuxBroker.ps1" + + if (-not [string]::IsNullOrWhiteSpace($SubscriptionId)) { + az account set --subscription $SubscriptionId | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Could not select subscription '$SubscriptionId'." + } + } + + $vmJson = (@(az vm list --resource-group $ResourceGroupName --show-details --output json --only-show-errors) -join "`n") + if ($LASTEXITCODE -ne 0) { + throw "Could not list the VMs in resource group '$ResourceGroupName'." + } + + $avdHosts = @(ConvertFrom-Json -InputObject $vmJson | ForEach-Object { $_ } | + Where-Object { (Get-VmTagValue -Vm $_ -Name 'broker-role') -eq 'avd-host' }) + + if ($avdHosts.Count -eq 0) { + Write-Host "No AVD session host VMs found in resource group '$ResourceGroupName'." + return + } + + if ($AvdHostNames -and $AvdHostNames.Count -gt 0) { + $requestedNames = New-Object 'System.Collections.Generic.HashSet[string]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($name in $AvdHostNames) { + if (-not [string]::IsNullOrWhiteSpace($name)) { + [void]$requestedNames.Add($name.Trim()) + } + } + + $filteredHosts = @($avdHosts | Where-Object { $requestedNames.Contains([string]$_.name) }) + foreach ($name in $requestedNames) { + if (-not ($filteredHosts | Where-Object { $_.name -eq $name })) { + Write-Warning "Requested AVD session host '$name' was not found in resource group '$ResourceGroupName'." + } + } + + $avdHosts = $filteredHosts + } + + if ($avdHosts.Count -eq 0) { + Write-Host 'No AVD session hosts matched the requested update scope.' + return + } + + $remoteScript = Get-AvdHostUpdateScript -SourceUrl $sourceUrl -ApiBaseUrl $ApiBaseUrl -ApiClientId $ApiClientId + + # One session host that is off or failing must not leave the rest on the old script, so every + # one is attempted and the failures are reported together at the end. + $updated = New-Object System.Collections.Generic.List[string] + $failures = New-Object System.Collections.Generic.List[string] + $skipped = New-Object System.Collections.Generic.List[string] + + foreach ($avdHost in $avdHosts) { + $name = [string]$avdHost.name + $powerState = [string](Get-PropertyValue -InputObject $avdHost -Name 'powerState') + if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -notmatch 'running') { + Write-Warning "Skipping AVD session host '$name' because it is not running ($powerState). Start it and run this script again with -AvdHostNames $name." + $skipped.Add($name) + continue + } + + Write-Host "Updating Connect-LinuxBroker.ps1 on AVD session host '$name'..." + try { + $output = Invoke-RunCommandWithRetry -ResourceGroupName $ResourceGroupName -VmName $name -Script $remoteScript + if (-not [string]::IsNullOrWhiteSpace($output.StdOut)) { + Write-Host $output.StdOut + } + # Run Command reports success whatever the script did, so the script's own closing + # line is the only evidence that it ran to the end. + if ($output.StdOut -notmatch [regex]::Escape($CompletionMarker)) { + $detail = if ([string]::IsNullOrWhiteSpace($output.StdErr)) { '' } else { " $($output.StdErr)" } + throw "The update did not run to completion on '$name'.$detail" + } + $updated.Add($name) + } + catch { + Write-Warning $_.Exception.Message + $failures.Add($name) + } + } + + if ($skipped.Count -gt 0) { + Write-Warning "Not updated because they are not running: $($skipped -join ', ')." + } + + if ($failures.Count -gt 0) { + throw "The Connect-LinuxBroker.ps1 update failed on: $($failures -join ', '). Rerun with -AvdHostNames to retry them." + } + + Write-Host "Connect-LinuxBroker.ps1 is up to date on $($updated.Count) AVD session host(s)." +} + +# Dot-sourcing the script, as its tests do, defines the functions without running the update. +if ($MyInvocation.InvocationName -ne '.') { + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + + Invoke-AvdHostBrokerScriptUpdate ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -AvdHostNames $AvdHostNames +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4c035af..8eff579 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -844,7 +844,36 @@ there before Phase 3. ### 4.1 Start a host on demand at checkout -**Status: Planned** · depends on Phase 1 (readiness and scaling fixes) +**Status: Done** · depends on Phase 1 (readiness and scaling fixes) + +**Shipped.** A checkout that finds no ready host starts one and answers `202`, the AVD script +waits for it, and a rule or window may keep a minimum of 0 hosts. Where it differs from the +design below: + +- `ReserveVmForStart` runs under the scaling application lock, so it never races a scaling + run. It starts at most one host per waiting user, and at most `MaxPendingStarts` at once (2 by + default, 1–20), never past the active phase's `MaxVMs`. A user who arrives while enough hosts + are starting waits for one of them. The API commits before it asks Azure, and records the host + as off again if Azure refuses. +- `retryAfterSeconds` isn't fixed at 60: it's the median start-to-reachable time over the last + week, less how long the oldest starting host has taken, kept between 30 and 120 seconds, and + the `Retry-After` header carries it too. While a user waits, each request probes the starting + hosts on port 22, so a host is handed out without waiting for the task's minute-by-minute + probe. +- Start on demand is on by default. The **Start on demand** card on the Scaling page turns it + off and sets the pending-start limit. A minimum of 0 can be saved only while it's on, and if + it's turned off later, scaling reads 0 as 1. +- The AVD script (2.0.0) waits in a modeless window with **Cancel**, up to `-MaxWaitSeconds` + (600 by default). It asks again after `Retry-After`, or 30, 30, then 60 seconds when the + broker gives none, gives up after three transient failures in a row, and renews a refused + token once. + It sends `clientVersion` with each checkout, and the Scaling page names the AVD hosts whose + latest checkout reported none, because their script can't wait. +- `Update-AvdHostBrokerScript.ps1` replaces the script on existing AVD hosts through Run + Command, and `Migrate-ExistingEnvironment.ps1` runs it after the Linux host migration. +- Checkout events gain a `Starting` outcome. Scaling counts waiting users as demand and keeps a + host for each of them when scaling down, and the dashboard reports waits, how many were + served, and their median and 95th percentile. **Why.** When the pool is exhausted, the user sees "No Linux host is available right now. Try again in a few minutes." (`Connect-LinuxBroker.ps1`). The script retries three times diff --git a/front_end/README.md b/front_end/README.md index 74d3df1..2896f4a 100644 --- a/front_end/README.md +++ b/front_end/README.md @@ -99,8 +99,8 @@ a path that serves the SPA shell. | POST | `/api/ui/scaling/rules//delete` | | | GET | `/api/ui/scaling/log` | Paged. | | GET | `/api/ui/scaling/rules/history` | Paged. | -| GET | `/api/ui/scaling/policy` | The time zone, the default rule, the windows, and what applies now and next. `{Available: false}` when the broker predates it, so the Scaling section offers the scaling rules instead. | -| POST | `/api/ui/scaling/policy` | Body `{timezone}`. | +| GET | `/api/ui/scaling/policy` | The time zone, the default rule, the windows, what applies now and next, the start on demand settings, and the broker script versions the AVD hosts run. `{Available: false}` when the broker predates it, so the Scaling section offers the scaling rules instead. | +| POST | `/api/ui/scaling/policy` | Body with any of `timezone`, `startondemandenabled` and `maxpendingstarts`, applied together. | | GET | `/api/ui/scaling/timezones` | | | POST | `/api/ui/scaling/schedules` | Adds a window. | | POST | `/api/ui/scaling/schedules//update` | | diff --git a/front_end/route_scaling_management.py b/front_end/route_scaling_management.py index 4bd213b..bd985c9 100644 --- a/front_end/route_scaling_management.py +++ b/front_end/route_scaling_management.py @@ -92,10 +92,28 @@ def ui_scaling_policy(): @login_required @broker_endpoint("Unable to update the scaling policy. Please try again later.") def ui_update_scaling_policy(): - zone = json_body().get('timezone') - if not isinstance(zone, str) or not zone.strip(): - raise BadRequest("Choose a time zone.") - return jsonify(api_post('/scaling/policy/update', {'timezone': zone.strip()[:64]})) + # Any of the time zone and the start on demand settings; the broker validates them and + # applies them together. + body = json_body() + payload = {} + if 'timezone' in body: + zone = body.get('timezone') + if not isinstance(zone, str) or not zone.strip(): + raise BadRequest("Choose a time zone.") + payload['timezone'] = zone.strip()[:64] + if 'startondemandenabled' in body: + enabled = body.get('startondemandenabled') + if not isinstance(enabled, bool): + raise BadRequest("Choose whether start on demand is on.") + payload['startondemandenabled'] = enabled + if 'maxpendingstarts' in body: + pending = body.get('maxpendingstarts') + if isinstance(pending, bool) or not isinstance(pending, (int, str)) or (isinstance(pending, str) and len(pending) > 8): + raise BadRequest("Enter how many hosts may start at once, from 1 to 20.") + payload['maxpendingstarts'] = pending + if not payload: + raise BadRequest("Choose a time zone or the start on demand settings.") + return jsonify(api_post('/scaling/policy/update', payload)) @app.route(f'{API_PREFIX}/scaling/timezones') @login_required diff --git a/front_end/tests/test_scaling_policy_bff.py b/front_end/tests/test_scaling_policy_bff.py index 80d8410..eb19413 100644 --- a/front_end/tests/test_scaling_policy_bff.py +++ b/front_end/tests/test_scaling_policy_bff.py @@ -68,6 +68,33 @@ def test_the_time_zone_is_set(signed_in_client, broker_api): assert broker_api.posts[-1]["json"] == {"timezone": "UTC"} +def test_start_on_demand_is_forwarded_with_or_without_the_time_zone(signed_in_client, broker_api): + broker_api.post_replies["/scaling/policy/update"] = (200, {"Result": "Updated", "StartOnDemandEnabled": False}) + + response = post(signed_in_client, f"{API}/scaling/policy", + {"startondemandenabled": False, "maxpendingstarts": 4, "junk": True}) + + assert response.status_code == 200 and response.get_json()["StartOnDemandEnabled"] is False + assert broker_api.posts[-1]["json"] == {"startondemandenabled": False, "maxpendingstarts": 4} + post(signed_in_client, f"{API}/scaling/policy", {"timezone": " UTC ", "maxpendingstarts": "3"}) + assert broker_api.posts[-1]["json"] == {"timezone": "UTC", "maxpendingstarts": "3"} + + +@pytest.mark.parametrize("payload", [ + {}, {"junk": True}, {"startondemandenabled": "yes"}, {"maxpendingstarts": True}, + {"maxpendingstarts": 2.5}, {"maxpendingstarts": "1" * 9}, {"timezone": "UTC", "startondemandenabled": 1}, +]) +def test_an_unusable_policy_update_never_reaches_the_broker(signed_in_client, broker_api, payload): + assert post(signed_in_client, f"{API}/scaling/policy", payload).status_code == 400 + assert broker_api.posts == [] + + +def test_the_brokers_refusal_of_the_pending_limit_is_kept(signed_in_client, broker_api): + broker_api.post_replies["/scaling/policy/update"] = (400, {"error": "maxpendingstarts must be a whole number from 1 to 20."}) + response = post(signed_in_client, f"{API}/scaling/policy", {"maxpendingstarts": 40}) + assert response.status_code == 400 and "1 to 20" in response.get_json()["error"] + + def test_the_preview_forwards_a_time_and_proposed_values(signed_in_client, broker_api): broker_api.get_replies["/scaling/preview"] = (200, {"Action": "None", "Summary": "No change."}) signed_in_client.get(f"{API}/scaling/preview?at=2026-09-21T09:00:00Z") diff --git a/front_end/web/src/App.test.tsx b/front_end/web/src/App.test.tsx index 9d56f61..64bd4a2 100644 --- a/front_end/web/src/App.test.tsx +++ b/front_end/web/src/App.test.tsx @@ -232,6 +232,13 @@ const SCALING_POLICY = { Schedules: [{ ...SCHEDULE_BASE, ScheduleID: 1, Name: 'Business hours', StartTime: '08:00', EndTime: '18:00' }], NextChange: { InMinutes: 465, AtLocal: 'Thursday 18:00', PhaseName: 'Default rule', ScheduleID: null }, LastRun: null, + StartOnDemandEnabled: true, + MaxPendingStarts: 2, + ZeroMinimumCount: 0, + AvdHostScripts: { + Seen: 3, Outdated: 1, OutdatedHostnames: ['avd-03'], CurrentVersion: '2.0.0', + Versions: [{ ClientVersion: '2.0.0', AvdHosts: 2, Current: true }], + }, }; let scalingPolicy: typeof SCALING_POLICY | { Available: false } = SCALING_POLICY; @@ -411,9 +418,14 @@ function stubFetch() { if (url.startsWith('/api/ui/scaling/log')) return jsonResponse(EMPTY_PAGE); if (url.startsWith('/api/ui/scaling/rules')) return jsonResponse(RULES); if (url.startsWith('/api/ui/scaling/policy')) { - return method === 'POST' - ? jsonResponse({ TimeZone: 'Eastern Standard Time', message: 'Schedules are now read in Eastern Standard Time.' }) - : jsonResponse(scalingPolicy); + if (method !== 'POST') return jsonResponse(scalingPolicy); + const body = JSON.parse(String(init?.body)) as Record; + return 'startondemandenabled' in body + ? jsonResponse({ + Result: 'Updated', StartOnDemandEnabled: body.startondemandenabled, MaxPendingStarts: body.maxpendingstarts, + ZeroMinimumCount: 0, message: `Start on demand is now ${body.startondemandenabled ? 'on' : 'off'}.`, + }) + : jsonResponse({ Result: 'Updated', TimeZone: 'Eastern Standard Time', message: 'Schedules are now read in Eastern Standard Time.' }); } if (url.startsWith('/api/ui/scaling/preview')) return jsonResponse(method === 'POST' ? PROPOSED_PREVIEW : PREVIEW); if (url.startsWith('/api/ui/scaling/timezones')) return jsonResponse(TIME_ZONES); @@ -1390,6 +1402,94 @@ describe('App', () => { expect(screen.queryByRole('link', { name: 'Add a window' })).not.toBeInTheDocument(); expect(screen.queryByRole('button', { name: 'Edit Business hours' })).not.toBeInTheDocument(); expect(screen.queryByLabelText('Time zone')).not.toBeInTheDocument(); + expect(screen.getByText('On. Up to 2 hosts start at once for waiting users.')).toBeInTheDocument(); + expect(screen.queryByRole('switch', { name: 'Start a host when a user finds none free' })).not.toBeInTheDocument(); + }); + + it('shows start on demand and which AVD hosts cannot wait for a host', async () => { + renderApp('/scaling'); + + expect(await screen.findByRole('heading', { name: 'Start on demand' })).toBeInTheDocument(); + expect(screen.getByRole('switch', { name: 'Start a host when a user finds none free' })).toBeChecked(); + expect(screen.getByText(/1 AVD host runs a script that cannot wait for a host to start: avd-03\./)).toBeInTheDocument(); + expect(screen.getByText('deploy/Update-AvdHostBrokerScript.ps1')).toBeInTheDocument(); + const scripts = screen.getByRole('region', { name: 'AVD host scripts' }); + expect(within(scripts).getByText('Current')).toBeInTheDocument(); + expect(within(scripts).getByText('Cannot wait')).toBeInTheDocument(); + }); + + it('turns start on demand off only after warning about minimums of 0', async () => { + scalingPolicy = { ...SCALING_POLICY, ZeroMinimumCount: 2 }; + renderApp('/scaling'); + + await userEvent.click(await screen.findByRole('switch', { name: 'Start a host when a user finds none free' })); + await userEvent.click(screen.getByRole('button', { name: 'Save start on demand' })); + + const dialog = await screen.findByRole('dialog'); + expect(within(dialog).getByText(/2 rules and windows keep a minimum of 0 hosts/)).toBeInTheDocument(); + const posted = () => + (globalThis.fetch as ReturnType).mock.calls.find( + ([url, init]) => String(url) === '/api/ui/scaling/policy' && init?.method === 'POST', + ); + expect(posted()).toBeUndefined(); + await userEvent.click(within(dialog).getByRole('button', { name: 'Turn it off' })); + + await waitFor(() => expect(JSON.parse(String(posted()?.[1]?.body))).toEqual({ startondemandenabled: false, maxpendingstarts: 2 })); + expect(await screen.findByText('Start on demand is now off.')).toBeInTheDocument(); + }); + + it('saves how many hosts may start at once and refuses a limit outside 1 to 20', async () => { + renderApp('/scaling'); + const limit = await screen.findByLabelText('Hosts that may start at once'); + const save = screen.getByRole('button', { name: 'Save start on demand' }); + expect(save).toBeDisabled(); + + await userEvent.clear(limit); + await userEvent.type(limit, '25'); + expect(screen.getByText('Enter a whole number from 1 to 20.')).toBeInTheDocument(); + expect(save).toBeDisabled(); + + await userEvent.clear(limit); + await userEvent.type(limit, '4'); + await userEvent.click(save); + await waitFor(() => { + const call = (globalThis.fetch as ReturnType).mock.calls.find( + ([url, init]) => String(url) === '/api/ui/scaling/policy' && init?.method === 'POST', + ); + expect(JSON.parse(String(call?.[1]?.body))).toEqual({ startondemandenabled: true, maxpendingstarts: 4 }); + }); + // Leaving it on needs no confirmation. + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + it('says when a broker cannot start hosts on demand yet', async () => { + const older: Record = { ...SCALING_POLICY }; + for (const key of ['StartOnDemandEnabled', 'MaxPendingStarts', 'ZeroMinimumCount', 'AvdHostScripts']) { + delete older[key]; + } + scalingPolicy = older as typeof SCALING_POLICY; + renderApp('/scaling'); + expect(await screen.findByText(/This broker does not support start on demand yet/)).toBeInTheDocument(); + expect(screen.queryByRole('switch', { name: 'Start a host when a user finds none free' })).not.toBeInTheDocument(); + }); + + it('lets a rule keep a minimum of 0 only while start on demand is on', async () => { + renderApp('/scaling/rules/create'); + const minimum = await screen.findByLabelText('Minimum VMs'); + await waitFor(() => expect(minimum).toHaveAttribute('min', '0')); + await userEvent.type(minimum, '0'); + expect(screen.getByText(/With a minimum of 0, no host runs while nobody needs one/)).toBeInTheDocument(); + expect(minimum).not.toHaveAttribute('aria-invalid'); + }); + + it('flags a minimum of 0 while start on demand is off', async () => { + scalingPolicy = { ...SCALING_POLICY, StartOnDemandEnabled: false }; + renderApp('/scaling/rules/create'); + const minimum = await screen.findByLabelText('Minimum VMs'); + await userEvent.type(minimum, '0'); + expect(await screen.findByText(/A minimum of 0 needs start on demand, which is off/)).toBeInTheDocument(); + expect(minimum).toHaveAttribute('aria-invalid', 'true'); + expect(minimum).toHaveAttribute('min', '1'); }); it('changes the policy time zone', async () => { diff --git a/front_end/web/src/components/dashboard/AttentionPanel.tsx b/front_end/web/src/components/dashboard/AttentionPanel.tsx index 22a4094..e966bc4 100644 --- a/front_end/web/src/components/dashboard/AttentionPanel.tsx +++ b/front_end/web/src/components/dashboard/AttentionPanel.tsx @@ -38,7 +38,12 @@ export function describeAttention(item: AttentionItem): Described { switch (item.Kind) { case 'no-ready-hosts': return { - text: <>No host can take a new user: anyone who needs one now will be turned away., + text: ( + <> + No host can take a new user and none is starting: anyone who needs one now must wait for one to start, or + is turned away. + + ), to: '/vms', action: 'View hosts', }; diff --git a/front_end/web/src/components/dashboard/CapacityCard.tsx b/front_end/web/src/components/dashboard/CapacityCard.tsx index 9821161..cc9cda7 100644 --- a/front_end/web/src/components/dashboard/CapacityCard.tsx +++ b/front_end/web/src/components/dashboard/CapacityCard.tsx @@ -63,6 +63,7 @@ export function describeCapacity(series: UtilizationPoint[], hours: UtilizationH const serviceable = range(series.map((point) => point.Serviceable)); const maximum = range(series.map((point) => point.MaxVMs)); const denied = series.reduce((total, point) => total + point.Denied, 0); + const waited = series.reduce((most, point) => Math.max(most, point.Waited ?? 0), 0); const parts = [`Capacity over ${windowPhrase(hours)}.`]; if (peak) parts.push(`At most ${peak.max} hosts were in use at once.`); @@ -74,10 +75,17 @@ export function describeCapacity(series: UtilizationPoint[], hours: UtilizationH ); } if (maximum) parts.push(`The scaling maximum was ${maximum.max === maximum.min ? maximum.max : `${maximum.min} to ${maximum.max}`}.`); + if (waited) parts.push(`Users waited for a host to start, at most ${waited} in one interval.`); parts.push(denied ? `${denied} checkout${denied === 1 ? '' : 's'} found no host.` : 'Every checkout found a host.'); return parts.join(' '); } +/** The latest scaling run left no host powered on, as a minimum of 0 allows. */ +export function scaledToZero(series: UtilizationPoint[]) { + const latest = [...series].reverse().find((point) => point.Runs > 0); + return latest !== undefined && latest.PoweredOn === 0 && latest.MinVMs === 0; +} + export interface CapacityCardProps { hours: UtilizationHours; onHoursChange: (hours: UtilizationHours) => void; @@ -93,7 +101,9 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam // While another window loads, the previous one stays on screen and is described as such. const shownHours = data?.Hours ?? hours; const hasRuns = series.some((point) => point.Runs > 0); - const hasCheckouts = series.some((point) => point.Checkouts > 0); + const hasCheckouts = series.some((point) => point.Checkouts > 0 || (point.Waited ?? 0) > 0); + const hasWaits = series.some((point) => (point.Waited ?? 0) > 0); + const atZero = scaledToZero(series); const lines = useMemo( () => [ @@ -128,8 +138,21 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam step: true, width: 1.5, }, + ...(hasWaits + ? [ + { + key: 'waited', + label: 'Users who waited for a host', + values: series.map((point) => point.Waited ?? 0), + colour: 'var(--lb-info-fg)', + dash: '6 2 1 2', + step: true, + width: 1.5, + }, + ] + : []), ], - [series], + [series, hasWaits], ); const markers = useMemo( () => series.flatMap((point, index) => (point.Denied > 0 ? [{ index, count: point.Denied }] : [])), @@ -159,15 +182,26 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam icon="activity" /> ) : ( - point.BucketStartUtc)} - bucketMinutes={data.BucketMinutes ?? 60} - series={lines} - markers={markers} - markerLabel="checkouts found no host" - summary={describeCapacity(series, shownHours)} - caption={`Capacity over ${windowPhrase(shownHours)}, by ${data.BucketMinutes ?? 60}-minute interval`} - /> + <> + point.BucketStartUtc)} + bucketMinutes={data.BucketMinutes ?? 60} + series={lines} + markers={markers} + markerLabel="checkouts found no host" + summary={describeCapacity(series, shownHours)} + caption={`Capacity over ${windowPhrase(shownHours)}, by ${data.BucketMinutes ?? 60}-minute interval`} + /> + {atZero ? ( +

+ + + Scaled to zero: no host was powered on after the latest scaling run. Start on demand starts one for + the next user, who waits a minute or two. + +

+ ) : null} + )} diff --git a/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx b/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx index 13a786d..f6a04b8 100644 --- a/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx +++ b/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx @@ -44,6 +44,27 @@ function Figure({ ); } +/** + * Users who waited while start on demand started a host: how long those who got one waited, + * and how many are waiting now. + */ +export function describeWaits(stats: CheckoutStats): string { + const waits = stats.Waits ?? 0; + const served = stats.WaitsServed ?? 0; + const waitingNow = stats.WaitingNow ?? 0; + const parts: string[] = []; + if (served) { + parts.push(`Median ${formatDuration(stats.WaitP50Seconds)} · 95% within ${formatDuration(stats.WaitP95Seconds)}`); + } + if (waits > served) { + parts.push(`${formatNumber(waits - served)} without a host yet`); + } + if (waitingNow) { + parts.push(`${formatNumber(waitingNow)} waiting now`); + } + return parts.length ? parts.join(' · ') : 'Nobody waited for a host to start'; +} + /** How checkouts went: unmet demand, how long users waited, and how long hosts take to start. */ export function CheckoutHealthCard({ stats, @@ -98,6 +119,13 @@ export function CheckoutHealthCard({ : 'No host was started' } /> + {stats.Waits !== undefined ? ( +
+ ) : null} {failed ? (
{ it('says when every checkout found a host', () => { expect(describeCapacity([point({})], 24)).toContain('Every checkout found a host.'); }); + + it('says when users waited for a host to start', () => { + const series = [point({ Waited: 1 }), point({ Waited: 3 }), point({})]; + expect(describeCapacity(series, 24)).toContain('Users waited for a host to start, at most 3 in one interval.'); + expect(describeCapacity([point({ Waited: 0 })], 24)).not.toContain('waited'); + }); +}); + +describe('scaledToZero', () => { + it('is true when the latest run left no host on under a minimum of 0', () => { + expect(scaledToZero([point({}), point({ PoweredOn: 0, MinVMs: 0 }), point({ Runs: 0, PoweredOn: null, MinVMs: null })])).toBe(true); + }); + + it('is false while a host is on, the minimum is above 0, or nothing ran', () => { + expect(scaledToZero([point({ PoweredOn: 0, MinVMs: 0 }), point({ PoweredOn: 1, MinVMs: 0 })])).toBe(false); + expect(scaledToZero([point({ PoweredOn: 0, MinVMs: 1 })])).toBe(false); + expect(scaledToZero([point({ Runs: 0, PoweredOn: 0, MinVMs: 0 })])).toBe(false); + expect(scaledToZero([])).toBe(false); + }); +}); + +describe('CapacityCard', () => { + const metrics = (series: UtilizationPoint[]) => ({ Available: true as const, Hours: 24 as const, BucketMinutes: 60, Series: series }); + + it('charts the users who waited and says when the pool scaled to zero', () => { + render( + undefined} + data={metrics([point({ Waited: 2 }), point({ PoweredOn: 0, InUse: 0, Serviceable: 0, MinVMs: 0 })])} + busy={false} + error={null} + />, + ); + expect(screen.getByText('Users who waited for a host')).toBeInTheDocument(); + expect(screen.getByText(/Scaled to zero: no host was powered on after the latest scaling run/)).toBeInTheDocument(); + }); + + it('leaves the waits out when nobody waited', () => { + render( undefined} data={metrics([point({})])} busy={false} error={null} />); + expect(screen.queryByText('Users who waited for a host')).not.toBeInTheDocument(); + expect(screen.queryByText(/Scaled to zero/)).not.toBeInTheDocument(); + }); }); describe('CheckoutHealthCard', () => { @@ -53,6 +96,29 @@ describe('CheckoutHealthCard', () => { expect(screen.getByText('No checkouts yet')).toBeInTheDocument(); expect(screen.getByText('No host was started')).toBeInTheDocument(); expect(screen.queryByText('Failed checkouts')).not.toBeInTheDocument(); + // A broker that predates start on demand reports no waits, so the figure is left out. + expect(screen.queryByText('Waited for a host')).not.toBeInTheDocument(); + }); + + it('shows how long users waited while a host started', () => { + render( + , + ); + expect(screen.getByText('Waited for a host')).toBeInTheDocument(); + expect(screen.getByText('4')).toBeInTheDocument(); + expect(screen.getByText('Median 1 min · 95% within 3 min · 1 without a host yet · 1 waiting now')).toBeInTheDocument(); + }); + + it('describes waits', () => { + const none = { ...STATS, Starting: 0, Waits: 0, WaitsServed: 0, WaitingNow: 0, WaitP50Seconds: null, WaitP95Seconds: null }; + expect(describeWaits(none)).toBe('Nobody waited for a host to start'); + expect(describeWaits({ ...none, WaitingNow: 2 })).toBe('2 waiting now'); + expect(describeWaits({ ...none, Waits: 2, WaitsServed: 2, WaitP50Seconds: 45, WaitP95Seconds: 90 })).toBe( + 'Median 45 s · 95% within 2 min', + ); }); it('formats times', () => { diff --git a/front_end/web/src/components/scaling/ScalingPreviewCard.tsx b/front_end/web/src/components/scaling/ScalingPreviewCard.tsx index 3680f77..942d1f4 100644 --- a/front_end/web/src/components/scaling/ScalingPreviewCard.tsx +++ b/front_end/web/src/components/scaling/ScalingPreviewCard.tsx @@ -63,11 +63,24 @@ export function ScalingPreviewCard({ preview, title, busy = false, error = null, + {preview.Counts.Waiting ? ( +

+ + + {preview.Counts.Waiting.toLocaleString()} {preview.Counts.Waiting === 1 ? 'user is' : 'users are'} waiting + for a host to start, counted as in use. + +

+ ) : null} + {preview.Phase.MinVMs !== null ? (

Using {preview.Phase.Name ?? 'the default rule'}: keep{' '} {preview.Phase.MinVMs}–{preview.Phase.MaxVMs} hosts, scale up at {preview.Phase.ScaleUpRatio}% and down at{' '} {preview.Phase.ScaleDownRatio}%. + {preview.Phase.MinVMs === 0 && preview.StartOnDemandEnabled ? ( + <> With a minimum of 0, the last idle host can stop; start on demand starts one for the next user. + ) : null} {preview.Phase.MaintenanceSurge ? ( <> {' '} diff --git a/front_end/web/src/components/scaling/StartOnDemandCard.tsx b/front_end/web/src/components/scaling/StartOnDemandCard.tsx new file mode 100644 index 0000000..6d57cd5 --- /dev/null +++ b/front_end/web/src/components/scaling/StartOnDemandCard.tsx @@ -0,0 +1,229 @@ +import { useEffect, useState } from 'react'; + +import { Badge } from '../ui/Badge'; +import { Button } from '../ui/Button'; +import { Notice } from '../ui/Feedback'; +import { Switch, TextField } from '../ui/Field'; +import { GlassCard } from '../ui/GlassCard'; +import { useToast } from '../ui/Toast'; +import { useSetStartOnDemand } from '../../hooks/useBroker'; +import { useConfirm } from '../../hooks/useConfirm'; +import { errorMessage } from '../../lib/api'; +import type { AvdHostScripts, ScalingPolicy } from '../../types/broker'; + +const DEFAULT_MAX_PENDING = 2; +const MAX_PENDING_LIMIT = 20; +const HOSTNAMES_SHOWN = 5; + +function plural(count: number, one: string, many: string) { + return `${count.toLocaleString()} ${count === 1 ? one : many}`; +} + +function validMaxPending(text: string) { + return /^\d{1,2}$/.test(text.trim()) && Number(text) >= 1 && Number(text) <= MAX_PENDING_LIMIT; +} + +function listHostnames(hostnames: string[], total: number) { + const shown = hostnames.slice(0, HOSTNAMES_SHOWN).join(', '); + const more = total - Math.min(total, HOSTNAMES_SHOWN); + return more > 0 ? `${shown} and ${more.toLocaleString()} more` : shown; +} + +/** Which broker script the AVD session hosts run, since only a current one waits for a host. */ +function AvdHostScriptSummary({ scripts }: { scripts: AvdHostScripts }) { + return ( +

+

+ AVD host scripts +

+

+ The broker script on each AVD session host that asked for a Linux host in the last seven days. Version{' '} + {scripts.CurrentVersion} or later waits while a host starts; an older script turns the user away. +

+ + {scripts.Seen === 0 ? ( +

No AVD session host has asked for a Linux host in the last seven days.

+ ) : ( + <> + {scripts.Outdated > 0 ? ( + + {plural(scripts.Outdated, 'AVD host runs', 'AVD hosts run')} a script that cannot wait for a host to start + {scripts.OutdatedHostnames.length ? `: ${listHostnames(scripts.OutdatedHostnames, scripts.Outdated)}` : ''}. + Their users are turned away when no host is free. Update them with{' '} + deploy/Update-AvdHostBrokerScript.ps1. + + ) : null} +
    + {scripts.Versions.map((version) => ( +
  • + + Version {version.ClientVersion} + + + {plural(version.AvdHosts, 'host', 'hosts')} + {version.Current ? ( + Current + ) : ( + Older + )} + +
    • + ))} + {scripts.Outdated > 0 ? ( +
  • + No version reported + + {plural(scripts.Outdated, 'host', 'hosts')} + Cannot wait + +
    • + ) : null} +
+ + )} +
+ ); +} + +function StartOnDemandEditor({ policy }: { policy: ScalingPolicy }) { + const setStartOnDemand = useSetStartOnDemand(); + const { showToast } = useToast(); + const { confirm, dialog } = useConfirm(); + const enabled = policy.StartOnDemandEnabled === true; + const savedMaxPending = policy.MaxPendingStarts ?? DEFAULT_MAX_PENDING; + const zeroMinimums = policy.ZeroMinimumCount ?? 0; + const [on, setOn] = useState(enabled); + const [maxPending, setMaxPending] = useState(String(savedMaxPending)); + + useEffect(() => setOn(enabled), [enabled]); + useEffect(() => setMaxPending(String(savedMaxPending)), [savedMaxPending]); + + const valid = validMaxPending(maxPending); + const changed = on !== enabled || (valid && Number(maxPending) !== savedMaxPending); + + async function save() { + try { + const result = await setStartOnDemand.mutateAsync({ + startondemandenabled: on, + maxpendingstarts: Number(maxPending), + }); + showToast(result.message, 'success'); + } catch (cause) { + showToast(errorMessage(cause, 'Unable to change start on demand.'), 'danger'); + } + } + + function submit() { + if (!valid || !changed) { + return; + } + if (enabled && !on) { + confirm({ + title: 'Turn off start on demand?', + body: + zeroMinimums > 0 + ? `${plural(zeroMinimums, 'rule or window keeps', 'rules and windows keep')} a minimum of 0 hosts. While start on demand is off the scaler keeps at least 1 host on for them, and a user who finds no free host is turned away instead of waiting.` + : 'A user who finds no free host is turned away instead of waiting while a host starts.', + confirmLabel: 'Turn it off', + variant: 'warning', + onConfirm: save, + }); + return; + } + void save(); + } + + return ( + <> +
{ + event.preventDefault(); + submit(); + }} + > + + setMaxPending(event.target.value)} + /> + + + {dialog} + + ); +} + +/** Whether a user who finds no free host waits while one starts, and what the AVD hosts run. */ +export function StartOnDemandCard({ policy, admin }: { policy: ScalingPolicy; admin: boolean }) { + const supported = policy.StartOnDemandEnabled !== null && policy.StartOnDemandEnabled !== undefined; + const enabled = policy.StartOnDemandEnabled === true; + const zeroMinimums = policy.ZeroMinimumCount ?? 0; + const scripts = policy.AvdHostScripts; + + return ( + +
+
+

Start on demand

+

+ When no host is free, the broker starts a stopped one and the user waits for it instead of being turned + away. It also lets a rule or window keep a minimum of 0 hosts, so idle hosts can stop. +

+
+ {supported ? ( + enabled ? ( + On + ) : ( + Off + ) + ) : null} +
+ + {!supported ? ( +

+ This broker does not support start on demand yet. Upgrade the broker API and run the database migration to + use it. +

+ ) : ( + <> + {!enabled && zeroMinimums > 0 ? ( + + {plural(zeroMinimums, 'rule or window keeps', 'rules and windows keep')} a minimum of 0 hosts, which needs + start on demand. While it is off the scaler keeps at least 1 host on for them. + + ) : null} +
+ {admin ? ( + + ) : ( +

+ {enabled + ? `On. Up to ${plural(policy.MaxPendingStarts ?? DEFAULT_MAX_PENDING, 'host starts', 'hosts start')} at once for waiting users.` + : 'Off. A user who finds no free host is turned away.'} +

+ )} + {scripts ? : null} +
+ + )} + + ); +} diff --git a/front_end/web/src/components/ui/Badge.test.tsx b/front_end/web/src/components/ui/Badge.test.tsx index ccca4a8..e0f9565 100644 --- a/front_end/web/src/components/ui/Badge.test.tsx +++ b/front_end/web/src/components/ui/Badge.test.tsx @@ -49,6 +49,7 @@ describe('status badges', () => { ['Scale Up', 'Scale up'], ['Scale Down', 'Scale down'], ['No Action', 'No action'], + ['Start On Demand', 'Start on demand'], ])('renders the scaling action %s as %s', (value, label) => { render(); expect(screen.getByText(label)).toBeInTheDocument(); diff --git a/front_end/web/src/components/ui/Badge.tsx b/front_end/web/src/components/ui/Badge.tsx index 4427a1d..1cdd94e 100644 --- a/front_end/web/src/components/ui/Badge.tsx +++ b/front_end/web/src/components/ui/Badge.tsx @@ -76,6 +76,7 @@ const SCALING_ACTION: Record = { 'Scale Up': { tone: 'ok', icon: 'arrow-up', label: 'Scale up' }, 'Scale Down': { tone: 'warn', icon: 'arrow-down', label: 'Scale down' }, 'No Action': { tone: 'neutral', icon: 'dash-circle', label: 'No action' }, + 'Start On Demand': { tone: 'info', icon: 'power', label: 'Start on demand' }, }; export const VmStatusBadge = ({ value }: { value: string | null | undefined }) => diff --git a/front_end/web/src/hooks/useBroker.ts b/front_end/web/src/hooks/useBroker.ts index c54c67f..847ae77 100644 --- a/front_end/web/src/hooks/useBroker.ts +++ b/front_end/web/src/hooks/useBroker.ts @@ -23,6 +23,7 @@ import type { MaintenanceRunsPage, MessageResult, Paged, + PolicyUpdateResult, PowerActionResult, PowerSyncResult, ProfileResetResult, @@ -34,6 +35,7 @@ import type { ScheduleInput, SessionsPage, SignOutResult, + StartOnDemandInput, TimeZoneOption, UserSearchResult, UtilizationHours, @@ -490,6 +492,17 @@ export function useScalingPolicy() { }); } +/** + * Whether a rule may keep a minimum of 0 VMs, which needs start on demand. Undefined until the + * policy has loaded, so a form does not flag a 0 it cannot yet judge. + */ +export function useZeroMinimumAllowed(): boolean | undefined { + const { data, isError } = useScalingPolicy(); + if (isError) return false; + if (!data) return undefined; + return data.Available !== false && data.StartOnDemandEnabled === true; +} + /** What the next scaling run would do now. Refreshed every minute while shown. */ export function useScalingPreview() { return useQuery({ @@ -518,7 +531,16 @@ export function useTimeZones() { export function useSetPolicyTimeZone() { const invalidate = useRuleInvalidation(); return useMutation({ - mutationFn: (timezone: string) => apiPost<{ message: string; TimeZone: string }>('/scaling/policy', { timezone }), + mutationFn: (timezone: string) => apiPost('/scaling/policy', { timezone }), + onSuccess: invalidate, + }); +} + +/** Turn start on demand on or off, and set how many hosts may start at once for waiting users. */ +export function useSetStartOnDemand() { + const invalidate = useRuleInvalidation(); + return useMutation({ + mutationFn: (input: StartOnDemandInput) => apiPost('/scaling/policy', input), onSuccess: invalidate, }); } diff --git a/front_end/web/src/pages/scaling/CreateRule.tsx b/front_end/web/src/pages/scaling/CreateRule.tsx index 9eac21e..d61b32d 100644 --- a/front_end/web/src/pages/scaling/CreateRule.tsx +++ b/front_end/web/src/pages/scaling/CreateRule.tsx @@ -5,7 +5,7 @@ import { ButtonLink } from '../../components/ui/Button'; import { PageHeader } from '../../components/ui/Feedback'; import { useToast } from '../../components/ui/Toast'; import { useConfirm } from '../../hooks/useConfirm'; -import { useCreateScalingRule } from '../../hooks/useBroker'; +import { useCreateScalingRule, useZeroMinimumAllowed } from '../../hooks/useBroker'; import { errorMessage } from '../../lib/api'; import { EMPTY_RULE, RuleForm } from './RuleForm'; @@ -13,6 +13,7 @@ export function CreateRule() { const navigate = useNavigate(); const { showToast } = useToast(); const createRule = useCreateScalingRule(); + const zeroMinimumAllowed = useZeroMinimumAllowed(); const { confirm, dialog } = useConfirm(); const [form, setForm] = useState(EMPTY_RULE); @@ -60,6 +61,7 @@ export function CreateRule() { submitLabel="Create rule" busy={createRule.isPending} cancelTo="/scaling" + zeroMinimumAllowed={zeroMinimumAllowed} /> {dialog} diff --git a/front_end/web/src/pages/scaling/RuleForm.tsx b/front_end/web/src/pages/scaling/RuleForm.tsx index 62fb3a3..b0e63b9 100644 --- a/front_end/web/src/pages/scaling/RuleForm.tsx +++ b/front_end/web/src/pages/scaling/RuleForm.tsx @@ -26,7 +26,7 @@ const FIELDS: FieldSpec[] = [ { key: 'minvms', label: 'Minimum VMs', - help: 'Keep at least this many VMs powered on for baseline capacity. Must be at least 1.', + help: 'Keep at least this many VMs powered on for baseline capacity. Must be at least 1, or 0 while start on demand is on.', }, { key: 'maxvms', @@ -66,6 +66,8 @@ export interface RuleFormProps { submitLabel: string; busy?: boolean; cancelTo: string; + /** Start on demand is on, so the minimum may be 0. Undefined while the policy loads. */ + zeroMinimumAllowed?: boolean; } type RuleValues = Omit & { stopmode: ScalingRuleInput['stopmode'] | '' }; @@ -75,10 +77,21 @@ export interface RuleFieldsProps { onChange: (value: T) => void; /** Offer "the default rule's" as the stop mode, for a schedule window. */ inheritStopMode?: boolean; + /** Start on demand is on, so the minimum may be 0. Undefined while the policy loads. */ + zeroMinimumAllowed?: boolean; } +const ZERO_MINIMUM_HELP = + 'Keep at least this many VMs powered on. 0 lets idle hosts stop until someone needs one: start on demand then starts a host for the first user, who waits a minute or two.'; + /** The scaling values and stop mode, shared by the default rule and schedule windows. */ -export function RuleFields({ value, onChange, inheritStopMode = false }: RuleFieldsProps) { +export function RuleFields({ + value, + onChange, + inheritStopMode = false, + zeroMinimumAllowed, +}: RuleFieldsProps) { + const zeroMinimum = String(value.minvms).trim() !== '' && Number(value.minvms) === 0; return ( <>
@@ -86,9 +99,14 @@ export function RuleFields({ value, onChange, inheritStopM ({ value, onChange, inheritStopM />
+ {zeroMinimum && zeroMinimumAllowed ? ( + + With a minimum of 0, no host runs while nobody needs one. The first user to connect waits while a host + starts, typically one to three minutes, and their AVD session connects when it is ready. + + ) : null} + {value.stopmode === 'Deallocate' ? ( Deallocate stops compute billing, but disks and IPs can still bill. Starts take longer, @@ -134,6 +159,7 @@ export function RuleForm({ submitLabel, busy = false, cancelTo, + zeroMinimumAllowed, }: RuleFormProps) { return ( @@ -144,7 +170,7 @@ export function RuleForm({ onSubmit(); }} > - +