kvm: dedicated live migration network with TLS, parallel streams and cluster CPU baseline

agent/conf/agent.properties

@@ -66,6 +66,41 @@ zone=default
 # If this is commented, the value of the private NIC device will be used.
 #guest.network.device=
 
+# Policy for encrypting the live-migration data stream using QEMU-native TLS.
+# "Disabled" (default, plaintext) or "Required" (use TLS, and fail the migration if
+# the host libvirt is older than 3.2.0, rather than silently sending plaintext).
+# "Required" needs a migration x509 trust set up on every participating host first:
+# a CA plus server and client certs under migrate_tls_x509_cert_dir in qemu.conf
+# (default /etc/pki/qemu), with the cert SAN covering the host's migration address,
+# then restart libvirtd. Without that, libvirt rejects the TLS migration.
+#migrate.encryption.policy=Disabled
+
+# Use multiple parallel TCP streams (multifd) for live migration to fill fast NICs.
+#migrate.parallel.enabled=false
+
+# Number of parallel connections (multifd channels) for live migration, used only
+# when migrate.parallel.enabled is true. 0 lets libvirt/QEMU pick its default.
+# To use more than one physical NIC for migration bandwidth, bond the NICs under the
+# migration bridge; libvirt sends all channels to a single destination address and
+# cannot bind them to separate NICs itself. For the channels to spread across bond
+# members the bond must hash on the L4 ports (802.3ad or balance-xor with
+# xmit_hash_policy=layer3+4); the default layer2 hash pins all channels to one member
+# because they share a MAC and IP pair, giving no spread. The spread is statistical,
+# not guaranteed.
+#migrate.parallel.connections=0
+
+# Allow migrating a VM whose disk cache mode libvirt considers unsafe (e.g. writeback).
+# Only enable on coherent shared storage such as Ceph RBD.
+#migrate.allow.unsafe=false
+
+# Run a virsh cpu-compare precheck against the destination before migrating, and abort
+# early with a clear error if the destination CPU cannot run the guest.
+#migrate.cpu.precheck.enabled=false
+
+# Memory-compression method for non-parallel migration, xbzrle or mt. Blank leaves libvirt's
+# default method. Ignored when migrate.parallel.enabled is true.
+#migrate.compression.method=
+
 # Local storage path. Multiple values can be entered and separated by commas.
 #local.storage.path=/var/lib/libvirt/images/
 

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -146,6 +146,63 @@ public class AgentProperties{
      */
     public static final Property<String> QEMU_SOCKETS_PATH = new Property<>("qemu.sockets.path", "/var/lib/libvirt/qemu");
 
+    /**
+     * Policy for encrypting the live-migration data stream (guest RAM, and disk contents
+     * during storage migration) using QEMU-native TLS (VIR_MIGRATE_TLS). Values: "Disabled"
+     * (default, plaintext TCP) or "Required" (encrypt, and fail the migration if the host has no
+     * TLS migration environment). Required needs migrate_tls_x509_cert_dir / default_tls_x509_cert_dir
+     * configured in qemu.conf on every host.
+     * Data type: String.<br>
+     * Default value: "Disabled".
+     */
+    public static final Property<String> MIGRATE_ENCRYPTION_POLICY = new Property<>("migrate.encryption.policy", "Disabled");
+
+    /**
+     * Use multiple parallel TCP streams (multifd) for live migration to saturate fast NICs
+     * (25/40/100GbE). Single-stream migration cannot fill such links. Requires libvirt 5.2.0+.
+     * Data type: Boolean.<br>
+     * Default value: false.
+     */
+    public static final Property<Boolean> MIGRATE_PARALLEL_ENABLED = new Property<>("migrate.parallel.enabled", false);
+
+    /**
+     * Number of parallel connections (multifd channels) to use for live migration when migrate.parallel.enabled
+     * is set. 0 lets libvirt/QEMU choose its default. To spread migration across more than one physical NIC,
+     * bond the NICs (for example LACP) under the migration bridge; libvirt sends all channels to a single
+     * destination address, so it cannot bind them to separate NICs itself.<br>
+     * Data type: Integer.<br>
+     * Default value: 0.
+     */
+    public static final Property<Integer> MIGRATE_PARALLEL_CONNECTIONS = new Property<>("migrate.parallel.connections", 0);
+
+    /**
+     * Allow live migration of a VM whose disk cache mode libvirt considers unsafe (anything
+     * other than none/directsync), by setting VIR_MIGRATE_UNSAFE. Safe only on coherent shared
+     * storage such as Ceph RBD, where writeback caching does not risk data loss on migration.
+     * Data type: Boolean.<br>
+     * Default value: false.
+     */
+    public static final Property<Boolean> MIGRATE_ALLOW_UNSAFE = new Property<>("migrate.allow.unsafe", false);
+
+    /**
+     * Run a CPU-compatibility precheck (virsh cpu-compare against the destination host) before
+     * a live migration, so an incompatible destination fails fast with a clear message instead of a
+     * cryptic mid-migration libvirt error. Best-effort and fail-open: if the check cannot run it does
+     * not block the migration. Requires the source host's virsh to reach the destination libvirt.
+     * Data type: Boolean.<br>
+     * Default value: false.
+     */
+    public static final Property<Boolean> MIGRATE_CPU_PRECHECK_ENABLED = new Property<>("migrate.cpu.precheck.enabled", false);
+
+    /**
+     * Migration memory-compression method, "xbzrle" or "mt". Empty (default) leaves libvirt's
+     * own default method in effect. Only takes effect when compression is enabled (libvirt &gt;= 1.0.3,
+     * which sets VIR_MIGRATE_COMPRESSED) and is skipped under multifd.
+     * Data type: String.<br>
+     * Default value: "" (empty).
+     */
+    public static final Property<String> MIGRATE_COMPRESSION_METHOD = new Property<>("migrate.compression.method", "");
+
     /**
      * MANDATORY: The UUID for the local storage pool.<br>
      * This property allows multiple values to be entered in a single String. The different values must be separated by commas.<br>

api/src/main/java/com/cloud/host/Host.java

@@ -66,6 +66,7 @@ public static String[] toStrings(Host.Type... types) {
     String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
     String HOST_SSH_PORT = "host.ssh.port";
     String HOST_CDROM_MAX_COUNT = "host.cdrom.max.count";
+    String HOST_MIGRATION_IP = "host.migration.ip";
     String GUEST_OS_CATEGORY_ID = "guest.os.category.id";
     String GUEST_OS_RULE = "guest.os.rule";
 

api/src/main/java/com/cloud/network/Networks.java

@@ -306,7 +306,7 @@ public static URI encodeStringIntoBroadcastUri(String candidate, BroadcastDomain
      * Different types of network traffic in the data center.
      */
     public enum TrafficType {
-        None, Public, Guest, Storage, Management, Control, Vpn;
+        None, Public, Guest, Storage, Management, Control, Vpn, Migration;
 
         public static boolean isSystemNetwork(TrafficType trafficType) {
             if (Storage.equals(trafficType) || Management.equals(trafficType) || Control.equals(trafficType)) {
@@ -322,6 +322,8 @@ public static TrafficType getTrafficType(String type) {
                 return Guest;
             } else if ("Storage".equals(type)) {
                 return Storage;
+            } else if ("Migration".equals(type)) {
+                return Migration;
             } else if ("Management".equals(type)) {
                 return Management;
             } else if ("Control".equals(type)) {

api/src/main/java/com/cloud/network/PhysicalNetworkSetupInfo.java

@@ -28,6 +28,7 @@ public class PhysicalNetworkSetupInfo {
     String publicNetworkName;
     String guestNetworkName;
     String storageNetworkName;
+    String migrationNetworkName;
     String mgmtVlan;
 
     public PhysicalNetworkSetupInfo() {
@@ -49,6 +50,14 @@ public String getStorageNetworkName() {
         return storageNetworkName;
     }
 
+    public String getMigrationNetworkName() {
+        return migrationNetworkName;
+    }
+
+    public void setMigrationNetworkName(String migrationNetworkName) {
+        this.migrationNetworkName = migrationNetworkName;
+    }
+
     public void setPrivateNetworkName(String privateNetworkName) {
         this.privateNetworkName = privateNetworkName;
     }

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -118,6 +118,9 @@ public interface VmDetailConstants {
     // CPU mode and model, ADMIN only
     String GUEST_CPU_MODE = "guest.cpu.mode";
     String GUEST_CPU_MODEL = "guest.cpu.model";
+    // Fallback policy for a custom CPU model ("allow" or "forbid"). "forbid" refuses a host that
+    // cannot provide the exact model instead of silently degrading it; set by the cluster CPU baseline.
+    String GUEST_CPU_MODEL_FALLBACK = "guest.cpu.model.fallback";
 
     // Lease related
     String INSTANCE_LEASE_EXPIRY_DATE = "leaseexpirydate";

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -307,6 +307,10 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "True if the host has capability to support UEFI boot")
     private Boolean uefiCapability;
 
+    @SerializedName("migrationip")
+    @Param(description = "the IP address the host uses for live migration traffic, set when a dedicated migration network is configured on this host", since = "24.0.0")
+    private String migrationIp;
+
     @SerializedName(ApiConstants.ENCRYPTION_SUPPORTED)
     @Param(description = "True if the host supports encryption", since = "4.18")
     private Boolean encryptionSupported;
@@ -896,6 +900,14 @@ public void setUefiCapability(Boolean hostCapability) {
         this.uefiCapability = hostCapability;
     }
 
+    public void setMigrationIp(String migrationIp) {
+        this.migrationIp = migrationIp;
+    }
+
+    public String getMigrationIp() {
+        return migrationIp;
+    }
+
     public void setEncryptionSupported(Boolean encryptionSupported) {
         this.encryptionSupported = encryptionSupported;
     }

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -110,7 +110,7 @@
  */
 public interface QueryService {
 
-    List<String> RootAdminOnlyVmSettings = Arrays.asList(VmDetailConstants.GUEST_CPU_MODE, VmDetailConstants.GUEST_CPU_MODEL);
+    List<String> RootAdminOnlyVmSettings = Arrays.asList(VmDetailConstants.GUEST_CPU_MODE, VmDetailConstants.GUEST_CPU_MODEL, VmDetailConstants.GUEST_CPU_MODEL_FALLBACK);
 
     // Config keys
     ConfigKey<Boolean> AllowUserViewDestroyedVM = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.destroyed.vm", "false",

core/src/main/java/com/cloud/agent/api/BaselineCpuCommand.java

@@ -0,0 +1,47 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import java.util.List;
+
+/**
+ * Runs {@code virsh cpu-baseline} over the given per-host {@code <cpu>} elements and returns the
+ * most feature-rich CPU compatible with all of them: the common-denominator cluster baseline.
+ */
+public class BaselineCpuCommand extends Command {
+
+    private List<String> hostCpuXmls;
+
+    protected BaselineCpuCommand() {
+    }
+
+    public BaselineCpuCommand(List<String> hostCpuXmls) {
+        this.hostCpuXmls = hostCpuXmls;
+    }
+
+    public List<String> getHostCpuXmls() {
+        return hostCpuXmls;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/CheckCpuCompatibilityCommand.java

@@ -0,0 +1,50 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+/**
+ * Checks on the agent, via {@code virsh cpu-compare}, that the host CPU can run the given {@code <cpu>} model.
+ */
+public class CheckCpuCompatibilityCommand extends Command {
+
+    private String cpuXml;
+    private String vmName;
+
+    protected CheckCpuCompatibilityCommand() {
+    }
+
+    public CheckCpuCompatibilityCommand(String vmName, String cpuXml) {
+        this.vmName = vmName;
+        this.cpuXml = cpuXml;
+    }
+
+    public String getCpuXml() {
+        return cpuXml;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/CheckNetworkAnswer.java

@@ -22,6 +22,8 @@
 public class CheckNetworkAnswer extends Answer {
     // indicate if agent reconnect is needed after setupNetworkNames command
     private boolean _reconnect;
+    // the local IP the host resolved on its dedicated migration network, if one is configured
+    private String migrationIp;
 
     public CheckNetworkAnswer() {
     }
@@ -39,4 +41,12 @@ public boolean needReconnect() {
         return _reconnect;
     }
 
+    public String getMigrationIp() {
+        return migrationIp;
+    }
+
+    public void setMigrationIp(String migrationIp) {
+        this.migrationIp = migrationIp;
+    }
+
 }

core/src/main/java/com/cloud/agent/api/GetHostCpuModelCommand.java

@@ -0,0 +1,35 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+/**
+ * Asks the agent for this host's {@code <cpu>} capabilities element, so the management server can
+ * compute a cluster CPU baseline that every host supports.
+ */
+public class GetHostCpuModelCommand extends Command {
+
+    public GetHostCpuModelCommand() {
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -31,10 +31,12 @@
 public class MigrateCommand extends Command {
     private String vmName;
     private String destinationIp;
+    private String migrateIp;
     private Map<String, MigrateDiskInfo> migrateStorage;
     private boolean migrateStorageManaged;
     private boolean migrateNonSharedInc;
     private boolean autoConvergence;
+    private String migrationEncryptionPolicy;
     private String hostGuid;
     private boolean windows;
     private VirtualMachineTO virtualMachine;
@@ -98,6 +100,14 @@ public void setMigrateNonSharedInc(boolean migrateNonSharedInc) {
         this.migrateNonSharedInc = migrateNonSharedInc;
     }
 
+    public String getMigrationEncryptionPolicy() {
+        return migrationEncryptionPolicy;
+    }
+
+    public void setMigrationEncryptionPolicy(String migrationEncryptionPolicy) {
+        this.migrationEncryptionPolicy = migrationEncryptionPolicy;
+    }
+
     public void setAutoConvergence(boolean autoConvergence) {
         this.autoConvergence = autoConvergence;
     }
@@ -118,6 +128,14 @@ public String getDestinationIp() {
         return destinationIp;
     }
 
+    public void setMigrateIp(String migrateIp) {
+        this.migrateIp = migrateIp;
+    }
+
+    public String getMigrateIp() {
+        return migrateIp;
+    }
+
     public String getVmName() {
         return vmName;
     }