migration/ram: Add RAM round synchronization notifiers

find_dirty_block() directly invokes multifd's per-round synchronization
when the RAM scan wraps.  Other transports need the same boundary to
order page versions.

Add a return-valued notifier for RAM round completion and register
multifd's existing synchronization through it.  This keeps the current
behavior while allowing other transports to synchronize at the same
boundary.

Signed-off-by: Yanfei Xu <yanfei.xu@bytedance.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Tested-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Signed-off-by: Fabiano Rosas <farosas@suse.de>

migration/multifd-nocomp.c

@@ -19,6 +19,7 @@
 #include "multifd-colo.h"
 #include "options.h"
 #include "migration.h"
+#include "ram.h"
 #include "qapi/error.h"
 #include "qemu/cutils.h"
 #include "qemu/error-report.h"
@@ -27,6 +28,28 @@
 
 static MultiFDSendData *multifd_ram_send;
 
+static int multifd_ram_round_notify(NotifierWithReturn *n G_GNUC_UNUSED,
+                                    void *data, Error **errp)
+{
+    RAMRoundNotifyData *round_data = data;
+    QEMUFile *f = round_data->file;
+    int ret;
+
+    if (!multifd_ram_sync_per_round()) {
+        return 0;
+    }
+
+    ret = multifd_ram_flush_and_sync(f);
+    if (ret < 0) {
+        error_setg(errp, "multifd RAM round synchronization failed");
+    }
+    return ret;
+}
+
+static NotifierWithReturn multifd_ram_round_notifier = {
+    .notify = multifd_ram_round_notify,
+};
+
 void multifd_ram_payload_alloc(MultiFDPages_t *pages)
 {
     pages->offset = g_new0(ram_addr_t, multifd_ram_page_count());
@@ -471,6 +494,7 @@ static const MultiFDMethods multifd_nocomp_ops = {
 static void multifd_nocomp_register(void)
 {
     multifd_register_ops(MULTIFD_COMPRESSION_NONE, &multifd_nocomp_ops);
+    ram_round_add_notifier(&multifd_ram_round_notifier);
 }
 
 migration_init(multifd_nocomp_register);

migration/ram.c

@@ -470,6 +470,8 @@ typedef struct RAMState RAMState;
 static RAMState *ram_state;
 
 static NotifierWithReturnList precopy_notifier_list;
+static NotifierWithReturnList ram_round_notifier_list =
+    NOTIFIER_WITH_RETURN_LIST_INITIALIZER(ram_round_notifier_list);
 
 /* Whether postcopy has queued requests? */
 static bool postcopy_has_request(RAMState *rs)
@@ -500,6 +502,38 @@ int precopy_notify(PrecopyNotifyReason reason, Error **errp)
     return notifier_with_return_list_notify(&precopy_notifier_list, &pnd, errp);
 }
 
+void ram_round_add_notifier(NotifierWithReturn *n)
+{
+    notifier_with_return_list_add(&ram_round_notifier_list, n);
+}
+
+void ram_round_remove_notifier(NotifierWithReturn *n)
+{
+    notifier_with_return_remove(n);
+}
+
+static int ram_round_notify(QEMUFile *f)
+{
+    RAMRoundNotifyData data = {
+        .file = f,
+    };
+    Error *local_err = NULL;
+    int ret;
+
+    ret = notifier_with_return_list_notify(&ram_round_notifier_list, &data,
+                                           &local_err);
+    if (ret) {
+        if (local_err) {
+            error_report_err(local_err);
+        } else {
+            error_report("RAM round notifier failed: %d", ret);
+        }
+        return -1;
+    }
+
+    return 0;
+}
+
 uint64_t ram_bytes_remaining(void)
 {
     return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
@@ -1410,12 +1444,11 @@ static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
         pss->page = 0;
         pss->block = QLIST_NEXT_RCU(pss->block, next);
         if (!pss->block) {
-            if (multifd_ram_sync_per_round()) {
-                QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
-                int ret = multifd_ram_flush_and_sync(f);
-                if (ret < 0) {
-                    return ret;
-                }
+            QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
+            int ret = ram_round_notify(f);
+
+            if (ret < 0) {
+                return ret;
             }
 
             /* Hit the end of the list */

migration/ram.h

@@ -31,6 +31,7 @@
 
 #include "qapi/qapi-types-migration.h"
 #include "exec/cpu-common.h"
+#include "qemu/notify.h"
 #include "system/ram_addr.h"
 #include "io/channel.h"
 
@@ -95,6 +96,17 @@ void ram_handle_zero(void *host, uint64_t size);
 void ram_transferred_add(uint64_t bytes);
 void ram_release_page(const char *rbname, uint64_t offset);
 
+/*
+ * The RAM round notifier is invoked whenever the migration RAM core
+ * completes one full scan of guest memory.
+ */
+typedef struct RAMRoundNotifyData {
+    QEMUFile *file;
+} RAMRoundNotifyData;
+
+void ram_round_add_notifier(NotifierWithReturn *n);
+void ram_round_remove_notifier(NotifierWithReturn *n);
+
 int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr);
 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset);
 void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr);