PXB-3113 : Improve debug sync framework to allow PXB to pause and res…

storage/innobase/include/xb0xb.h

@@ -103,6 +103,11 @@ const std::string KEYRING_NOT_LOADED =
 @param[in]	name	sync point name */
 void debug_sync_point(const char *name);
 
+/** Pause xtrabackup thread and wait for resume.
+Thread can be resumed by deleting the sync_point filename
+@param[in]	name	sync point name */
+void debug_sync_thread(const char *name);
+
 extern char *xtrabackup_debug_sync;
 
 #endif

storage/innobase/xtrabackup/src/redo_log.cc

@@ -1207,6 +1207,7 @@ void Redo_Log_Data_Manager::copy_func() {
       xb::info() << ">> log scanned up to (" << reader.get_scanned_lsn() << ")";
 
       debug_sync_point("xtrabackup_copy_logfile_pause");
+      debug_sync_thread("xtrabackup_copy_logfile_pause");
 
       os_event_reset(event);
       os_event_wait_time_low(event, std::chrono::milliseconds{copy_interval},

storage/innobase/xtrabackup/src/xtrabackup.cc

@@ -364,6 +364,10 @@ it every INNOBASE_WAKE_INTERVAL'th step. */
 ulong innobase_active_counter = 0;
 
 char *xtrabackup_debug_sync = NULL;
+#ifdef UNIV_DEBUG
+char *xtrabackup_debug_sync_thread = NULL;
+static std::string debug_sync_file_content;
+#endif /* UNIV_DEBUG */
 static const char *dbug_setting = nullptr;
 
 bool xtrabackup_incremental_force_scan = false;
@@ -685,6 +689,9 @@ enum options_xtrabackup {
   OPT_INNODB_REDO_LOG_ENCRYPT,
   OPT_INNODB_UNDO_LOG_ENCRYPT,
   OPT_XTRA_DEBUG_SYNC,
+#ifdef UNIV_DEBUG
+  OPT_XTRA_DEBUG_SYNC_THREAD,
+#endif /* UNIV_DEBUG */
   OPT_XTRA_COMPACT,
   OPT_XTRA_REBUILD_INDEXES,
   OPT_XTRA_REBUILD_THREADS,
@@ -1559,6 +1566,18 @@ Disable with --skip-innodb-checksums.",
      "Debug sync point. This is only used by the xtrabackup test suite",
      (G_PTR *)&xtrabackup_debug_sync, (G_PTR *)&xtrabackup_debug_sync, 0,
      GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+#ifdef UNIV_DEBUG
+    {"debug-sync-thread", OPT_XTRA_DEBUG_SYNC_THREAD,
+     "Thread sleeps at sync point and creates a filename with the "
+     "sync_point_name_<thread_number>. There can be multiple threads"
+     "Delete the file to resume the thread. This is only used by the "
+     "xtrabackup test suite",
+     (G_PTR *)&xtrabackup_debug_sync_thread,
+     (G_PTR *)&xtrabackup_debug_sync_thread, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
+     0, 0, 0},
+#endif /* UNIV_DEBUG */
+
 #endif
 
 #ifndef NDEBUG
@@ -1658,6 +1677,37 @@ static void sigcont_handler(int sig);
 static void sigcont_handler(int sig __attribute__((unused))) {
   debug_sync_resumed = 1;
 }
+
+#ifdef UNIV_DEBUG
+static void sigusr1_handler(int sig __attribute__((unused))) {
+  xb::info() << "SIGUSR1 received. Reading debug_sync point from "
+                "xb_debug_sync_thread file in backup directory";
+
+  std::string debug_sync_file =
+      std::string(xtrabackup_target_dir) + "/xb_debug_sync_thread";
+  std::ifstream ifs(debug_sync_file);
+
+  if (ifs.fail()) {
+    xb::warn() << "SIGUSR1 signal sent but the xb_debug_sync_thread file with"
+                  "a debug sync point name in it is not present at "
+               << xtrabackup_target_dir;
+    return;
+  }
+
+  debug_sync_file_content.assign((std::istreambuf_iterator<char>(ifs)),
+                                 (std::istreambuf_iterator<char>()));
+  debug_sync_file_content.erase(
+      std::remove(debug_sync_file_content.begin(),
+                  debug_sync_file_content.end(), '\n'),
+      debug_sync_file_content.cend());
+  *const_cast<const char **>(&xtrabackup_debug_sync_thread) =
+      debug_sync_file_content.c_str();
+
+  xb::info() << "DEBUG_SYNC_THREAD: Deleting  file" << debug_sync_file;
+  os_file_delete(0, debug_sync_file.c_str());
+}
+#endif /* UNIV_DEBUG */
+
 #endif
 
 void debug_sync_point(const char *name) {
@@ -1701,6 +1751,52 @@ void debug_sync_point(const char *name) {
 #endif
 }
 
+#ifdef UNIV_DEBUG
+void debug_sync_thread(const char *name) {
+#ifndef __WIN__
+  FILE *fp;
+  char pid_path[FN_REFLEN];
+
+  if (xtrabackup_debug_sync_thread == nullptr) {
+    return;
+  }
+
+  if (strcmp(xtrabackup_debug_sync_thread, name)) {
+    return;
+  }
+
+  auto thread_id = std::hash<std::thread::id>{}(std::this_thread::get_id());
+
+  snprintf(pid_path, sizeof(pid_path), "%s/%s_%lu", xtrabackup_target_dir, name,
+           thread_id);
+
+  fp = fopen(pid_path, "w");
+  if (fp == NULL) {
+    xb::error() << "cannot open " << pid_path;
+    exit(EXIT_FAILURE);
+  }
+
+  fclose(fp);
+
+  while (1) {
+    xb::info() << "DEBUG_SYNC_THREAD: sleeping 1sec.  Resume this thread by "
+                  "deleting file "
+               << pid_path;
+    sleep(1);
+    bool exists = os_file_exists(pid_path);
+    if (!exists) break;
+  }
+
+  xb::info() << "DEBUG_SYNC_THREAD: thread " << thread_id
+             << " resumed from sync point: " << name;
+
+  *const_cast<const char **>(&xtrabackup_debug_sync_thread) = nullptr;
+
+#endif
+}
+
+#endif /* UNIV_DEBUG */
+
 static const char *xb_client_default_groups[] = {"xtrabackup", "client", 0, 0,
                                                  0};
 
@@ -4221,6 +4317,8 @@ void xtrabackup_backup_func(void) {
     exit(EXIT_FAILURE);
   }
 
+  debug_sync_thread("before_file_copy");
+
   /* Create data copying threads */
   data_threads = (data_thread_ctxt_t *)ut::malloc_withkey(
       UT_NEW_THIS_FILE_PSI_KEY,
@@ -7605,6 +7703,9 @@ int main(int argc, char **argv) {
 
 #ifndef __WIN__
   signal(SIGCONT, sigcont_handler);
+#ifdef UNIV_DEBUG
+  signal(SIGUSR1, sigusr1_handler);
+#endif /* UNIV_DEBUG */
 #endif
 
   /* --backup */

storage/innobase/xtrabackup/test/inc/common.sh

@@ -28,6 +28,30 @@ function xtrabackup()
   fi
 }
 
+function xtrabackup_background() {
+  # Check if XB_ERROR_LOG is set
+  if [ -z "${XB_ERROR_LOG+x}" ]; then
+    echo "Error: XB_ERROR_LOG variable is not set. Please set XB_ERROR_LOG to the desired error log file path." >&2
+    return 1
+  fi
+
+  # Check if the number of arguments passed is correct
+  if [ $# -lt 1 ]; then
+    echo "Usage: xtrabackup_background [args...]" >&2
+    return 1
+  fi
+
+  local args="$@"  # Combine all arguments to form the xtrabackup command
+
+  # Execute xtrabackup in the background using the run_cmd_background function and store the PID in XB_PID
+  run_cmd_background $XB_BIN $XB_ARGS $args
+  XB_PID=$!
+
+  # Optionally, return the PID of the background process
+  echo $XB_PID
+}
+
+
 function rr_xtrabackup()
 {
   run_cmd rr $XB_BIN $XB_ARGS "$@"
@@ -135,6 +159,32 @@ function run_cmd_expect_failure()
   fi
 }
 
+function run_cmd_background() {
+  # Check if XB_ERROR_LOG is set
+  if [ -z "${XB_ERROR_LOG+x}" ]; then
+    echo "Error: XB_ERROR_LOG variable is not set. Please set XB_ERROR_LOG to the desired error log file path." >&2
+    return 1
+  fi
+
+  # Check if the number of arguments passed is correct
+  if [ $# -lt 1 ]; then
+    echo "Usage: run_cmd_background <command> [args...]" >&2
+    return 1
+  fi
+
+  local cmd="$@"  # Combine all arguments to form the command
+
+  # Execute the command in the background, piping stdout to tee to duplicate the output to both the error log file and stdout
+  # Dont use PIPE here as the PID will be pid of pipe
+  $cmd > >(tee -a "$XB_ERROR_LOG") 2>&1 &
+
+  # Capture the PID of the background process and store it in JOB_ID
+  CMD_PID=$!
+
+  # Optionally, return the PID of the command
+  echo $CMD_PID
+}
+
 function load_sakila()
 {
     vlog "Loading sakila"
@@ -1190,6 +1240,13 @@ function require_debug_sync()
     fi
 }
 
+function require_debug_sync_thread()
+{
+    if ! $XB_BIN --help 2>&1 | grep -q debug-sync-thread; then
+        skip_test "Requires --debug-sync-thread support"
+    fi
+}
+
 ##############################################################################
 # Execute a multi-row INSERT into a specified table.
 #
@@ -1462,5 +1519,53 @@ function wait_for_file_to_generated() {
   done
 }
 
+function wait_for_debug_sync_thread()
+{
+   # Check if the number of arguments passed is correct
+  if [ $# -lt 1 ]; then
+    echo "Usage: wait_for_debug_sync_thread <debug_sync_point_name>"
+    return 1
+  fi
+
+  local sync_point=$1
+  if [ -z "${XB_ERROR_LOG+x}" ]; then
+    echo "Error: XB_ERROR_LOG variable is not set. Please set XB_ERROR_LOG to the desired error log file path." >&2
+    return 1
+  fi
+
+  while ! egrep -q "DEBUG_SYNC_THREAD: sleeping 1sec.  Resume this thread by deleting file.*$sync_point.*" ${XB_ERROR_LOG} ; do
+    sleep 1
+    vlog "waiting for debug_sync_thread $sync_point in $XB_ERROR_LOG"
+  done
+}
+
+function resume_debug_sync_thread()
+{
+   # Check if the number of arguments passed is correct
+  if [ $# -lt 2 ]; then
+    echo "Usage: resume_debug_sync_thread <debug_sync_point_name> <backup_dir>"
+    return 1
+  fi
+
+  local sync_point=$1
+  local backup_dir=$2
+
+  if [ -z "${XB_ERROR_LOG+x}" ]; then
+    echo "Error: XB_ERROR_LOG variable is not set. Please set XB_ERROR_LOG to the desired error log file path." >&2
+    return 1
+  fi
+
+  echo "Resume from sync point: $sync_point"
+  rm  $backup_dir/"$sync_point"_*
+
+  echo "wait for resumed signal of $sync_point"
+
+  while ! egrep -q "DEBUG_SYNC_THREAD: thread .* resumed from sync point: $sync_point" ${XB_ERROR_LOG} ; do
+   sleep 1
+   vlog "waiting for debug_sync_thread point resume $sync_point in pxb $XB_ERROR_LOG"
+  done
+
+}
+
 # To avoid unbound variable error when no server have been started
 SRV_MYSQLD_IDS=