diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml
index 6f4eed7f7c259..7e60322b49870 100644
--- a/distribution/src/assembly/cli.xml
+++ b/distribution/src/assembly/cli.xml
@@ -54,9 +54,11 @@
*data.*
*schema.*
*tsfile.*
+ *logical-backup*
**/*data.*
**/*schema.*
**/*tsfile.*
+ **/*logical-backup*
0755
diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml
index 5de247a084a09..c9824ef4aec11 100644
--- a/iotdb-client/cli/pom.xml
+++ b/iotdb-client/cli/pom.xml
@@ -124,6 +124,10 @@
commons-cli
commons-cli
+
+ com.google.code.gson
+ gson
+
org.jline
jline
@@ -204,6 +208,7 @@
org.apache.iotdb:iotdb-server
org/apache/iotdb/db/utils/*
+ org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.class
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 0329812f2d4d3..8aa2feccc907a 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -96,4 +96,60 @@ private CliMessages() {}
public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "Input time format {} is not supported, ";
public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n";
+ // Pipe logical backup tool
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 =
+ "Logical backup command failed: %s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 =
+ "Unknown logical backup command: %s";
+ public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 =
+ "--output is required for logical backup export";
+ public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 =
+ "Stream %s: %d records, %d committed event groups";
+ public static final String
+ LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B =
+ "Logical backup verified: %d streams, %d records, %d committed event groups";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 =
+ "Logical backup export target must not be inside source";
+ public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 =
+ "Logical backup exported from %s to %s";
+ public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B =
+ "Dry run completed; no data was written";
+ public static final String
+ LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D =
+ "Logical backup import completed: %d event groups, checkpoint %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA =
+ "Logical backup option --%s is required";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 =
+ "Logical backup password environment variable is not set";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 =
+ "Logical backup handshake failed: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
+ "Logical backup request type %d failed: %s";
+ public static final String
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
+ "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
+ public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
+ "pipe-logical-backup ";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
+ "Logical backup archive entry is unsafe: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
+ "Logical backup archive exceeds safety limit";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 =
+ "Logical backup export source must be a directory or manifest: %s";
+ public static final String
+ EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 =
+ "Specify exactly one of --password-stdin and --password-env for logical backup import";
+ public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E =
+ "No password was read from standard input";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D =
+ "Logical backup checkpoint does not match the source or target: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C =
+ "Logical backup checkpoint is invalid: %s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
+ "Unsupported logical backup export format: %s";
+
}
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index 61a14d94d5884..35346133f19e5 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -92,4 +92,60 @@ private CliMessages() {}
public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "不支持输入时间格式 {}，";
public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "请输入类似 yyyy-MM-dd\\ HH:mm:ss.SSS 或 yyyy-MM-dd'T'HH:mm:ss.SSS 的格式%n";
+ // Pipe 逻辑备份工具
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 =
+ "逻辑备份命令执行失败：%s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 =
+ "未知的逻辑备份命令：%s";
+ public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 =
+ "逻辑备份导出必须指定 --output";
+ public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 =
+ "流 %s：%d 条记录，%d 个已提交事件组";
+ public static final String
+ LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B =
+ "逻辑备份校验通过：%d 个流，%d 条记录，%d 个已提交事件组";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 =
+ "逻辑备份导出目标不能位于源目录内";
+ public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 =
+ "逻辑备份已从 %s 导出到 %s";
+ public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B =
+ "试运行完成，未写入数据";
+ public static final String
+ LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D =
+ "逻辑备份导入完成：%d 个事件组，checkpoint 为 %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA =
+ "逻辑备份选项 --%s 为必填项";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 =
+ "未设置逻辑备份密码环境变量";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 =
+ "逻辑备份握手失败：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
+ "逻辑备份请求类型 %d 执行失败：%s";
+ public static final String
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
+ "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
+ public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
+ "pipe-logical-backup ";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
+ "逻辑备份归档项路径不安全：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
+ "逻辑备份归档超过安全限制";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 =
+ "逻辑备份导出源必须是目录或 manifest：%s";
+ public static final String
+ EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 =
+ "逻辑备份导入必须且只能指定 --password-stdin 或 --password-env 中的一项";
+ public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E =
+ "未能从标准输入读取密码";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D =
+ "逻辑备份 checkpoint 与源备份或目标实例不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C =
+ "逻辑备份 checkpoint 无效：%s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
+ "不支持的逻辑备份导出格式：%s";
+
}
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
new file mode 100644
index 0000000000000..1d2315f17c04a
--- /dev/null
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -0,0 +1,914 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool.pipe;
+
+import org.apache.iotdb.cli.i18n.CliMessages;
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.thrift.TException;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */
+public final class PipeLogicalBackupTool {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final String COMMAND_INSPECT = "inspect";
+ private static final String COMMAND_VERIFY = "verify";
+ private static final String COMMAND_EXPORT = "export";
+ private static final String COMMAND_IMPORT = "import";
+ private static final String OPTION_INPUT = "input";
+ private static final String OPTION_OUTPUT = "output";
+ private static final String OPTION_RESUME = "resume";
+ private static final String OPTION_FORMAT = "format";
+ private static final String OPTION_DEEP = "deep";
+ // Compatibility aliases for the initial command-line prototype.
+ private static final String OPTION_SOURCE = "source";
+ private static final String OPTION_TARGET = "target";
+ private static final String OPTION_HOST = "host";
+ private static final String OPTION_PORT = "port";
+ private static final String OPTION_USER = "user";
+ private static final String OPTION_PASSWORD_STDIN = "password-stdin";
+ private static final String OPTION_PASSWORD_ENV = "password-env";
+ private static final String OPTION_CHECKPOINT = "checkpoint";
+ private static final String OPTION_DRY_RUN = "dry-run";
+ private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete";
+ private static final String OPTION_JSON = "json";
+ private static final int MAX_ARCHIVE_ENTRIES = 1_000_000;
+ private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024;
+ private static final String CHECKPOINT_FORMAT_NAME =
+ "iotdb-pipe-logical-backup-import-checkpoint";
+ private static final String CHECKPOINT_FORMAT_VERSION = "1.0";
+
+ private PipeLogicalBackupTool() {}
+
+ public static void main(final String[] args) {
+ int exitCode = 0;
+ try {
+ exitCode = run(args);
+ } catch (final Exception e) {
+ System.err.println(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0, e.getMessage()));
+ exitCode = 1;
+ }
+ if (exitCode != 0) {
+ System.exit(exitCode);
+ }
+ }
+
+ static int run(final String[] args) throws Exception {
+ if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) {
+ printUsage();
+ return 0;
+ }
+ final String command = args[0].toLowerCase(java.util.Locale.ROOT);
+ final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length));
+ switch (command) {
+ case COMMAND_INSPECT:
+ return inspect(line);
+ case COMMAND_VERIFY:
+ return verify(line);
+ case COMMAND_EXPORT:
+ return export(line);
+ case COMMAND_IMPORT:
+ return importBackup(line);
+ default:
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command));
+ }
+ }
+
+ private static CommandLine parse(final String command, final String[] args)
+ throws ParseException {
+ final Options options = new Options();
+ addOption(options, OPTION_INPUT, true, false);
+ addOption(options, OPTION_OUTPUT, true, false);
+ addOption(options, OPTION_RESUME, true, false);
+ addOption(options, OPTION_FORMAT, true, false);
+ addOption(options, OPTION_DEEP, false, false);
+ addOption(options, OPTION_SOURCE, true, false);
+ addOption(options, OPTION_TARGET, true, false);
+ addOption(options, OPTION_HOST, true, false);
+ addOption(options, OPTION_PORT, true, false);
+ addOption(options, OPTION_USER, true, false);
+ addOption(options, OPTION_PASSWORD_STDIN, false, false);
+ addOption(options, OPTION_PASSWORD_ENV, true, false);
+ addOption(options, OPTION_CHECKPOINT, true, false);
+ addOption(options, OPTION_DRY_RUN, false, false);
+ addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
+ addOption(options, OPTION_JSON, false, false);
+ final CommandLine line = new DefaultParser().parse(options, args);
+ final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
+ if (input == null || input.isBlank()) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
+ }
+ final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET);
+ if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) {
+ throw new ParseException(
+ CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1);
+ }
+ final String format = line.getOptionValue(OPTION_FORMAT, "binary");
+ if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format));
+ }
+ return line;
+ }
+
+ private static void addOption(
+ final Options options, final String name, final boolean hasArg, final boolean required) {
+ options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build());
+ }
+
+ private static int inspect(final CommandLine line) throws IOException {
+ final List streams = read(line);
+ if (line.hasOption(OPTION_JSON)) {
+ final List> output = new ArrayList<>();
+ for (final BackupStream stream : streams) {
+ final Map item = new HashMap<>();
+ item.put("manifest", stream.getManifest());
+ item.put("recordCount", stream.getRecords().size());
+ item.put("eventGroupCount", stream.getEventGroups().size());
+ output.add(item);
+ }
+ System.out.println(GSON.toJson(output));
+ return 0;
+ }
+ for (final BackupStream stream : streams) {
+ System.out.println(
+ String.format(
+ CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
+ stream.getManifest().streamId,
+ stream.getRecords().size(),
+ stream.getEventGroups().size()));
+ }
+ return 0;
+ }
+
+ private static int verify(final CommandLine line) throws IOException {
+ final List streams = read(line);
+ long recordCount = 0;
+ long eventCount = 0;
+ for (final BackupStream stream : streams) {
+ recordCount += stream.getRecords().size();
+ eventCount += stream.getEventGroups().size();
+ }
+ System.out.println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B,
+ streams.size(),
+ recordCount,
+ eventCount));
+ return 0;
+ }
+
+ private static int export(final CommandLine line) throws IOException {
+ final Path source = Path.of(input(line)).toAbsolutePath().normalize();
+ final Path target =
+ Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
+ final List streams = read(line);
+ final Path root;
+ if (Files.isDirectory(source)) {
+ root = source;
+ } else if (Files.isRegularFile(source)
+ && "manifest.json".equals(source.getFileName().toString())) {
+ root = source.getParent();
+ } else {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
+ source));
+ }
+ if (target.startsWith(root)) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994);
+ }
+ if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
+ throw new java.nio.file.FileAlreadyExistsException(target.toString());
+ }
+ final Path parent = target.toAbsolutePath().normalize().getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp");
+ boolean published = false;
+ try {
+ try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) {
+ for (final Path file : exportFiles(root, streams)) {
+ final String entryName = root.relativize(file).toString().replace('\\', '/');
+ zip.putNextEntry(new ZipEntry(entryName));
+ Files.copy(file, zip);
+ zip.closeEntry();
+ }
+ }
+ try (final java.nio.channels.FileChannel archiveChannel =
+ java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) {
+ archiveChannel.force(true);
+ }
+ try {
+ Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE);
+ } catch (final java.nio.file.AtomicMoveNotSupportedException e) {
+ Files.move(temporary, target);
+ }
+ published = true;
+ } finally {
+ if (!published) {
+ Files.deleteIfExists(temporary);
+ }
+ }
+ System.out.println(
+ String.format(
+ CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target));
+ return 0;
+ }
+
+ private static List exportFiles(final Path root, final List streams)
+ throws IOException {
+ final Set files = new LinkedHashSet<>();
+ for (final BackupStream stream : streams) {
+ final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize();
+ if (!manifestPath.startsWith(root)) {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
+ root));
+ }
+ files.add(manifestPath);
+ final Path streamDirectory = manifestPath.getParent();
+ for (final LogicalBackupManifest.Segment segment : stream.getManifest().segments) {
+ files.add(streamDirectory.resolve(segment.file).normalize());
+ }
+ }
+ final List sortedFiles = new ArrayList<>(files);
+ sortedFiles.sort(Comparator.comparing(Path::toString));
+ return sortedFiles;
+ }
+
+ private static int importBackup(final CommandLine line) throws Exception {
+ final List streams = read(line);
+ final boolean dryRun = line.hasOption(OPTION_DRY_RUN);
+ if (dryRun) {
+ System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B);
+ for (final BackupStream stream : streams) {
+ System.out.println(
+ String.format(
+ CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
+ stream.getManifest().streamId,
+ stream.getRecords().size(),
+ stream.getEventGroups().size()));
+ }
+ return 0;
+ }
+ final String host = required(line, OPTION_HOST);
+ final int port = Integer.parseInt(required(line, OPTION_PORT));
+ final String user = line.getOptionValue(OPTION_USER, "root");
+ final String password = password(line);
+ final Path checkpoint = checkpointPath(line);
+ final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user);
+ try (final IoTDBSyncClient client =
+ new IoTDBSyncClient(
+ new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(),
+ host,
+ port,
+ false,
+ "",
+ "")) {
+ handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
+ long importedGroups = 0;
+ for (final BackupStream stream : streams) {
+ importedGroups +=
+ importStream(
+ checkpoint,
+ checkpointState,
+ stream,
+ request -> transfer(client, request),
+ state -> writeCheckpoint(checkpoint, state));
+ }
+ writeCheckpoint(checkpoint, checkpointState);
+ System.out.println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D,
+ importedGroups,
+ checkpoint));
+ }
+ return 0;
+ }
+
+ static long importStream(
+ final Path checkpoint,
+ final ImportCheckpoint checkpointState,
+ final BackupStream stream,
+ final RequestTransfer requestTransfer,
+ final CheckpointPersister checkpointPersister)
+ throws Exception {
+ final String streamId = stream.getManifest().streamId;
+ final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L);
+ long importedGroups = 0;
+ for (final EventGroup group : stream.getEventGroups()) {
+ if (group.getLastSequence() <= lastApplied) {
+ continue;
+ }
+
+ int nextRequestIndex = 0;
+ if (checkpointState.inProgress == null) {
+ checkpointState.inProgress = ImportProgress.start(streamId, group);
+ checkpointPersister.persist(checkpointState);
+ } else if (checkpointState.inProgress.matches(streamId, group)) {
+ nextRequestIndex = checkpointState.inProgress.nextRequestIndex;
+ } else {
+ throw invalidCheckpoint(checkpoint);
+ }
+
+ final List requests = group.getRequests();
+ if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) {
+ throw invalidCheckpoint(checkpoint);
+ }
+ for (int index = nextRequestIndex; index < requests.size(); index++) {
+ requestTransfer.transfer(requests.get(index).toTPipeTransferReq());
+ checkpointState.inProgress.nextRequestIndex = index + 1;
+ checkpointPersister.persist(checkpointState);
+ }
+
+ checkpointState.appliedSequences.put(streamId, group.getLastSequence());
+ checkpointState.inProgress = null;
+ checkpointPersister.persist(checkpointState);
+ importedGroups++;
+ }
+ return importedGroups;
+ }
+
+ private static List read(final CommandLine line) throws IOException {
+ final String source = input(line);
+ try (final BackupInput input = BackupInput.open(Path.of(source))) {
+ return new LogicalBackupArchiveReader()
+ .read(input.getPath(), line.hasOption(OPTION_ALLOW_INCOMPLETE));
+ }
+ }
+
+ private static String input(final CommandLine line) throws IOException {
+ final String value = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
+ if (value == null || value.isBlank()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
+ }
+ return value;
+ }
+
+ private static String optionValue(
+ final CommandLine line, final String preferredOption, final String compatibilityOption) {
+ return line.hasOption(preferredOption)
+ ? line.getOptionValue(preferredOption)
+ : line.getOptionValue(compatibilityOption);
+ }
+
+ private static String required(final CommandLine line, final String option) throws IOException {
+ final String value = line.getOptionValue(option);
+ if (value == null || value.isBlank()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, option));
+ }
+ return value;
+ }
+
+ private static String password(final CommandLine line) throws IOException {
+ final boolean fromStandardInput = line.hasOption(OPTION_PASSWORD_STDIN);
+ final boolean fromEnvironment = line.hasOption(OPTION_PASSWORD_ENV);
+ if (fromStandardInput == fromEnvironment) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9);
+ }
+ if (fromStandardInput) {
+ final BufferedReader reader =
+ new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
+ final String value = reader.readLine();
+ if (value == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E);
+ }
+ return value;
+ }
+ if (fromEnvironment) {
+ final String value = System.getenv(line.getOptionValue(OPTION_PASSWORD_ENV));
+ if (value == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2);
+ }
+ return value;
+ }
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9);
+ }
+
+ private static void handshake(
+ final IoTDBSyncClient client,
+ final String timestampPrecision,
+ final String user,
+ final String password)
+ throws IOException, TException {
+ final Map params = new HashMap<>();
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId());
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false");
+ final TPipeTransferResp response =
+ client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params));
+ if (response == null
+ || response.getStatus() == null
+ || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697,
+ response == null ? "null" : response.getStatus()));
+ }
+ }
+
+ static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
+ throws TException, IOException {
+ final TPipeTransferResp response = client.pipeTransfer(request);
+ if (response == null
+ || response.getStatus() == null
+ || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
+ && response.getStatus().getCode()
+ != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11,
+ request.getType(),
+ response == null ? "null" : response.getStatus()));
+ }
+ }
+
+ private static Path checkpointPath(final CommandLine line) throws IOException {
+ final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT);
+ if (configuredCheckpoint != null) {
+ return Path.of(configuredCheckpoint);
+ }
+ final Path source = Path.of(input(line)).toAbsolutePath().normalize();
+ return source.resolveSibling(source.getFileName() + ".import.checkpoint.json");
+ }
+
+ static ImportCheckpoint readCheckpoint(
+ final Path checkpoint,
+ final List streams,
+ final String host,
+ final int port,
+ final String user)
+ throws IOException {
+ final Map sourceStreams = sourceStreamIdentities(streams);
+ if (!Files.exists(checkpoint)) {
+ final ImportCheckpoint state = new ImportCheckpoint();
+ state.formatName = CHECKPOINT_FORMAT_NAME;
+ state.formatVersion = CHECKPOINT_FORMAT_VERSION;
+ state.sourceStreams = sourceStreams;
+ state.targetHost = host;
+ state.targetPort = port;
+ state.targetUser = user;
+ return state;
+ }
+ final ImportCheckpoint state;
+ try {
+ state =
+ GSON.fromJson(
+ Files.readString(checkpoint, StandardCharsets.UTF_8), ImportCheckpoint.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint),
+ e);
+ }
+ if (state == null
+ || !CHECKPOINT_FORMAT_NAME.equals(state.formatName)
+ || !CHECKPOINT_FORMAT_VERSION.equals(state.formatVersion)
+ || state.sourceStreams == null
+ || state.appliedSequences == null) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint));
+ }
+ if (!sourceStreams.equals(state.sourceStreams)
+ || !host.equals(state.targetHost)
+ || port != state.targetPort
+ || !user.equals(state.targetUser)
+ || !checkpointStateIsValid(streams, state)) {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D,
+ checkpoint));
+ }
+ return state;
+ }
+
+ static Map sourceStreamIdentities(final List streams) {
+ final Map identities = new LinkedHashMap<>();
+ for (final BackupStream stream : streams) {
+ final LogicalBackupManifest manifest = stream.getManifest();
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ updateDigest(digest, manifest.backupId);
+ updateDigest(digest, manifest.streamId);
+ updateDigest(digest, manifest.pipeName);
+ updateDigest(digest, manifest.pipeCreationTime);
+ updateDigest(digest, manifest.sourceClusterId);
+ updateDigest(digest, manifest.regionId);
+ updateDigest(digest, manifest.streamType);
+ updateDigest(digest, manifest.segments.size());
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ updateDigest(digest, segment.file);
+ updateDigest(digest, segment.segmentId);
+ updateDigest(digest, segment.firstSequence);
+ updateDigest(digest, segment.lastSequence);
+ updateDigest(digest, segment.recordCount);
+ updateDigest(digest, segment.sizeBytes);
+ updateDigest(digest, segment.sha256);
+ updateDigest(digest, segment.status);
+ }
+ updateDigest(digest, stream.getRecords().size());
+ for (final LogicalBackupRecord record : stream.getRecords()) {
+ digest.update(record.getRecordType().getCode());
+ digest.update(
+ ByteBuffer.allocate(Long.BYTES * 4 + Integer.BYTES + Byte.BYTES + Short.BYTES)
+ .putLong(record.getSequence())
+ .putLong(record.getEventGroupId().getMostSignificantBits())
+ .putLong(record.getEventGroupId().getLeastSignificantBits())
+ .putInt(record.getOperationIndex())
+ .putLong(record.getEventTime())
+ .put(record.getRequestVersion())
+ .putShort(record.getRequestType())
+ .array());
+ updateDigest(digest, record.getMetadata());
+ updateDigest(digest, record.getPayload());
+ }
+ identities.put(manifest.streamId, LogicalBackupFormat.toHex(digest.digest()));
+ }
+ return identities;
+ }
+
+ private static void updateDigest(final MessageDigest digest, final String value) {
+ if (value == null) {
+ digest.update((byte) 0);
+ return;
+ }
+ digest.update((byte) 1);
+ updateDigest(digest, value.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void updateDigest(final MessageDigest digest, final byte[] value) {
+ updateDigest(digest, value.length);
+ digest.update(value);
+ }
+
+ private static void updateDigest(final MessageDigest digest, final int value) {
+ digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array());
+ }
+
+ private static void updateDigest(final MessageDigest digest, final long value) {
+ digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array());
+ }
+
+ private static boolean checkpointStateIsValid(
+ final List streams, final ImportCheckpoint state) {
+ final Map appliedSequences = state.appliedSequences;
+ if (appliedSequences.size() > streams.size()) {
+ return false;
+ }
+ boolean foundIncompleteStream = false;
+ boolean matchedInProgress = false;
+ for (final BackupStream stream : streams) {
+ final String streamId = stream.getManifest().streamId;
+ final Long applied = appliedSequences.getOrDefault(streamId, -1L);
+ if (applied == null || applied < -1) {
+ return false;
+ }
+
+ int appliedGroupIndex = -1;
+ if (applied != -1) {
+ for (int index = 0; index < stream.getEventGroups().size(); index++) {
+ if (stream.getEventGroups().get(index).getLastSequence() == applied) {
+ appliedGroupIndex = index;
+ break;
+ }
+ }
+ if (appliedGroupIndex < 0) {
+ return false;
+ }
+ }
+
+ final boolean streamComplete = appliedGroupIndex == stream.getEventGroups().size() - 1;
+ if (foundIncompleteStream && appliedGroupIndex >= 0) {
+ return false;
+ }
+ if (!streamComplete && !foundIncompleteStream) {
+ foundIncompleteStream = true;
+ if (state.inProgress != null) {
+ final EventGroup nextGroup = stream.getEventGroups().get(appliedGroupIndex + 1);
+ if (!state.inProgress.matches(streamId, nextGroup)
+ || state.inProgress.nextRequestIndex < 0
+ || state.inProgress.nextRequestIndex > nextGroup.getRequests().size()) {
+ return false;
+ }
+ matchedInProgress = true;
+ }
+ }
+ }
+ return (state.inProgress == null || matchedInProgress)
+ && appliedSequences.keySet().stream()
+ .allMatch(
+ streamId ->
+ streams.stream()
+ .anyMatch(stream -> stream.getManifest().streamId.equals(streamId)));
+ }
+
+ private static IOException invalidCheckpoint(final Path checkpoint) {
+ return new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint));
+ }
+
+ static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState)
+ throws IOException {
+ final Path parent = checkpoint.toAbsolutePath().normalize().getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp");
+ Files.deleteIfExists(temporary);
+ final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState));
+ try (final java.nio.channels.FileChannel checkpointChannel =
+ java.nio.channels.FileChannel.open(
+ temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ while (checkpointBytes.hasRemaining()) {
+ checkpointChannel.write(checkpointBytes);
+ }
+ checkpointChannel.force(true);
+ }
+ try {
+ Files.move(
+ temporary,
+ checkpoint,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final java.nio.file.AtomicMoveNotSupportedException e) {
+ Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ private static String getClusterId() {
+ final byte[] bytes = new byte[16];
+ new SecureRandom().nextBytes(bytes);
+ return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes);
+ }
+
+ private static void printUsage() {
+ final HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ new PrintWriter(System.out),
+ 120,
+ CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE,
+ CliMessages
+ .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E,
+ optionsForHelp(),
+ 2,
+ 2,
+ "",
+ true);
+ }
+
+ private static Options optionsForHelp() {
+ final Options options = new Options();
+ addOption(options, OPTION_INPUT, true, false);
+ addOption(options, OPTION_OUTPUT, true, false);
+ addOption(options, OPTION_RESUME, true, false);
+ addOption(options, OPTION_FORMAT, true, false);
+ addOption(options, OPTION_DEEP, false, false);
+ addOption(options, OPTION_HOST, true, false);
+ addOption(options, OPTION_PORT, true, false);
+ addOption(options, OPTION_USER, true, false);
+ addOption(options, OPTION_PASSWORD_STDIN, false, false);
+ addOption(options, OPTION_PASSWORD_ENV, true, false);
+ addOption(options, OPTION_CHECKPOINT, true, false);
+ addOption(options, OPTION_DRY_RUN, false, false);
+ addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
+ addOption(options, OPTION_JSON, false, false);
+ return options;
+ }
+
+ static class ImportCheckpoint {
+ String formatName;
+ String formatVersion;
+ Map sourceStreams = new LinkedHashMap<>();
+ String targetHost;
+ int targetPort;
+ String targetUser;
+ Map appliedSequences = new LinkedHashMap<>();
+ ImportProgress inProgress;
+ }
+
+ static class ImportProgress {
+ String streamId;
+ String eventGroupId;
+ long lastSequence;
+ int nextRequestIndex;
+
+ static ImportProgress start(final String streamId, final EventGroup group) {
+ final ImportProgress progress = new ImportProgress();
+ progress.streamId = streamId;
+ progress.eventGroupId = group.getEventGroupId().toString();
+ progress.lastSequence = group.getLastSequence();
+ return progress;
+ }
+
+ boolean matches(final String candidateStreamId, final EventGroup group) {
+ return candidateStreamId.equals(streamId)
+ && group.getEventGroupId().toString().equals(eventGroupId)
+ && group.getLastSequence() == lastSequence;
+ }
+ }
+
+ @FunctionalInterface
+ interface RequestTransfer {
+ void transfer(TPipeTransferReq request) throws Exception;
+ }
+
+ @FunctionalInterface
+ interface CheckpointPersister {
+ void persist(ImportCheckpoint state) throws IOException;
+ }
+
+ private static class BackupInput implements AutoCloseable {
+ private final Path path;
+ private final Path temporaryDirectory;
+
+ private BackupInput(final Path path, final Path temporaryDirectory) {
+ this.path = path;
+ this.temporaryDirectory = temporaryDirectory;
+ }
+
+ private static BackupInput open(final Path source) throws IOException {
+ final Path normalizedSource = source.toAbsolutePath().normalize();
+ if (!Files.isRegularFile(normalizedSource)
+ || "manifest.json".equals(normalizedSource.getFileName().toString())) {
+ return new BackupInput(normalizedSource, null);
+ }
+ final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-");
+ try {
+ extract(normalizedSource, temporaryDirectory);
+ return new BackupInput(temporaryDirectory, temporaryDirectory);
+ } catch (final IOException | RuntimeException e) {
+ deleteRecursively(temporaryDirectory);
+ throw e;
+ }
+ }
+
+ private static void extract(final Path archive, final Path target) throws IOException {
+ int entryCount = 0;
+ long totalBytes = 0;
+ final byte[] buffer = new byte[64 * 1024];
+ try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) {
+ ZipEntry entry;
+ while ((entry = zip.getNextEntry()) != null) {
+ if (++entryCount > MAX_ARCHIVE_ENTRIES) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ final Path output = target.resolve(entry.getName()).normalize();
+ if (!output.startsWith(target)) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152,
+ entry.getName()));
+ }
+ if (entry.isDirectory()) {
+ Files.createDirectories(output);
+ } else {
+ final Path parent = output.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ try (final OutputStream stream =
+ Files.newOutputStream(
+ output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ int length;
+ while ((length = zip.read(buffer)) >= 0) {
+ if (length == 0) {
+ continue;
+ }
+ totalBytes += length;
+ if (totalBytes > MAX_ARCHIVE_BYTES) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ stream.write(buffer, 0, length);
+ }
+ }
+ }
+ zip.closeEntry();
+ }
+ }
+ }
+
+ private Path getPath() {
+ return path;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (temporaryDirectory != null) {
+ deleteRecursively(temporaryDirectory);
+ }
+ }
+
+ private static void deleteRecursively(final Path directory) throws IOException {
+ if (!Files.exists(directory)) {
+ return;
+ }
+ try (final java.util.stream.Stream paths = Files.walk(directory)) {
+ for (final Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
+ Files.deleteIfExists(path);
+ }
+ }
+ }
+ }
+}
diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
new file mode 100644
index 0000000000000..6e7af8726ab35
--- /dev/null
+++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool.pipe;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.commons.cli.ParseException;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
+
+public class PipeLogicalBackupToolTest {
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("source");
+ writeBackup(source);
+
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", source.toString(), "--deep"}));
+
+ final Path archive = root.resolve("backup.zip");
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "export",
+ "--input",
+ source.toString(),
+ "--output",
+ archive.toString(),
+ "--format=binary"
+ }));
+ Assert.assertTrue(Files.isRegularFile(archive));
+ try (final ZipFile zip = new ZipFile(archive.toFile())) {
+ Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal"));
+ Assert.assertNull(zip.getEntry(".backup.lock"));
+ Assert.assertEquals(2, zip.size());
+ }
+
+ Assert.assertEquals(
+ 0, PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()}));
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {"import", "--input", archive.toString(), "--dry-run"}));
+
+ final byte[] originalArchive = Files.readAllBytes(archive);
+ try {
+ PipeLogicalBackupTool.run(
+ new String[] {"export", "--input", source.toString(), "--output", archive.toString()});
+ Assert.fail();
+ } catch (final java.nio.file.FileAlreadyExistsException expected) {
+ Assert.assertArrayEquals(originalArchive, Files.readAllBytes(archive));
+ }
+ }
+
+ @Test
+ public void testCheckpointSourceIdentityIncludesContent() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ final Map before = PipeLogicalBackupTool.sourceStreamIdentities(streams);
+
+ streams.get(0).getManifest().segments.get(0).sha256 = "different-content";
+ final Map after = PipeLogicalBackupTool.sourceStreamIdentities(streams);
+
+ Assert.assertNotEquals(before, after);
+ }
+
+ @Test
+ public void testZipSlipIsRejected() throws Exception {
+ final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip");
+ try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive))) {
+ zip.putNextEntry(new ZipEntry("../outside"));
+ zip.write(1);
+ zip.closeEntry();
+ }
+
+ try {
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()});
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testPlaintextPasswordOptionIsRejected() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+
+ try {
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "import", "--source", source.toString(), "--dry-run", "--password", "secret"
+ });
+ Assert.fail();
+ } catch (final ParseException expected) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testRemovedCommandAliasesAreRejected() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+
+ for (final String command : Arrays.asList("restore", "stats")) {
+ try {
+ PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()});
+ Assert.fail();
+ } catch (final ParseException expected) {
+ // Expected.
+ }
+ }
+ }
+
+ @Test
+ public void testTransferAcceptsIdempotentConflict() throws Exception {
+ final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class);
+ final TPipeTransferReq request = request(1);
+ Mockito.when(client.pipeTransfer(request))
+ .thenReturn(
+ new TPipeTransferResp(
+ new TSStatus(
+ TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())));
+
+ PipeLogicalBackupTool.transfer(client, request);
+ }
+
+ @Test
+ public void testImportCheckpointResumesAtRequestGranularity() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("source");
+ writeBackup(source, Arrays.asList(request(1), request(2)));
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ final BackupStream stream = streams.get(0);
+ final Path checkpoint = root.resolve("import.checkpoint.json");
+ final PipeLogicalBackupTool.ImportCheckpoint state =
+ PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root");
+ final List transferredBodies = new ArrayList<>();
+ final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true);
+
+ try {
+ PipeLogicalBackupTool.importStream(
+ checkpoint,
+ state,
+ stream,
+ request -> transferredBodies.add((int) request.getBody()[0]),
+ checkpointState -> {
+ if (checkpointState.inProgress != null
+ && checkpointState.inProgress.nextRequestIndex == 1
+ && failCheckpointOnce.getAndSet(false)) {
+ throw new IOException("simulated checkpoint failure");
+ }
+ PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState);
+ });
+ Assert.fail();
+ } catch (final IOException expected) {
+ // The request succeeded, but advancing its checkpoint was interrupted.
+ }
+
+ Assert.assertEquals(Collections.singletonList(1), transferredBodies);
+ final PipeLogicalBackupTool.ImportCheckpoint durableState =
+ PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root");
+ Assert.assertNotNull(durableState.inProgress);
+ Assert.assertEquals(0, durableState.inProgress.nextRequestIndex);
+
+ Assert.assertEquals(
+ 1,
+ PipeLogicalBackupTool.importStream(
+ checkpoint,
+ durableState,
+ stream,
+ request -> transferredBodies.add((int) request.getBody()[0]),
+ checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState)));
+ Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies);
+ Assert.assertNull(durableState.inProgress);
+ Assert.assertEquals(
+ Long.valueOf(stream.getEventGroups().get(0).getLastSequence()),
+ durableState.appliedSequences.get(stream.getManifest().streamId));
+ }
+
+ private static void writeBackup(final Path directory) throws Exception {
+ writeBackup(directory, Collections.singletonList(request(3)));
+ }
+
+ private static void writeBackup(final Path directory, final List requests)
+ throws Exception {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "data-1";
+ manifest.streamType = "data";
+ manifest.timestampPrecision = "ms";
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, requests, "metadata");
+ }
+ }
+
+ private static TPipeTransferReq request(final int value) {
+ return new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 10)
+ .setBody(ByteBuffer.wrap(new byte[] {(byte) value}));
+ }
+}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index b818a52e83877..3a914bc80c763 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2643,4 +2643,18 @@ private DataNodePipeMessages() {}
"Interrupted while reading OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}";
public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 =
"Failed to read OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 =
+ "Unsupported logical backup event policy: %s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
+ "Unsupported logical backup resume policy: %s";
+ public static final String
+ EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
+ "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory";
+ public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
+ "Unsupported Pipe event for logical backup: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 =
+ "Logical backup ID must resolve below the configured backup directory: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB =
+ "Logical backup directory must not be empty";
}
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index fe768a1f0181c..08803638ea60a 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2467,4 +2467,18 @@ private DataNodePipeMessages() {}
"读取 OPC UA 服务器操作限制时被中断，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}";
public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 =
"读取 OPC UA 服务器操作限制失败，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 =
+ "不支持的逻辑备份事件策略：%s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
+ "不支持的逻辑备份恢复策略：%s";
+ public static final String
+ EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
+ "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录";
+ public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
+ "逻辑备份不支持 Pipe 事件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 =
+ "Pipe 逻辑备份 ID 必须解析到配置的备份目录下：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB =
+ "Pipe 逻辑备份目录不能为空";
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
index c8e87890afd69..08061c954afcb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
@@ -26,6 +26,7 @@
import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink;
import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink;
+import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink;
import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
@@ -74,6 +75,9 @@ protected void initConstructors() {
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WRITE_BACK_CONNECTOR.getPipePluginName(), WriteBackSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(),
+ () -> new LogicalBackupSink("data"));
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBDataRegionAsyncSink::new);
@@ -96,6 +100,9 @@ protected void initConstructors() {
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WRITE_BACK_SINK.getPipePluginName(), WriteBackSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(),
+ () -> new LogicalBackupSink("data"));
pluginConstructors.put(
BuiltinPipePlugin.SUBSCRIPTION_SINK.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
index 160ecf54c0158..a4eba6400c178 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
@@ -23,6 +23,7 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink;
+import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -46,6 +47,9 @@ protected void initConstructors() {
IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(),
+ () -> new LogicalBackupSink("schema"));
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new);
@@ -59,6 +63,9 @@ protected void initConstructors() {
BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(),
+ () -> new LogicalBackupSink("schema"));
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
new file mode 100644
index 0000000000000..ef152b58b7c34
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -0,0 +1,621 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup;
+
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
+import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent;
+import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
+import org.apache.iotdb.pipe.api.PipeConnector;
+import org.apache.iotdb.pipe.api.annotation.TableModel;
+import org.apache.iotdb.pipe.api.annotation.TreeModel;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+
+@TreeModel
+@TableModel
+public class LogicalBackupSink implements PipeConnector {
+
+ private static final int FILE_PIECE_BYTES = 64 * 1024;
+ private LogicalBackupWriter writer;
+ private String pipeName;
+ private long creationTime;
+ private int regionId;
+ private boolean includeHeartbeat;
+ private String unsupportedEventPolicy;
+ private final String streamType;
+
+ public LogicalBackupSink() {
+ this("unknown");
+ }
+
+ public LogicalBackupSink(final String streamType) {
+ this.streamType = streamType;
+ }
+
+ @Override
+ public void validate(final PipeParameterValidator validator) throws Exception {
+ final PipeParameters parameters = validator.getParameters();
+ validator.validateSynonymAttributes(
+ Collections.singletonList(PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY),
+ Collections.singletonList(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY),
+ true);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
+ validateDataRegionResumePolicy(parameters);
+ final String policy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+ if (!PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL.equals(policy)
+ && !PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(policy)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405,
+ policy));
+ }
+ }
+
+ private static void validateSynonymAttributes(
+ final PipeParameterValidator validator,
+ final String connectorAttribute,
+ final String sinkAttribute) {
+ validator.validateSynonymAttributes(
+ Collections.singletonList(connectorAttribute),
+ Collections.singletonList(sinkAttribute),
+ false);
+ }
+
+ private void validateDataRegionResumePolicy(final PipeParameters parameters)
+ throws PipeParameterNotValidException {
+ if (!"data".equals(streamType)) {
+ return;
+ }
+ final boolean explicitlyConfigured =
+ parameters.hasAnyAttributes(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ final String resume =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE);
+ if (!explicitlyConfigured
+ || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) {
+ throw new PipeParameterNotValidException(
+ DataNodePipeMessages
+ .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8);
+ }
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ pipeName = configuration.getRuntimeEnvironment().getPipeName();
+ creationTime = configuration.getRuntimeEnvironment().getCreationTime();
+ regionId = configuration.getRuntimeEnvironment().getRegionId();
+ final String root =
+ parameters.getStringByKeys(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY);
+ if (root == null || root.isBlank()) {
+ throw new PipeParameterNotValidException(
+ DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB);
+ }
+ final String backupId =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY),
+ pipeName + "-" + creationTime);
+ if (backupId == null || backupId.isBlank()) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId;
+ final Path backupRoot = Paths.get(root).toAbsolutePath().normalize();
+ final Path backupIdPath;
+ try {
+ backupIdPath = Path.of(backupId);
+ } catch (final RuntimeException e) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ if (backupIdPath.isAbsolute()
+ || backupIdPath.getNameCount() != 1
+ || ".".equals(backupId)
+ || "..".equals(backupId)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize();
+ if (!directory.startsWith(backupRoot)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ final String resume =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+ validateDataRegionResumePolicy(parameters);
+ final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume);
+ if (!append
+ && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)
+ && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9,
+ resume));
+ }
+ if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) {
+ directory =
+ directory.resolveSibling(
+ directory.getFileName()
+ + "-new-"
+ + System.currentTimeMillis()
+ + "-"
+ + UUID.randomUUID().toString().substring(0, 8));
+ }
+ final long segmentSize =
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE);
+ final int maxRecord =
+ parameters.getIntOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE);
+ final String fsync =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE)
+ .toUpperCase(Locale.ROOT);
+ final LogicalBackupWriter.FsyncPolicy fsyncPolicy =
+ LogicalBackupWriter.FsyncPolicy.valueOf(fsync);
+ final int fsyncBatch =
+ parameters.getIntOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE);
+ final long fsyncPeriod =
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE);
+ includeHeartbeat =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE);
+ unsupportedEventPolicy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = backupId;
+ manifest.pipeName = pipeName;
+ manifest.pipeCreationTime = creationTime;
+ manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId();
+ manifest.sourceVersion = IoTDBConstant.VERSION;
+ manifest.timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+ manifest.streamId = streamId;
+ manifest.regionId = regionId;
+ manifest.streamType = streamType;
+ manifest.sinkTaskId = streamId;
+ writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ segmentSize,
+ maxRecord,
+ fsyncPolicy,
+ fsyncBatch,
+ fsyncPeriod,
+ append);
+ }
+
+ @Override
+ public void handshake() {}
+
+ @Override
+ public void heartbeat() throws Exception {
+ if (writer != null) {
+ writer.heartbeat();
+ }
+ }
+
+ @Override
+ public void transfer(final TabletInsertionEvent event) throws Exception {
+ if (event instanceof PipeInsertNodeTabletInsertionEvent) {
+ final PipeInsertNodeTabletInsertionEvent insert = (PipeInsertNodeTabletInsertionEvent) event;
+ transferEnriched(insert, () -> requests(insert));
+ } else if (event instanceof PipeRawTabletInsertionEvent) {
+ final PipeRawTabletInsertionEvent raw = (PipeRawTabletInsertionEvent) event;
+ transferEnriched(
+ raw,
+ () ->
+ Collections.singletonList(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ raw.convertToTablet(),
+ raw.isAligned(),
+ raw.isTableModelEvent()
+ ? raw.getTableModelDatabaseName()
+ : raw.getTreeModelDatabaseName())));
+ } else {
+ handleUnsupported(event);
+ }
+ }
+
+ @Override
+ public void transfer(final TsFileInsertionEvent event) throws Exception {
+ if (!(event instanceof PipeTsFileInsertionEvent)) {
+ handleUnsupported(event);
+ return;
+ }
+ final PipeTsFileInsertionEvent tsFile = (PipeTsFileInsertionEvent) event;
+ final List requests = new ArrayList<>();
+ if (!tsFile.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ tsFile.consumeTabletInsertionEventsWithRetry(
+ tablet -> {
+ try {
+ requests.add(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ tablet.convertToTablet(),
+ tablet.isAligned(),
+ tablet.isTableModelEvent()
+ ? tablet.getTableModelDatabaseName()
+ : tablet.getTreeModelDatabaseName()));
+ } catch (final IOException e) {
+ throw new PipeException(e.getMessage(), e);
+ }
+ },
+ getClass().getName());
+ if (!requests.isEmpty()) {
+ writer.writeEvent(eventId(tsFile), System.currentTimeMillis(), requests, metadata(tsFile));
+ }
+ } finally {
+ event.close();
+ tsFile.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ @Override
+ public void transfer(final Event event) throws Exception {
+ if (event instanceof PipeDeleteDataNodeEvent) {
+ final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event;
+ transferEnriched(
+ deletion,
+ () ->
+ Collections.singletonList(
+ PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode())));
+ } else if (event instanceof PipeSchemaRegionWritePlanEvent) {
+ final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event;
+ transferEnriched(
+ schema,
+ () ->
+ Collections.singletonList(
+ PipeTransferPlanNodeReq.toTPipeTransferReq(schema.getPlanNode())));
+ } else if (event instanceof PipeSchemaRegionSnapshotEvent) {
+ transferSnapshot((PipeSchemaRegionSnapshotEvent) event);
+ } else if (event instanceof PipeHeartbeatEvent) {
+ if (includeHeartbeat) {
+ writer.writeControl(
+ LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event));
+ }
+ } else if (event instanceof PipeTerminateEvent) {
+ writer.writeControl(
+ LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event));
+ } else {
+ handleUnsupported(event);
+ }
+ }
+
+ private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws Exception {
+ if (!event.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ final List requests = new ArrayList<>();
+ final File[] files =
+ new File[] {
+ event.getMTreeSnapshotFile(),
+ event.getTagLogSnapshotFile(),
+ event.getAttributeSnapshotFile()
+ };
+ for (final File file : files) {
+ if (file == null) {
+ continue;
+ }
+ try (final FileInputStream input = new FileInputStream(file)) {
+ final byte[] piece = new byte[FILE_PIECE_BYTES];
+ long offset = 0;
+ int length;
+ while ((length = input.read(piece)) >= 0) {
+ if (length == 0) {
+ continue;
+ }
+ requests.add(
+ PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(
+ file.getName(), offset, Arrays.copyOf(piece, length)));
+ offset += length;
+ }
+ }
+ }
+ requests.add(
+ PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq(
+ event.getTreePattern().getPattern(),
+ event.getTablePattern().getDatabasePattern(),
+ event.getTablePattern().getTablePattern(),
+ event.getTreePattern().isTreeModelDataAllowedToBeCaptured(),
+ event.getTablePattern().isTableModelDataAllowedToBeCaptured(),
+ files[0] == null ? null : files[0].getName(),
+ files[0] == null ? 0 : files[0].length(),
+ files[1] == null ? null : files[1].getName(),
+ files[1] == null ? 0 : files[1].length(),
+ files[2] == null ? null : files[2].getName(),
+ files[2] == null ? 0 : files[2].length(),
+ event.getDatabaseName(),
+ event.toSealTypeString()));
+ writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event));
+ } finally {
+ event.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ private List requests(final PipeInsertNodeTabletInsertionEvent event)
+ throws IOException {
+ final String databaseName =
+ event.isTableModelEvent()
+ ? event.getTableModelDatabaseName()
+ : event.getTreeModelDatabaseName();
+ if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) {
+ return Collections.singletonList(
+ PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq(
+ event.getInsertNode(), databaseName));
+ }
+
+ final List tablets = event.convertToTablets();
+ final List requests = new ArrayList<>(tablets.size());
+ for (int index = 0; index < tablets.size(); index++) {
+ requests.add(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ tablets.get(index), event.isAligned(index), databaseName));
+ }
+ return requests;
+ }
+
+ private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier)
+ throws Exception {
+ if (!event.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ writer.writeEvent(
+ eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event));
+ } finally {
+ event.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ @FunctionalInterface
+ private interface RequestSupplier {
+ List get() throws Exception;
+ }
+
+ private void handleUnsupported(final Event event) throws Exception {
+ if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) {
+ writer.recordSkippedEvent(
+ System.currentTimeMillis(), event == null ? "null" : event.getClass().getName());
+ return;
+ }
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97,
+ event == null ? "null" : event.getClass().getName()));
+ }
+
+ private UUID eventId(final EnrichedEvent event) {
+ return UUID.nameUUIDFromBytes(
+ (event.getClass().getName()
+ + '\0'
+ + event.getPipeName()
+ + '\0'
+ + event.getCreationTime()
+ + '\0'
+ + event.getRegionId()
+ + '\0'
+ + commitIds(event))
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ private String commitIds(final EnrichedEvent event) {
+ try {
+ return String.valueOf(event.getCommitIds());
+ } catch (final UnsupportedOperationException ignored) {
+ return Long.toString(event.getCommitId());
+ }
+ }
+
+ private String metadata(final Event event) {
+ if (event instanceof EnrichedEvent) {
+ final EnrichedEvent enriched = (EnrichedEvent) event;
+ return "pipe="
+ + enriched.getPipeName()
+ + ";creationTime="
+ + enriched.getCreationTime()
+ + ";regionId="
+ + enriched.getRegionId()
+ + ";commitIds="
+ + commitIds(enriched)
+ + ";eventType="
+ + event.getClass().getName();
+ }
+ return "eventType=" + (event == null ? "null" : event.getClass().getName());
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
new file mode 100644
index 0000000000000..37c3d622b4f55
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LogicalBackupSinkTest {
+
+ @Test
+ public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup");
+
+ assertInvalid(new LogicalBackupSink("data"), attributes);
+
+ attributes.put(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS);
+ assertInvalid(new LogicalBackupSink("data"), attributes);
+
+ attributes.put(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND);
+ new LogicalBackupSink("data")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+
+ attributes.remove(PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ attributes.put(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND);
+ new LogicalBackupSink("data")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+ }
+
+ @Test
+ public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup");
+
+ new LogicalBackupSink("schema")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+ }
+
+ private static void assertInvalid(
+ final LogicalBackupSink sink, final Map attributes) throws Exception {
+ try {
+ sink.validate(new PipeParameterValidator(new PipeParameters(new HashMap<>(attributes))));
+ Assert.fail();
+ } catch (final PipeParameterNotValidException expected) {
+ Assert.assertTrue(expected.getMessage().contains("append"));
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml
index 3bb68a29562c3..a632dbff5ddef 100644
--- a/iotdb-core/node-commons/pom.xml
+++ b/iotdb-core/node-commons/pom.xml
@@ -139,6 +139,10 @@
com.google.guava
guava
+
+ com.google.code.gson
+ gson
+
com.google.code.findbugs
jsr305
diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
new file mode 100644
index 0000000000000..1629e2dd00028
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.i18n;
+
+public final class LogicalBackupMessages {
+
+ public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "directory";
+ public static final String EXCEPTION_MANIFEST_7F5CB74A = "manifest";
+ public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsyncPolicy";
+ public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "eventGroupId";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D =
+ "Logical backup directory already exists: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC =
+ "Logical backup event must contain at least one request";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 =
+ "Logical backup request body must not be null";
+ public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 =
+ "Logical backup record exceeds max-record-bytes";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E =
+ "Unsupported logical backup manifest format: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 =
+ "Logical backup manifest does not match %s: expected %s, found %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 =
+ "Logical backup directory is locked: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB =
+ "Logical backup writer is closed";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A =
+ "Incomplete logical backup segment header: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 =
+ "Invalid logical backup segment footer size: %s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 =
+ "Invalid logical backup record magic at offset %d in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 =
+ "Logical backup record header CRC mismatch at offset %d in %s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E =
+ "Invalid logical backup record length at offset %d in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 =
+ "Logical backup record payload CRC mismatch at offset %d in %s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D =
+ "Unknown logical backup record type %d in %s";
+ public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A =
+ "Nested logical backup event groups in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 =
+ "Logical backup commit without begin in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F =
+ "Logical backup operation index mismatch at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B =
+ "Logical backup sequence mismatch at offset %d in %s";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 =
+ "Incomplete logical backup record tail: %s";
+ public static final String
+ EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 =
+ "Sealed logical backup segment contains an open event group: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 =
+ "Invalid logical backup segment magic: %s";
+ public static final String
+ EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB =
+ "Unsupported logical backup segment major version %d: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 =
+ "Logical backup segment header CRC mismatch: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 =
+ "Invalid logical backup segment footer magic: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D =
+ "Logical backup segment footer ID mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 =
+ "Logical backup segment digest mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 =
+ "Logical backup segment footer CRC mismatch: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB =
+ "Logical backup segment footer metadata mismatch: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 =
+ "Logical backup event ID %s was already written with a different digest";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 =
+ "Invalid logical backup writer configuration: segment-size, max-record, fsync-batch and"
+ + " fsync-period must be positive";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F =
+ "Logical backup request outside an event group at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B =
+ "Logical backup control record inside an event group at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A =
+ "Logical backup segment ID mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 =
+ "Logical backup sequence gap in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 =
+ "Logical backup has no segments";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 =
+ "Invalid logical backup segment path: %s";
+ public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E =
+ "No logical backup manifest found under %s";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE =
+ "Duplicate logical backup stream ID: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 =
+ "Logical backup manifest has no segments: %s";
+ public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B =
+ "Unlisted logical backup segment %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 =
+ "Logical backup segment metadata mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF =
+ "Logical backup segment is not sealed: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB =
+ "Logical backup manifest counters do not match segment contents: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E =
+ "Logical backup sequence is not continuous across segments: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 =
+ "Logical backup manifest is invalid: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 =
+ "Logical backup contains skipped events: %s";
+
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 =
+ "Logical backup request version %d is not supported";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 =
+ "Logical backup request type %d is not allowed";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 =
+ "Duplicate logical backup event group ID: %s";
+ public static final String
+ EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 =
+ "Symbolic links are not allowed in logical backup directory paths: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD =
+ "Logical backup directory is unavailable: %s";
+
+ private LogicalBackupMessages() {}
+}
diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
new file mode 100644
index 0000000000000..bfc2bed98fed1
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.i18n;
+
+public final class LogicalBackupMessages {
+
+ public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "目录";
+ public static final String EXCEPTION_MANIFEST_7F5CB74A = "清单";
+ public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsync 策略";
+ public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "事件组 ID";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D =
+ "Pipe 逻辑备份目录已存在：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC =
+ "Pipe 逻辑备份事件必须至少包含一个请求";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 =
+ "Pipe 逻辑备份请求体不能为空";
+ public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 =
+ "Pipe 逻辑备份记录超过 max-record-bytes";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E =
+ "不支持的 Pipe 逻辑备份清单格式：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 =
+ "Pipe 逻辑备份清单的 %s 不匹配：预期 %s，实际 %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 =
+ "Pipe 逻辑备份目录已被锁定：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB =
+ "Pipe 逻辑备份写入器已关闭";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A =
+ "Pipe 逻辑备份 segment 文件头不完整：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 =
+ "Pipe 逻辑备份 segment 文件尾大小无效：%s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 =
+ "偏移 %d 处的 Pipe 逻辑备份记录魔数无效，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 =
+ "偏移 %d 处的 Pipe 逻辑备份记录头 CRC 不匹配，文件：%s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E =
+ "偏移 %d 处的 Pipe 逻辑备份记录长度无效，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 =
+ "偏移 %d 处的 Pipe 逻辑备份记录载荷 CRC 不匹配，文件：%s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D =
+ "未知的 Pipe 逻辑备份记录类型 %d，文件：%s";
+ public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A =
+ "Pipe 逻辑备份事件组发生嵌套，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 =
+ "Pipe 逻辑备份事件组缺少开始记录，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F =
+ "偏移 %d 处的 Pipe 逻辑备份操作序号不匹配，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B =
+ "偏移 %d 处的 Pipe 逻辑备份 sequence 不连续，文件：%s";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 =
+ "Pipe 逻辑备份记录尾部不完整：%s";
+ public static final String
+ EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 =
+ "已封存的 Pipe 逻辑备份 segment 包含未提交事件组：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 =
+ "Pipe 逻辑备份 segment 魔数无效：%s";
+ public static final String
+ EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB =
+ "不支持的 Pipe 逻辑备份 segment 主版本 %d：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 =
+ "Pipe 逻辑备份 segment 文件头 CRC 不匹配：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 =
+ "Pipe 逻辑备份 segment 文件尾魔数无效：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D =
+ "Pipe 逻辑备份 segment 文件尾 ID 不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 =
+ "Pipe 逻辑备份 segment 摘要不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 =
+ "Pipe 逻辑备份 segment 文件尾 CRC 不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB =
+ "Pipe 逻辑备份 segment 文件尾元数据不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 =
+ "Pipe 逻辑备份事件 ID %s 已使用不同摘要写入";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 =
+ "Pipe 逻辑备份写入器配置无效：segment-size、max-record、fsync-batch 和 fsync-period"
+ + " 必须为正数";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F =
+ "偏移 %d 处的 Pipe 逻辑备份请求不在事件组中，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B =
+ "偏移 %d 处的 Pipe 逻辑备份控制记录位于事件组内，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A =
+ "Pipe 逻辑备份 segment ID 不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 =
+ "Pipe 逻辑备份 sequence 在 %s 中不连续";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 =
+ "Pipe 逻辑备份不包含 segment";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 =
+ "Pipe 逻辑备份 segment 路径无效：%s";
+ public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E =
+ "在 %s 下未找到 Pipe 逻辑备份 manifest";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE =
+ "Pipe 逻辑备份 stream ID 重复：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 =
+ "Pipe 逻辑备份 manifest 不包含 segment：%s";
+ public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B =
+ "未在 manifest 中列出 Pipe 逻辑备份 segment：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 =
+ "Pipe 逻辑备份 segment 元数据不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF =
+ "Pipe 逻辑备份 segment 尚未封存：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB =
+ "Pipe 逻辑备份 manifest 计数与 segment 内容不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E =
+ "Pipe 逻辑备份 sequence 在 segment 之间不连续：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 =
+ "Pipe 逻辑备份 manifest 无效：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 =
+ "Pipe 逻辑备份包含已跳过的事件：%s";
+
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 =
+ "不支持逻辑备份请求版本 %d";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 =
+ "不允许逻辑备份请求类型 %d";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 =
+ "逻辑备份事件组 ID 重复：%s";
+ public static final String
+ EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 =
+ "逻辑备份目录路径中不允许使用符号链接：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD =
+ "逻辑备份目录不可用：%s";
+
+ private LogicalBackupMessages() {}
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index 76b45d1e79895..2a10db843618c 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -38,6 +38,7 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
+import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
@@ -95,6 +96,7 @@ public enum BuiltinPipePlugin {
OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class),
OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class),
WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class),
+ LOGICAL_BACKUP_CONNECTOR("logical-backup-connector", LogicalBackupSink.class),
DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class),
IOTDB_THRIFT_SINK("iotdb-thrift-sink", IoTDBThriftSink.class),
@@ -107,6 +109,7 @@ public enum BuiltinPipePlugin {
OPC_UA_SINK("opc-ua-sink", OpcUaSink.class),
OPC_DA_SINK("opc-da-sink", OpcDaSink.class),
WRITE_BACK_SINK("write-back-sink", WriteBackSink.class),
+ LOGICAL_BACKUP_SINK("logical-backup-sink", LogicalBackupSink.class),
SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class),
IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class),
// Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename.
@@ -178,6 +181,7 @@ public String getClassName() {
OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(),
OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(),
WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(),
+ LOGICAL_BACKUP_CONNECTOR.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
// Sinks
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java
new file mode 100644
index 0000000000000..d5fb9b22021ef
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.PlaceholderSink;
+
+public class LogicalBackupSink extends PlaceholderSink {}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
index f2d9ea283c823..6d5dac18c3d9c 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
@@ -80,6 +80,55 @@ public class PipeSinkConstant {
public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first";
public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true;
+ public static final String CONNECTOR_LOGICAL_BACKUP_DIR_KEY = "connector.dir";
+ public static final String SINK_LOGICAL_BACKUP_DIR_KEY = "sink.dir";
+ public static final String CONNECTOR_LOGICAL_BACKUP_ID_KEY = "connector.backup-id";
+ public static final String SINK_LOGICAL_BACKUP_ID_KEY = "sink.backup-id";
+ public static final String CONNECTOR_LOGICAL_BACKUP_RESUME_KEY = "connector.resume";
+ public static final String SINK_LOGICAL_BACKUP_RESUME_KEY = "sink.resume";
+ public static final String LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS = "fail-if-exists";
+ public static final String LOGICAL_BACKUP_RESUME_APPEND = "append";
+ public static final String LOGICAL_BACKUP_RESUME_NEW = "new";
+ public static final String LOGICAL_BACKUP_RESUME_DEFAULT_VALUE =
+ LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS;
+ public static final String CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY =
+ "connector.segment-size-bytes";
+ public static final String SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = "sink.segment-size-bytes";
+ public static final long LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE = 256L * MB;
+ public static final String CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY =
+ "connector.max-record-bytes";
+ public static final String SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = "sink.max-record-bytes";
+ public static final int LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE = 64 * 1024 * 1024;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "connector.fsync-policy";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "sink.fsync-policy";
+ public static final String LOGICAL_BACKUP_FSYNC_ALWAYS = "always";
+ public static final String LOGICAL_BACKUP_FSYNC_BATCH = "batch";
+ public static final String LOGICAL_BACKUP_FSYNC_PERIODIC = "periodic";
+ public static final String LOGICAL_BACKUP_FSYNC_NONE = "none";
+ public static final String LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE =
+ LOGICAL_BACKUP_FSYNC_ALWAYS;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY =
+ "connector.fsync-batch-operations";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY =
+ "sink.fsync-batch-operations";
+ public static final int LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE = 1000;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY =
+ "connector.fsync-period-ms";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = "sink.fsync-period-ms";
+ public static final long LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE = 1000;
+ public static final String CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY =
+ "connector.on-unsupported-event";
+ public static final String SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY =
+ "sink.on-unsupported-event";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL = "fail";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP = "skip";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE =
+ LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL;
+ public static final String CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY =
+ "connector.include-heartbeat";
+ public static final String SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = "sink.include-heartbeat";
+ public static final boolean LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE = false;
+
public static final String CONNECTOR_SERIALIZE_BY_REGION_KEY = "connector.serialize-by-region";
public static final String SINK_SERIALIZE_BY_REGION_KEY = "sink.serialize-by-region";
public static final boolean CONNECTOR_SERIALIZE_BY_REGION_DEFAULT_VALUE = false;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
new file mode 100644
index 0000000000000..6ac9ca23f67bd
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonParseException;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class LogicalBackupArchiveReader {
+
+ private static final Gson GSON = new Gson();
+ private static final Set REPLAYABLE_REQUEST_TYPES =
+ Set.of(
+ PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType(),
+ PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(),
+ PipeRequestType.TRANSFER_PLAN_NODE.getType(),
+ PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(),
+ PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType());
+
+ public List read(final Path source, final boolean allowIncomplete)
+ throws IOException {
+ final Path normalizedSource = source.toAbsolutePath().normalize();
+ final List manifests = discoverManifests(normalizedSource);
+ if (manifests.isEmpty()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E,
+ source));
+ }
+ final List streams = new ArrayList<>();
+ final Set streamIds = new HashSet<>();
+ LogicalBackupManifest archiveIdentity = null;
+ for (final Path manifest : manifests) {
+ final BackupStream stream = readStream(manifest, allowIncomplete);
+ if (archiveIdentity == null) {
+ archiveIdentity = stream.getManifest();
+ } else {
+ validateArchiveIdentity(archiveIdentity, stream.getManifest());
+ }
+ if (!streamIds.add(stream.getManifest().streamId)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE,
+ stream.getManifest().streamId));
+ }
+ streams.add(stream);
+ }
+ streams.sort(
+ Comparator.comparingInt(
+ (BackupStream stream) ->
+ "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1)
+ .thenComparing((BackupStream stream) -> stream.getManifest().streamId));
+ return Collections.unmodifiableList(streams);
+ }
+
+ private static void validateArchiveIdentity(
+ final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException {
+ validateArchiveIdentityField("backupId", expected.backupId, actual.backupId);
+ validateArchiveIdentityField("pipeName", expected.pipeName, actual.pipeName);
+ validateArchiveIdentityField(
+ "pipeCreationTime", expected.pipeCreationTime, actual.pipeCreationTime);
+ validateArchiveIdentityField(
+ "sourceClusterId", expected.sourceClusterId, actual.sourceClusterId);
+ validateArchiveIdentityField("sourceVersion", expected.sourceVersion, actual.sourceVersion);
+ validateArchiveIdentityField(
+ "timestampPrecision", expected.timestampPrecision, actual.timestampPrecision);
+ }
+
+ private static void validateArchiveIdentityField(
+ final String field, final Object expected, final Object actual) throws IOException {
+ if (!java.util.Objects.equals(expected, actual)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete)
+ throws IOException {
+ final LogicalBackupManifest manifest;
+ try {
+ manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath),
+ e);
+ }
+ validateManifest(manifest, manifestPath);
+ if (!allowIncomplete && manifest.skippedEventCount > 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
+ manifestPath));
+ }
+
+ final Path streamDirectory = manifestPath.getParent().toAbsolutePath().normalize();
+ final LogicalBackupSegmentReader segmentReader =
+ new LogicalBackupSegmentReader(manifest.maxRecordBytes);
+ final List records = new ArrayList<>();
+ final Map operationCounts = new LinkedHashMap<>();
+ final Set listedSegments = new HashSet<>();
+ long expectedSequence = -1;
+ long expectedSegmentId = 0;
+
+ for (int index = 0; index < manifest.segments.size(); index++) {
+ final LogicalBackupManifest.Segment segment = manifest.segments.get(index);
+ if (segment == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ "null"));
+ }
+ final Path segmentPath = resolveSegmentPath(streamDirectory, segment.file);
+ if (segmentPath == null
+ || Files.isSymbolicLink(segmentPath)
+ || !Files.isRegularFile(segmentPath, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ segment.file));
+ }
+ listedSegments.add(segmentPath);
+ final boolean lastSegment = index == manifest.segments.size() - 1;
+ final LogicalBackupSegmentReader.ScanResult scan =
+ segmentReader.scan(segmentPath, allowIncomplete && lastSegment);
+ if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF,
+ segmentPath));
+ }
+ if (scan.isSealed()) {
+ validateSegmentManifest(segment, scan, segmentPath);
+ }
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ validateReplayableRequest(record);
+ }
+
+ final List committedRecords =
+ scan.hasOpenEventGroup() ? removeOpenEventGroup(scan.getRecords()) : scan.getRecords();
+ for (final LogicalBackupRecord record : committedRecords) {
+ if (expectedSequence >= 0 && record.getSequence() != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E,
+ manifestPath));
+ }
+ expectedSequence = record.getSequence() + 1;
+ records.add(record);
+ operationCounts.merge(record.getRecordType().name(), 1L, Long::sum);
+ }
+ }
+
+ if (!allowIncomplete) {
+ validateUnlistedSegments(streamDirectory, listedSegments);
+ final long skippedEventCount =
+ operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L);
+ if (skippedEventCount > 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
+ manifestPath));
+ }
+ if (skippedEventCount != manifest.skippedEventCount
+ || !operationCounts.equals(manifest.operationCounts)
+ || (records.isEmpty()
+ ? manifest.firstSequence != -1 || manifest.lastSequence != -1
+ : records.get(0).getSequence() != manifest.firstSequence
+ || records.get(records.size() - 1).getSequence() != manifest.lastSequence)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB,
+ manifestPath));
+ }
+ }
+ return new BackupStream(manifestPath, manifest, records, toEventGroups(records));
+ }
+
+ private static void validateReplayableRequest(final LogicalBackupRecord record)
+ throws IOException {
+ if (record.getRecordType() != LogicalBackupRecordType.PIPE_REQUEST) {
+ return;
+ }
+ if (record.getRequestVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7,
+ record.getRequestVersion()));
+ }
+ if (!REPLAYABLE_REQUEST_TYPES.contains(record.getRequestType())) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38,
+ record.getRequestType()));
+ }
+ }
+
+ private static List discoverManifests(final Path source) throws IOException {
+ if (!Files.isSymbolicLink(source)
+ && Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS)
+ && LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())) {
+ return Collections.singletonList(source);
+ }
+ if (!Files.isDirectory(source)) {
+ return Collections.emptyList();
+ }
+ try (final Stream paths = Files.walk(source)) {
+ return paths
+ .filter(path -> !Files.isSymbolicLink(path))
+ .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS))
+ .filter(
+ path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString()))
+ .sorted()
+ .collect(Collectors.toList());
+ }
+ }
+
+ private static Path resolveSegmentPath(final Path streamDirectory, final String segmentFile) {
+ if (segmentFile == null) {
+ return null;
+ }
+ final Path relativePath;
+ try {
+ relativePath = Path.of(segmentFile);
+ } catch (final RuntimeException e) {
+ return null;
+ }
+ if (relativePath.isAbsolute() || relativePath.getNameCount() != 1) {
+ return null;
+ }
+ final Path segmentPath = streamDirectory.resolve(relativePath).normalize();
+ return segmentPath.startsWith(streamDirectory)
+ && segmentPath.getParent().equals(streamDirectory)
+ ? segmentPath
+ : null;
+ }
+
+ private static void validateManifest(
+ final LogicalBackupManifest manifest, final Path manifestPath) throws IOException {
+ if (manifest == null
+ || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName)
+ || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion)
+ || manifest.streamId == null
+ || manifest.backupId == null
+ || manifest.timestampPrecision == null
+ || manifest.maxRecordBytes <= 0
+ || manifest.maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES
+ || manifest.skippedEventCount < 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath));
+ }
+ if (manifest.segments == null || manifest.segments.isEmpty()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138,
+ manifestPath));
+ }
+ if (manifest.operationCounts == null) {
+ manifest.operationCounts = new LinkedHashMap<>();
+ }
+ }
+
+ private static void validateSegmentManifest(
+ final LogicalBackupManifest.Segment segment,
+ final LogicalBackupSegmentReader.ScanResult scan,
+ final Path segmentPath)
+ throws IOException {
+ final LogicalBackupSegmentReader.Footer footer = scan.getFooter();
+ if (!"SEALED".equals(segment.status)
+ || segment.firstSequence != footer.getFirstSequence()
+ || segment.lastSequence != footer.getLastSequence()
+ || segment.recordCount != footer.getRecordCount()
+ || segment.sizeBytes != scan.getValidLength()
+ || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3,
+ segmentPath));
+ }
+ }
+
+ private static void validateUnlistedSegments(
+ final Path streamDirectory, final Set listedSegments) throws IOException {
+ try (final Stream paths = Files.list(streamDirectory)) {
+ final Path unlisted =
+ paths
+ .filter(Files::isRegularFile)
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .map(path -> path.toAbsolutePath().normalize())
+ .filter(path -> !listedSegments.contains(path))
+ .findFirst()
+ .orElse(null);
+ if (unlisted != null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ unlisted));
+ }
+ }
+ }
+
+ private static List removeOpenEventGroup(
+ final List records) {
+ for (int i = records.size() - 1; i >= 0; i--) {
+ if (records.get(i).getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ return records.subList(0, i);
+ }
+ }
+ return records;
+ }
+
+ private static List toEventGroups(final List records)
+ throws IOException {
+ final List groups = new ArrayList<>();
+ final Set eventGroupIds = new HashSet<>();
+ UUID eventGroupId = null;
+ long firstSequence = -1;
+ String metadata = "";
+ final List requests = new ArrayList<>();
+ for (final LogicalBackupRecord record : records) {
+ if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ eventGroupId = record.getEventGroupId();
+ if (!eventGroupIds.add(eventGroupId)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89,
+ eventGroupId));
+ }
+ firstSequence = record.getSequence();
+ metadata = record.getMetadata();
+ requests.clear();
+ } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) {
+ requests.add(record);
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) {
+ groups.add(
+ new EventGroup(
+ eventGroupId,
+ firstSequence,
+ record.getSequence(),
+ metadata,
+ new ArrayList<>(requests)));
+ eventGroupId = null;
+ requests.clear();
+ }
+ }
+ return Collections.unmodifiableList(groups);
+ }
+
+ public static class BackupStream {
+ private final Path manifestPath;
+ private final LogicalBackupManifest manifest;
+ private final List records;
+ private final List eventGroups;
+
+ private BackupStream(
+ final Path manifestPath,
+ final LogicalBackupManifest manifest,
+ final List records,
+ final List eventGroups) {
+ this.manifestPath = manifestPath;
+ this.manifest = manifest;
+ this.records = Collections.unmodifiableList(records);
+ this.eventGroups = eventGroups;
+ }
+
+ public Path getManifestPath() {
+ return manifestPath;
+ }
+
+ public LogicalBackupManifest getManifest() {
+ return manifest;
+ }
+
+ public List getRecords() {
+ return records;
+ }
+
+ public List getEventGroups() {
+ return eventGroups;
+ }
+ }
+
+ public static class EventGroup {
+ private final UUID eventGroupId;
+ private final long firstSequence;
+ private final long lastSequence;
+ private final String metadata;
+ private final List requests;
+
+ private EventGroup(
+ final UUID eventGroupId,
+ final long firstSequence,
+ final long lastSequence,
+ final String metadata,
+ final List requests) {
+ this.eventGroupId = eventGroupId;
+ this.firstSequence = firstSequence;
+ this.lastSequence = lastSequence;
+ this.metadata = metadata;
+ this.requests = Collections.unmodifiableList(requests);
+ }
+
+ public UUID getEventGroupId() {
+ return eventGroupId;
+ }
+
+ public long getFirstSequence() {
+ return firstSequence;
+ }
+
+ public long getLastSequence() {
+ return lastSequence;
+ }
+
+ public String getMetadata() {
+ return metadata;
+ }
+
+ public List getRequests() {
+ return requests;
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java
new file mode 100644
index 0000000000000..6309595202b52
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+import java.util.zip.CRC32C;
+
+public final class LogicalBackupFormat {
+
+ public static final String FORMAT_NAME = "iotdb-pipe-logical-backup";
+ public static final String FORMAT_VERSION = "1.0";
+ public static final short MAJOR_VERSION = 1;
+ public static final short MINOR_VERSION = 0;
+ public static final long SEGMENT_MAGIC = 0x494F545057414C31L;
+ public static final int RECORD_MAGIC = 0x50575231;
+ public static final int FOOTER_MAGIC = 0x50575346;
+ public static final int SEGMENT_HEADER_SIZE = 32;
+ public static final int RECORD_HEADER_SIZE = 56;
+ public static final int SEGMENT_FOOTER_SIZE = 96;
+ public static final int SHA256_SIZE = 32;
+ public static final int MAX_RECORD_BYTES = 256 * 1024 * 1024;
+ public static final String MANIFEST_FILE_NAME = "manifest.json";
+
+ private LogicalBackupFormat() {}
+
+ public static int crc32c(final byte[] bytes, final int offset, final int length) {
+ final CRC32C crc32c = new CRC32C();
+ crc32c.update(bytes, offset, length);
+ return (int) crc32c.getValue();
+ }
+
+ public static MessageDigest newSha256() {
+ try {
+ return MessageDigest.getInstance("SHA-256");
+ } catch (final NoSuchAlgorithmException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public static String toHex(final byte[] bytes) {
+ return HexFormat.of().formatHex(bytes);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
new file mode 100644
index 0000000000000..3742ca88d8016
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class LogicalBackupManifest {
+
+ public String formatName = LogicalBackupFormat.FORMAT_NAME;
+ public String formatVersion = LogicalBackupFormat.FORMAT_VERSION;
+ public String backupId;
+ public String status;
+ public String pipeName;
+ public long pipeCreationTime;
+ public String sourceClusterId;
+ public String sourceVersion;
+ public String timestampPrecision;
+ public String createdAt;
+ public String closedAt;
+ public String streamId;
+ public int regionId;
+ public String sinkTaskId;
+ public String streamType;
+ public String fsyncPolicy;
+ public long segmentSizeBytes;
+ public int maxRecordBytes;
+ public long firstSequence = -1;
+ public long lastSequence = -1;
+ public long lastDurableSequence = -1;
+ public String lastEventGroupId;
+ public String lastEventDigest;
+ public long lastEventFirstSequence = -1;
+ public long skippedEventCount;
+ public boolean recovered;
+ public Map operationCounts = new LinkedHashMap<>();
+ public List segments = new ArrayList<>();
+
+ public static class Segment {
+ public String file;
+ public long segmentId;
+ public long firstSequence = -1;
+ public long lastSequence = -1;
+ public long recordCount;
+ public long sizeBytes;
+ public String sha256;
+ public String status;
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java
new file mode 100644
index 0000000000000..b779c4fc4aa3e
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.UUID;
+
+public class LogicalBackupRecord {
+
+ private final LogicalBackupRecordType recordType;
+ private final long sequence;
+ private final UUID eventGroupId;
+ private final int operationIndex;
+ private final long eventTime;
+ private final byte requestVersion;
+ private final short requestType;
+ private final String metadata;
+ private final byte[] payload;
+
+ public LogicalBackupRecord(
+ final LogicalBackupRecordType recordType,
+ final long sequence,
+ final UUID eventGroupId,
+ final int operationIndex,
+ final long eventTime,
+ final byte requestVersion,
+ final short requestType,
+ final String metadata,
+ final byte[] payload) {
+ this.recordType = recordType;
+ this.sequence = sequence;
+ this.eventGroupId = eventGroupId;
+ this.operationIndex = operationIndex;
+ this.eventTime = eventTime;
+ this.requestVersion = requestVersion;
+ this.requestType = requestType;
+ this.metadata = metadata;
+ this.payload = payload;
+ }
+
+ public LogicalBackupRecordType getRecordType() {
+ return recordType;
+ }
+
+ public long getSequence() {
+ return sequence;
+ }
+
+ public UUID getEventGroupId() {
+ return eventGroupId;
+ }
+
+ public int getOperationIndex() {
+ return operationIndex;
+ }
+
+ public long getEventTime() {
+ return eventTime;
+ }
+
+ public byte getRequestVersion() {
+ return requestVersion;
+ }
+
+ public short getRequestType() {
+ return requestType;
+ }
+
+ public String getMetadata() {
+ return metadata;
+ }
+
+ public byte[] getPayload() {
+ return payload;
+ }
+
+ public TPipeTransferReq toTPipeTransferReq() {
+ return new TPipeTransferReq()
+ .setVersion(requestVersion)
+ .setType(requestType)
+ .setBody(ByteBuffer.wrap(payload));
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof LogicalBackupRecord)) {
+ return false;
+ }
+ final LogicalBackupRecord that = (LogicalBackupRecord) obj;
+ return sequence == that.sequence
+ && operationIndex == that.operationIndex
+ && eventTime == that.eventTime
+ && requestVersion == that.requestVersion
+ && requestType == that.requestType
+ && recordType == that.recordType
+ && eventGroupId.equals(that.eventGroupId)
+ && metadata.equals(that.metadata)
+ && Arrays.equals(payload, that.payload);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = eventGroupId.hashCode();
+ result = 31 * result + recordType.hashCode();
+ result = 31 * result + Long.hashCode(sequence);
+ result = 31 * result + operationIndex;
+ result = 31 * result + Long.hashCode(eventTime);
+ result = 31 * result + requestVersion;
+ result = 31 * result + requestType;
+ result = 31 * result + metadata.hashCode();
+ return 31 * result + Arrays.hashCode(payload);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java
new file mode 100644
index 0000000000000..e188669548ca6
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum LogicalBackupRecordType {
+ EVENT_BEGIN((byte) 1),
+ PIPE_REQUEST((byte) 2),
+ EVENT_COMMIT((byte) 3),
+ HEARTBEAT((byte) 4),
+ SKIPPED_EVENT((byte) 5),
+ STREAM_END((byte) 6);
+
+ private static final Map TYPE_MAP = new HashMap<>();
+
+ static {
+ for (final LogicalBackupRecordType type : values()) {
+ TYPE_MAP.put(type.code, type);
+ }
+ }
+
+ private final byte code;
+
+ LogicalBackupRecordType(final byte code) {
+ this.code = code;
+ }
+
+ public byte getCode() {
+ return code;
+ }
+
+ public static LogicalBackupRecordType valueOf(final byte code) {
+ return TYPE_MAP.get(code);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java
new file mode 100644
index 0000000000000..e400d6133c27d
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupSegmentReader {
+
+ private final int maxRecordBytes;
+
+ public LogicalBackupSegmentReader(final int maxRecordBytes) {
+ this.maxRecordBytes = maxRecordBytes;
+ }
+
+ public ScanResult scan(final Path segment, final boolean allowIncompleteTail) throws IOException {
+ try (final RandomAccessFile reader = new RandomAccessFile(segment.toFile(), "r")) {
+ final long fileLength = reader.length();
+ if (fileLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw new EOFException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A,
+ segment));
+ }
+
+ final byte[] segmentHeader = readBytes(reader, LogicalBackupFormat.SEGMENT_HEADER_SIZE);
+ validateSegmentHeader(segmentHeader, segment);
+ final ByteBuffer segmentHeaderBuffer = wrap(segmentHeader);
+ segmentHeaderBuffer.position(Long.BYTES + Short.BYTES * 2);
+ final long segmentId = segmentHeaderBuffer.getLong();
+ final long createdAt = segmentHeaderBuffer.getLong();
+
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ digest.update(segmentHeader);
+ final List records = new ArrayList<>();
+ long validLength = LogicalBackupFormat.SEGMENT_HEADER_SIZE;
+ long lastCommittedLength = validLength;
+ boolean openEventGroup = false;
+ UUID openEventGroupId = null;
+ int nextOperationIndex = 0;
+ long expectedSequence = -1;
+ boolean incompleteTail = false;
+ Footer footer = null;
+
+ while (reader.getFilePointer() < fileLength) {
+ final long frameOffset = reader.getFilePointer();
+ if (fileLength - frameOffset < Integer.BYTES) {
+ incompleteTail = true;
+ break;
+ }
+
+ final int magic = reader.readInt();
+ reader.seek(frameOffset);
+ if (magic == LogicalBackupFormat.FOOTER_MAGIC) {
+ if (fileLength - frameOffset < LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ incompleteTail = true;
+ break;
+ }
+ if (fileLength - frameOffset > LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278,
+ segment));
+ }
+ final byte[] footerBytes = readBytes(reader, LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ footer =
+ parseAndValidateFooter(
+ footerBytes,
+ segmentId,
+ createdAt,
+ records.isEmpty() ? -1 : records.get(0).getSequence(),
+ records.isEmpty() ? -1 : records.get(records.size() - 1).getSequence(),
+ frameOffset,
+ records.size(),
+ digest.digest(),
+ segment);
+ validLength = reader.getFilePointer();
+ lastCommittedLength = validLength;
+ break;
+ }
+ if (magic != LogicalBackupFormat.RECORD_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5,
+ frameOffset,
+ segment));
+ }
+ if (fileLength - frameOffset < LogicalBackupFormat.RECORD_HEADER_SIZE) {
+ incompleteTail = true;
+ break;
+ }
+
+ final byte[] header = readBytes(reader, LogicalBackupFormat.RECORD_HEADER_SIZE);
+ final ByteBuffer headerBuffer = wrap(header);
+ headerBuffer.getInt();
+ final byte recordTypeCode = headerBuffer.get();
+ final long sequence = headerBuffer.getLong();
+ final UUID eventGroupId = new UUID(headerBuffer.getLong(), headerBuffer.getLong());
+ final int operationIndex = headerBuffer.getInt();
+ final long eventTime = headerBuffer.getLong();
+ final byte requestVersion = headerBuffer.get();
+ final short requestType = headerBuffer.getShort();
+ final int metadataLength = headerBuffer.getInt();
+ final int payloadLength = headerBuffer.getInt();
+ final int headerCrc = headerBuffer.getInt();
+ if (headerCrc
+ != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79,
+ frameOffset,
+ segment));
+ }
+ if (metadataLength < 0
+ || payloadLength < 0
+ || metadataLength > maxRecordBytes
+ || payloadLength > maxRecordBytes
+ || (long) metadataLength + payloadLength > maxRecordBytes) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E,
+ frameOffset,
+ segment));
+ }
+ final long remainingLength = (long) metadataLength + payloadLength + Integer.BYTES;
+ if (fileLength - reader.getFilePointer() < remainingLength) {
+ incompleteTail = true;
+ break;
+ }
+
+ final byte[] metadataBytes = readBytes(reader, metadataLength);
+ final byte[] payload = readBytes(reader, payloadLength);
+ final int frameCrc = reader.readInt();
+ final byte[] frameContent = new byte[metadataLength + payloadLength];
+ System.arraycopy(metadataBytes, 0, frameContent, 0, metadataLength);
+ System.arraycopy(payload, 0, frameContent, metadataLength, payloadLength);
+ if (frameCrc != LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552,
+ frameOffset,
+ segment));
+ }
+
+ final LogicalBackupRecordType recordType = LogicalBackupRecordType.valueOf(recordTypeCode);
+ if (recordType == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D,
+ recordTypeCode,
+ segment));
+ }
+ if (expectedSequence >= 0 && sequence != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B,
+ frameOffset,
+ segment));
+ }
+ final LogicalBackupRecord record =
+ new LogicalBackupRecord(
+ recordType,
+ sequence,
+ eventGroupId,
+ operationIndex,
+ eventTime,
+ requestVersion,
+ requestType,
+ new String(metadataBytes, StandardCharsets.UTF_8),
+ payload);
+ records.add(record);
+ digest.update(header);
+ digest.update(frameContent);
+ digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(frameCrc).array());
+ validLength = reader.getFilePointer();
+ if (recordType == LogicalBackupRecordType.EVENT_BEGIN) {
+ if (openEventGroup || operationIndex != -1) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A,
+ segment));
+ }
+ openEventGroup = true;
+ openEventGroupId = eventGroupId;
+ nextOperationIndex = 0;
+ } else if (recordType == LogicalBackupRecordType.EVENT_COMMIT) {
+ if (!openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541,
+ segment));
+ }
+ if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F,
+ frameOffset,
+ segment));
+ }
+ openEventGroup = false;
+ openEventGroupId = null;
+ lastCommittedLength = validLength;
+ } else if (recordType == LogicalBackupRecordType.PIPE_REQUEST) {
+ if (!openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F,
+ frameOffset,
+ segment));
+ }
+ if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F,
+ frameOffset,
+ segment));
+ }
+ nextOperationIndex++;
+ } else if (openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B,
+ frameOffset,
+ segment));
+ } else if (!openEventGroup) {
+ lastCommittedLength = validLength;
+ }
+ expectedSequence = sequence + 1;
+ }
+
+ if (incompleteTail && !allowIncompleteTail) {
+ throw new EOFException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7,
+ segment));
+ }
+ if (footer != null && openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8,
+ segment));
+ }
+ return new ScanResult(
+ segmentId,
+ createdAt,
+ records,
+ footer,
+ validLength,
+ lastCommittedLength,
+ incompleteTail,
+ openEventGroup);
+ }
+ }
+
+ private static void validateSegmentHeader(final byte[] header, final Path segment)
+ throws IOException {
+ final ByteBuffer buffer = wrap(header);
+ if (buffer.getLong() != LogicalBackupFormat.SEGMENT_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73,
+ segment));
+ }
+ final short majorVersion = buffer.getShort();
+ buffer.getShort();
+ if (majorVersion != LogicalBackupFormat.MAJOR_VERSION) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB,
+ majorVersion,
+ segment));
+ }
+ buffer.position(LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES);
+ final int headerCrc = buffer.getInt();
+ if (headerCrc
+ != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55,
+ segment));
+ }
+ }
+
+ private static Footer parseAndValidateFooter(
+ final byte[] bytes,
+ final long expectedSegmentId,
+ final long expectedCreatedAt,
+ final long expectedFirstSequence,
+ final long expectedLastSequence,
+ final long expectedValidBytes,
+ final long expectedRecordCount,
+ final byte[] expectedDigest,
+ final Path segment)
+ throws IOException {
+ final ByteBuffer buffer = wrap(bytes);
+ if (buffer.getInt() != LogicalBackupFormat.FOOTER_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4,
+ segment));
+ }
+ final long segmentId = buffer.getLong();
+ final long firstSequence = buffer.getLong();
+ final long lastSequence = buffer.getLong();
+ final long recordCount = buffer.getLong();
+ final long validBytes = buffer.getLong();
+ final long createdAt = buffer.getLong();
+ final long sealedAt = buffer.getLong();
+ final byte[] digest = new byte[LogicalBackupFormat.SHA256_SIZE];
+ buffer.get(digest);
+ final int footerCrc = buffer.getInt();
+ if (segmentId != expectedSegmentId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D,
+ segment));
+ }
+ if (createdAt != expectedCreatedAt
+ || firstSequence != expectedFirstSequence
+ || lastSequence != expectedLastSequence
+ || validBytes != expectedValidBytes
+ || recordCount != expectedRecordCount) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB,
+ segment));
+ }
+ if (!Arrays.equals(digest, expectedDigest)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116,
+ segment));
+ }
+ if (footerCrc
+ != LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91,
+ segment));
+ }
+ return new Footer(
+ segmentId,
+ firstSequence,
+ lastSequence,
+ recordCount,
+ validBytes,
+ createdAt,
+ sealedAt,
+ digest);
+ }
+
+ private static byte[] readBytes(final RandomAccessFile reader, final int length)
+ throws IOException {
+ final byte[] bytes = new byte[length];
+ reader.readFully(bytes);
+ return bytes;
+ }
+
+ private static ByteBuffer wrap(final byte[] bytes) {
+ return ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN);
+ }
+
+ public static class ScanResult {
+ private final long segmentId;
+ private final long createdAt;
+ private final List records;
+ private final Footer footer;
+ private final long validLength;
+ private final long lastCommittedLength;
+ private final boolean incompleteTail;
+ private final boolean openEventGroup;
+
+ private ScanResult(
+ final long segmentId,
+ final long createdAt,
+ final List records,
+ final Footer footer,
+ final long validLength,
+ final long lastCommittedLength,
+ final boolean incompleteTail,
+ final boolean openEventGroup) {
+ this.segmentId = segmentId;
+ this.createdAt = createdAt;
+ this.records = Collections.unmodifiableList(records);
+ this.footer = footer;
+ this.validLength = validLength;
+ this.lastCommittedLength = lastCommittedLength;
+ this.incompleteTail = incompleteTail;
+ this.openEventGroup = openEventGroup;
+ }
+
+ public long getSegmentId() {
+ return segmentId;
+ }
+
+ public long getCreatedAt() {
+ return createdAt;
+ }
+
+ public List getRecords() {
+ return records;
+ }
+
+ public Footer getFooter() {
+ return footer;
+ }
+
+ public long getValidLength() {
+ return validLength;
+ }
+
+ public long getLastCommittedLength() {
+ return lastCommittedLength;
+ }
+
+ public boolean hasIncompleteTail() {
+ return incompleteTail;
+ }
+
+ public boolean hasOpenEventGroup() {
+ return openEventGroup;
+ }
+
+ public boolean isSealed() {
+ return footer != null;
+ }
+ }
+
+ public static class Footer {
+ private final long segmentId;
+ private final long firstSequence;
+ private final long lastSequence;
+ private final long recordCount;
+ private final long validBytes;
+ private final long createdAt;
+ private final long sealedAt;
+ private final byte[] digest;
+
+ private Footer(
+ final long segmentId,
+ final long firstSequence,
+ final long lastSequence,
+ final long recordCount,
+ final long validBytes,
+ final long createdAt,
+ final long sealedAt,
+ final byte[] digest) {
+ this.segmentId = segmentId;
+ this.firstSequence = firstSequence;
+ this.lastSequence = lastSequence;
+ this.recordCount = recordCount;
+ this.validBytes = validBytes;
+ this.createdAt = createdAt;
+ this.sealedAt = sealedAt;
+ this.digest = digest;
+ }
+
+ public long getSegmentId() {
+ return segmentId;
+ }
+
+ public long getFirstSequence() {
+ return firstSequence;
+ }
+
+ public long getLastSequence() {
+ return lastSequence;
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public long getValidBytes() {
+ return validBytes;
+ }
+
+ public long getCreatedAt() {
+ return createdAt;
+ }
+
+ public long getSealedAt() {
+ return sealedAt;
+ }
+
+ public byte[] getDigest() {
+ return digest;
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
new file mode 100644
index 0000000000000..9c8ba6f9a9a63
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
@@ -0,0 +1,1162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+
+public class LogicalBackupWriter implements Closeable {
+
+ public enum FsyncPolicy {
+ ALWAYS,
+ BATCH,
+ PERIODIC,
+ NONE
+ }
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+
+ private final Path directory;
+ private final Path manifestPath;
+ private final Path lockPath;
+ private final int maxRecordBytes;
+ private final long segmentSizeBytes;
+ private final FsyncPolicy fsyncPolicy;
+ private final int fsyncBatchOperations;
+ private final long fsyncPeriodMs;
+ private final LogicalBackupManifest manifest;
+ private final LogicalBackupSegmentReader reader;
+
+ private FileChannel channel;
+ private FileChannel lockChannel;
+ private FileLock directoryLock;
+ private long segmentId;
+ private long segmentCreatedAt;
+ private long segmentFirstSequence = -1;
+ private long segmentLastSequence = -1;
+ private long segmentRecordCount;
+ private MessageDigest segmentDigest;
+ private long nextSequence;
+ private int operationsSinceFsync;
+ private long lastFsyncAt;
+ private boolean closed;
+ private boolean streamEnded;
+ private boolean recoveredDuringOpen;
+
+ public LogicalBackupWriter(
+ final Path directory,
+ final LogicalBackupManifest manifest,
+ final long segmentSizeBytes,
+ final int maxRecordBytes,
+ final FsyncPolicy fsyncPolicy,
+ final int fsyncBatchOperations,
+ final long fsyncPeriodMs,
+ final boolean append)
+ throws IOException {
+ this.directory =
+ Objects.requireNonNull(directory, LogicalBackupMessages.EXCEPTION_DIRECTORY_5F8F22B8)
+ .toAbsolutePath()
+ .normalize();
+ this.manifest =
+ Objects.requireNonNull(manifest, LogicalBackupMessages.EXCEPTION_MANIFEST_7F5CB74A);
+ this.segmentSizeBytes = segmentSizeBytes;
+ this.maxRecordBytes = maxRecordBytes;
+ this.fsyncPolicy =
+ Objects.requireNonNull(fsyncPolicy, LogicalBackupMessages.EXCEPTION_FSYNC_POLICY_6D493614);
+ if (segmentSizeBytes
+ <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ || maxRecordBytes <= 0
+ || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES
+ || fsyncBatchOperations <= 0
+ || fsyncPeriodMs <= 0) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99);
+ }
+ this.fsyncBatchOperations = fsyncBatchOperations;
+ this.fsyncPeriodMs = fsyncPeriodMs;
+ this.manifestPath = this.directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ this.lockPath = this.directory.resolve(".backup.lock");
+ this.reader = new LogicalBackupSegmentReader(maxRecordBytes);
+ try {
+ prepareDirectory(append);
+ acquireLock();
+ if (Files.isSymbolicLink(manifestPath)) {
+ throw symbolicLinkException(manifestPath);
+ }
+ if (Files.exists(manifestPath)) {
+ if (!append) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D,
+ directory));
+ }
+ loadExistingManifest();
+ } else {
+ manifest.status = "WRITING";
+ manifest.createdAt = Instant.now().toString();
+ manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT);
+ manifest.segmentSizeBytes = segmentSizeBytes;
+ manifest.maxRecordBytes = maxRecordBytes;
+ writeManifest();
+ }
+ manifest.status = manifest.recovered ? "RECOVERED" : "WRITING";
+ manifest.closedAt = null;
+ nextSequence = Math.max(0, manifest.lastSequence + 1);
+ openOrCreateSegment();
+ rebuildManifestFromSegments();
+ if (recoveredDuringOpen) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ } else {
+ manifest.lastDurableSequence =
+ Math.max(-1, Math.min(manifest.lastDurableSequence, manifest.lastSequence));
+ }
+ updateActiveSegmentManifest();
+ manifest.status = manifest.recovered ? "RECOVERED" : "WRITING";
+ manifest.closedAt = null;
+ writeManifest();
+ lastFsyncAt = System.currentTimeMillis();
+ } catch (final IOException | RuntimeException e) {
+ if (channel != null) {
+ try {
+ channel.close();
+ } catch (final IOException closeException) {
+ e.addSuppressed(closeException);
+ }
+ channel = null;
+ }
+ releaseLock();
+ throw e;
+ }
+ }
+
+ private void prepareDirectory(final boolean append) throws IOException {
+ rejectSymbolicLinksInDirectoryPath();
+ if (append) {
+ Files.createDirectories(directory);
+ rejectSymbolicLinksInDirectoryPath();
+ return;
+ }
+ final Path parent = directory.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ rejectSymbolicLinksInDirectoryPath();
+ }
+ try {
+ Files.createDirectory(directory);
+ } catch (final FileAlreadyExistsException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D,
+ directory),
+ e);
+ }
+ rejectSymbolicLinksInDirectoryPath();
+ }
+
+ private void rejectSymbolicLinksInDirectoryPath() throws IOException {
+ Path current = directory.getRoot();
+ for (final Path part : directory) {
+ current = current == null ? part : current.resolve(part);
+ if (Files.isSymbolicLink(current)) {
+ throw symbolicLinkException(current);
+ }
+ }
+ }
+
+ private static IOException symbolicLinkException(final Path path) {
+ return new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569,
+ path));
+ }
+
+ public synchronized long writeEvent(
+ final UUID eventGroupId,
+ final long eventTime,
+ final List requests,
+ final String metadata)
+ throws IOException {
+ ensureOpen();
+ if (requests == null || requests.isEmpty()) {
+ throw new IOException(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC);
+ }
+ Objects.requireNonNull(eventGroupId, LogicalBackupMessages.EXCEPTION_EVENT_GROUP_ID_C6F6268A);
+ final String safeMetadata = metadata == null ? "" : metadata;
+ final String eventDigest = computeEventDigest(requests);
+ if (eventGroupId.toString().equals(manifest.lastEventGroupId)) {
+ if (eventDigest.equals(manifest.lastEventDigest)) {
+ updateActiveSegmentManifest();
+ writeManifest();
+ return manifest.lastEventFirstSequence;
+ }
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330,
+ eventGroupId));
+ }
+ rollBeforeEventIfNecessary(requests, safeMetadata);
+ final long eventStartPosition = channel.position();
+ final long firstSequence = nextSequence;
+ try {
+ appendRecord(
+ LogicalBackupRecordType.EVENT_BEGIN, eventGroupId, -1, eventTime, null, safeMetadata);
+ int operationIndex = 0;
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ appendRecord(
+ LogicalBackupRecordType.PIPE_REQUEST,
+ eventGroupId,
+ operationIndex++,
+ eventTime,
+ request,
+ safeMetadata);
+ }
+ appendRecord(
+ LogicalBackupRecordType.EVENT_COMMIT,
+ eventGroupId,
+ operationIndex,
+ eventTime,
+ null,
+ safeMetadata);
+ } catch (final IOException | RuntimeException e) {
+ try {
+ rollbackTo(eventStartPosition);
+ } catch (final IOException rollbackException) {
+ e.addSuppressed(rollbackException);
+ }
+ throw e;
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ }
+ updateActiveSegmentManifest();
+ manifest.lastEventGroupId = eventGroupId.toString();
+ manifest.lastEventDigest = eventDigest;
+ manifest.lastEventFirstSequence = firstSequence;
+ writeManifest();
+ return firstSequence;
+ }
+
+ public synchronized void writeControl(
+ final LogicalBackupRecordType recordType, final long eventTime, final String metadata)
+ throws IOException {
+ ensureOpen();
+ final String safeMetadata = metadata == null ? "" : metadata;
+ rollBeforeControlIfNecessary(safeMetadata);
+ final long recordStartPosition = channel.position();
+ try {
+ appendRecord(recordType, new UUID(0, 0), -1, eventTime, null, safeMetadata);
+ } catch (final IOException | RuntimeException e) {
+ try {
+ rollbackTo(recordStartPosition);
+ } catch (final IOException rollbackException) {
+ e.addSuppressed(rollbackException);
+ }
+ throw e;
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ }
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+
+ public synchronized void recordSkippedEvent(final long eventTime, final String metadata)
+ throws IOException {
+ writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata);
+ manifest.skippedEventCount++;
+ writeManifest();
+ }
+
+ public synchronized LogicalBackupManifest getManifest() {
+ return manifest;
+ }
+
+ public synchronized void heartbeat() throws IOException {
+ ensureOpen();
+ if (!Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)
+ || directoryLock == null
+ || !directoryLock.isValid()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD,
+ directory));
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+ }
+
+ private void appendRecord(
+ final LogicalBackupRecordType recordType,
+ final UUID eventGroupId,
+ final int operationIndex,
+ final long eventTime,
+ final TPipeTransferReq request,
+ final String metadata)
+ throws IOException {
+ final byte[] metadataBytes = metadata.getBytes(StandardCharsets.UTF_8);
+ final byte[] payload =
+ request == null
+ ? new byte[0]
+ : java.util.Arrays.copyOf(request.getBody(), request.getBody().length);
+ if ((long) metadataBytes.length + payload.length > maxRecordBytes) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996);
+ }
+ final long requiredBytes =
+ LogicalBackupFormat.RECORD_HEADER_SIZE
+ + metadataBytes.length
+ + payload.length
+ + Integer.BYTES;
+ final long sequence = nextSequence++;
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.RECORD_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putInt(LogicalBackupFormat.RECORD_MAGIC);
+ header.put(recordType.getCode());
+ header.putLong(sequence);
+ header.putLong(eventGroupId.getMostSignificantBits());
+ header.putLong(eventGroupId.getLeastSignificantBits());
+ header.putInt(operationIndex);
+ header.putLong(eventTime);
+ header.put(request == null ? (byte) 0 : request.getVersion());
+ header.putShort(request == null ? (short) 0 : request.getType());
+ header.putInt(metadataBytes.length);
+ header.putInt(payload.length);
+ header.putInt(0);
+ final byte[] headerBytes = header.array();
+ final int headerCrc =
+ LogicalBackupFormat.crc32c(headerBytes, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4);
+ ByteBuffer.wrap(headerBytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(LogicalBackupFormat.RECORD_HEADER_SIZE - 4, headerCrc);
+ final byte[] frameContent = new byte[metadataBytes.length + payload.length];
+ System.arraycopy(metadataBytes, 0, frameContent, 0, metadataBytes.length);
+ System.arraycopy(payload, 0, frameContent, metadataBytes.length, payload.length);
+ final int frameCrc = LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length);
+
+ writeFully(ByteBuffer.wrap(headerBytes));
+ writeFully(ByteBuffer.wrap(frameContent));
+ writeFully(
+ ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).flip());
+ segmentDigest.update(headerBytes);
+ segmentDigest.update(frameContent);
+ segmentDigest.update(
+ ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).array());
+ if (segmentFirstSequence < 0) {
+ segmentFirstSequence = sequence;
+ }
+ segmentLastSequence = sequence;
+ segmentRecordCount++;
+ manifest.firstSequence = manifest.firstSequence < 0 ? sequence : manifest.firstSequence;
+ manifest.lastSequence = sequence;
+ final String countKey = recordType.name();
+ manifest.operationCounts.put(countKey, manifest.operationCounts.getOrDefault(countKey, 0L) + 1);
+ operationsSinceFsync++;
+ streamEnded = recordType == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private void openOrCreateSegment() throws IOException {
+ if (manifest.segments.isEmpty()) {
+ segmentId = 0;
+ validateSegmentFiles(segmentFileName(segmentId));
+ openNewSegment();
+ return;
+ }
+ final LogicalBackupManifest.Segment last = manifest.segments.get(manifest.segments.size() - 1);
+ final Path segmentPath = resolveExistingSegmentPath(last.file);
+ LogicalBackupSegmentReader.ScanResult scan = scanLastSegmentForAppend(segmentPath, last);
+ if (scan.isSealed()) {
+ if (!"SEALED".equals(last.status)) {
+ manifest.recovered = true;
+ recoveredDuringOpen = true;
+ }
+ segmentId = scan.getSegmentId() + 1;
+ // Validate every existing sealed segment before creating a new file in the backup.
+ rebuildManifestFromSegments();
+ validateSegmentFiles(segmentFileName(segmentId));
+ openNewSegment();
+ return;
+ }
+ validateSegmentFiles(null);
+ if (scan.hasIncompleteTail() || scan.hasOpenEventGroup()) {
+ try (final FileChannel truncateChannel =
+ FileChannel.open(segmentPath, StandardOpenOption.WRITE)) {
+ truncateChannel.truncate(scan.getLastCommittedLength());
+ truncateChannel.force(true);
+ }
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ recoveredDuringOpen = true;
+ scan = reader.scan(segmentPath, false);
+ }
+ if (!scan.getRecords().isEmpty()) {
+ nextSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence() + 1;
+ manifest.lastSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence();
+ }
+ segmentId = scan.getSegmentId();
+ openExistingSegment(segmentPath, scan);
+ }
+
+ private LogicalBackupSegmentReader.ScanResult scanLastSegmentForAppend(
+ final Path segmentPath, final LogicalBackupManifest.Segment segment) throws IOException {
+ try {
+ return reader.scan(segmentPath, true);
+ } catch (final IOException scanException) {
+ final long recoverableLength =
+ "SEALED".equals(segment.status)
+ ? -1
+ : findRecoverableTailStart(segmentPath, segment.sizeBytes);
+ if (recoverableLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw scanException;
+ }
+ try (final FileChannel truncateChannel =
+ FileChannel.open(segmentPath, StandardOpenOption.WRITE)) {
+ truncateChannel.truncate(recoverableLength);
+ truncateChannel.force(true);
+ }
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ recoveredDuringOpen = true;
+ return reader.scan(segmentPath, false);
+ }
+ }
+
+ private static long findRecoverableTailStart(
+ final Path segmentPath, final long manifestSegmentSize) throws IOException {
+ final long fileSize = Files.size(segmentPath);
+ if (fileSize
+ >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ final ByteBuffer magic = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN);
+ try (final FileChannel readChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) {
+ readChannel.position(fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ while (magic.hasRemaining()) {
+ if (readChannel.read(magic) < 0) {
+ break;
+ }
+ }
+ }
+ if (!magic.hasRemaining() && magic.flip().getInt() == LogicalBackupFormat.FOOTER_MAGIC) {
+ return fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ }
+ }
+ if (manifestSegmentSize >= LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && manifestSegmentSize < fileSize
+ && fileSize - manifestSegmentSize <= LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ return manifestSegmentSize;
+ }
+ return -1;
+ }
+
+ private void openExistingSegment(
+ final Path segmentPath, final LogicalBackupSegmentReader.ScanResult scan) throws IOException {
+ channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ);
+ channel.position(channel.size());
+ segmentCreatedAt = scan.getCreatedAt();
+ segmentDigest = LogicalBackupFormat.newSha256();
+ try (final FileChannel digestChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) {
+ final ByteBuffer digestBuffer = ByteBuffer.allocate(64 * 1024);
+ long remaining = scan.getValidLength();
+ while (remaining > 0) {
+ digestBuffer.clear();
+ digestBuffer.limit((int) Math.min(digestBuffer.capacity(), remaining));
+ final int read = digestChannel.read(digestBuffer);
+ if (read < 0) {
+ break;
+ }
+ remaining -= read;
+ segmentDigest.update(digestBuffer.array(), 0, read);
+ }
+ }
+ segmentFirstSequence = -1;
+ segmentLastSequence = -1;
+ segmentRecordCount = 0;
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ segmentFirstSequence = segmentFirstSequence < 0 ? record.getSequence() : segmentFirstSequence;
+ segmentLastSequence = record.getSequence();
+ segmentRecordCount++;
+ }
+ streamEnded =
+ !scan.getRecords().isEmpty()
+ && scan.getRecords().get(scan.getRecords().size() - 1).getRecordType()
+ == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private void openNewSegment() throws IOException {
+ final String fileName = segmentFileName(segmentId);
+ final Path segmentPath = resolveSegmentPath(fileName);
+ final Path temporarySegmentPath = resolveSegmentPath(fileName + ".tmp");
+ if (Files.exists(segmentPath, LinkOption.NOFOLLOW_LINKS)) {
+ adoptOrphanSegment(resolveExistingSegmentPath(fileName), fileName);
+ return;
+ }
+
+ // A temporary segment is never exposed through the manifest and contains only a header, so it
+ // is safe to discard after an interrupted creation attempt.
+ Files.deleteIfExists(temporarySegmentPath);
+ segmentCreatedAt = System.currentTimeMillis();
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putLong(LogicalBackupFormat.SEGMENT_MAGIC);
+ header.putShort(LogicalBackupFormat.MAJOR_VERSION);
+ header.putShort(LogicalBackupFormat.MINOR_VERSION);
+ header.putLong(segmentId);
+ header.putLong(segmentCreatedAt);
+ header.putInt(0);
+ final byte[] bytes = header.array();
+ ByteBuffer.wrap(bytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4,
+ LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4));
+ try (final FileChannel temporaryChannel =
+ FileChannel.open(
+ temporarySegmentPath,
+ StandardOpenOption.CREATE_NEW,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.READ)) {
+ final ByteBuffer headerBuffer = ByteBuffer.wrap(bytes);
+ while (headerBuffer.hasRemaining()) {
+ temporaryChannel.write(headerBuffer);
+ }
+ temporaryChannel.force(true);
+ }
+ try {
+ Files.move(temporarySegmentPath, segmentPath, StandardCopyOption.ATOMIC_MOVE);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(temporarySegmentPath, segmentPath);
+ }
+
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ openExistingSegment(segmentPath, scan);
+ addActiveSegmentToManifest(fileName);
+ }
+
+ private void adoptOrphanSegment(final Path segmentPath, final String fileName)
+ throws IOException {
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ if (scan.getSegmentId() != segmentId
+ || scan.isSealed()
+ || !scan.getRecords().isEmpty()
+ || scan.getValidLength() != LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ segmentPath));
+ }
+ openExistingSegment(segmentPath, scan);
+ addActiveSegmentToManifest(fileName);
+ }
+
+ private void addActiveSegmentToManifest(final String fileName) throws IOException {
+ final LogicalBackupManifest.Segment segment = new LogicalBackupManifest.Segment();
+ segment.file = fileName;
+ segment.segmentId = segmentId;
+ segment.status = "ACTIVE";
+ manifest.segments.add(segment);
+ writeManifest();
+ }
+
+ private void validateSegmentFiles(final String allowedOrphanFile) throws IOException {
+ final Set listedFiles = new HashSet<>();
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ listedFiles.add(segment.file);
+ }
+ try (final DirectoryStream segmentFiles = Files.newDirectoryStream(directory, "*.pwal")) {
+ for (final Path segmentFile : segmentFiles) {
+ final String fileName = segmentFile.getFileName().toString();
+ if (!listedFiles.contains(fileName) && !fileName.equals(allowedOrphanFile)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ segmentFile));
+ }
+ }
+ }
+ }
+
+ private static String segmentFileName(final long id) {
+ return String.format(Locale.ROOT, "segment-%020d.pwal", id);
+ }
+
+ private void sealSegment() throws IOException {
+ if (channel == null || segmentRecordCount == 0) {
+ return;
+ }
+ final byte[] digest = segmentDigest.digest();
+ final ByteBuffer footer =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ footer.putInt(LogicalBackupFormat.FOOTER_MAGIC);
+ footer.putLong(segmentId);
+ footer.putLong(segmentFirstSequence);
+ footer.putLong(segmentLastSequence);
+ footer.putLong(segmentRecordCount);
+ footer.putLong(channel.position());
+ footer.putLong(segmentCreatedAt);
+ footer.putLong(System.currentTimeMillis());
+ footer.put(digest);
+ footer.putInt(0);
+ final byte[] footerBytes = footer.array();
+ ByteBuffer.wrap(footerBytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4,
+ LogicalBackupFormat.crc32c(
+ footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4));
+ writeFully(ByteBuffer.wrap(footerBytes));
+ channel.force(true);
+ final LogicalBackupManifest.Segment segment =
+ manifest.segments.get(manifest.segments.size() - 1);
+ segment.firstSequence = segmentFirstSequence;
+ segment.lastSequence = segmentLastSequence;
+ segment.recordCount = segmentRecordCount;
+ segment.sizeBytes = channel.position();
+ segment.sha256 = LogicalBackupFormat.toHex(digest);
+ segment.status = "SEALED";
+ manifest.lastDurableSequence = segmentLastSequence;
+ channel.close();
+ channel = null;
+ writeManifest();
+ }
+
+ private boolean forceIfNeeded() throws IOException {
+ if (channel == null) {
+ return false;
+ }
+ final long now = System.currentTimeMillis();
+ final boolean shouldForce =
+ fsyncPolicy == FsyncPolicy.ALWAYS
+ || (fsyncPolicy == FsyncPolicy.BATCH && operationsSinceFsync >= fsyncBatchOperations)
+ || (fsyncPolicy == FsyncPolicy.PERIODIC && now - lastFsyncAt >= fsyncPeriodMs);
+ if (shouldForce) {
+ channel.force(true);
+ operationsSinceFsync = 0;
+ lastFsyncAt = now;
+ }
+ return shouldForce;
+ }
+
+ private void writeManifest() throws IOException {
+ final Path temporary =
+ manifestPath.resolveSibling(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp");
+ final byte[] manifestBytes = GSON.toJson(manifest).getBytes(StandardCharsets.UTF_8);
+ Files.deleteIfExists(temporary);
+ try (final FileChannel manifestChannel =
+ FileChannel.open(temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ final ByteBuffer manifestBuffer = ByteBuffer.wrap(manifestBytes);
+ while (manifestBuffer.hasRemaining()) {
+ manifestChannel.write(manifestBuffer);
+ }
+ manifestChannel.force(true);
+ }
+ try {
+ Files.move(
+ temporary,
+ manifestPath,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(temporary, manifestPath, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ private void loadExistingManifest() throws IOException {
+ final LogicalBackupManifest existing;
+ try {
+ existing =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath),
+ e);
+ }
+ if (existing == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath));
+ }
+ if (!LogicalBackupFormat.FORMAT_NAME.equals(existing.formatName)
+ || !LogicalBackupFormat.FORMAT_VERSION.equals(existing.formatVersion)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E,
+ manifestPath));
+ }
+ validateManifestIdentity("backupId", manifest.backupId, existing.backupId);
+ validateManifestIdentity("streamId", manifest.streamId, existing.streamId);
+ validateManifestIdentity("pipeName", manifest.pipeName, existing.pipeName);
+ validateManifestIdentity("streamType", manifest.streamType, existing.streamType);
+ validateManifestIdentity("sourceClusterId", manifest.sourceClusterId, existing.sourceClusterId);
+ validateManifestIdentity("sourceVersion", manifest.sourceVersion, existing.sourceVersion);
+ validateManifestIdentity(
+ "timestampPrecision", manifest.timestampPrecision, existing.timestampPrecision);
+ validateManifestIdentity("sinkTaskId", manifest.sinkTaskId, existing.sinkTaskId);
+ validateManifestIdentity(
+ "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy);
+ validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes);
+ validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes);
+ if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ "pipeCreationTime",
+ manifest.pipeCreationTime,
+ existing.pipeCreationTime));
+ }
+ if (manifest.regionId != existing.regionId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ "regionId",
+ manifest.regionId,
+ existing.regionId));
+ }
+ manifest.status = existing.status;
+ manifest.backupId = existing.backupId;
+ manifest.pipeName = existing.pipeName;
+ manifest.pipeCreationTime = existing.pipeCreationTime;
+ manifest.sourceClusterId = existing.sourceClusterId;
+ manifest.sourceVersion = existing.sourceVersion;
+ manifest.timestampPrecision = existing.timestampPrecision;
+ manifest.streamId = existing.streamId;
+ manifest.regionId = existing.regionId;
+ manifest.sinkTaskId = existing.sinkTaskId;
+ manifest.streamType = existing.streamType;
+ manifest.firstSequence = existing.firstSequence;
+ manifest.lastSequence = existing.lastSequence;
+ manifest.lastDurableSequence = existing.lastDurableSequence;
+ manifest.recovered = existing.recovered;
+ manifest.segments = existing.segments == null ? new ArrayList<>() : existing.segments;
+ manifest.operationCounts =
+ existing.operationCounts == null
+ ? new java.util.LinkedHashMap<>()
+ : existing.operationCounts;
+ manifest.createdAt = existing.createdAt;
+ manifest.closedAt = existing.closedAt;
+ manifest.fsyncPolicy = existing.fsyncPolicy;
+ manifest.segmentSizeBytes = existing.segmentSizeBytes;
+ manifest.maxRecordBytes = existing.maxRecordBytes;
+ manifest.lastEventGroupId = existing.lastEventGroupId;
+ manifest.lastEventDigest = existing.lastEventDigest;
+ manifest.lastEventFirstSequence = existing.lastEventFirstSequence;
+ manifest.skippedEventCount = existing.skippedEventCount;
+ }
+
+ private void validateManifestIdentity(
+ final String field, final String expected, final String actual) throws IOException {
+ if (expected != null && !Objects.equals(expected, actual)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private void validateManifestConfiguration(
+ final String field, final long expected, final long actual) throws IOException {
+ if (expected != actual) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private void acquireLock() throws IOException {
+ try {
+ if (Files.isSymbolicLink(lockPath)) {
+ throw symbolicLinkException(lockPath);
+ }
+ lockChannel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ directoryLock = lockChannel.tryLock();
+ if (directoryLock == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800,
+ directory));
+ }
+ lockChannel.truncate(0);
+ lockChannel.write(
+ StandardCharsets.UTF_8.encode(Long.toString(ProcessHandle.current().pid())));
+ lockChannel.force(true);
+ } catch (final OverlappingFileLockException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800,
+ directory),
+ e);
+ }
+ }
+
+ private void releaseLock() {
+ try {
+ if (directoryLock != null) {
+ directoryLock.release();
+ directoryLock = null;
+ }
+ } catch (final IOException ignored) {
+ // Best effort during failure/close.
+ }
+ try {
+ if (lockChannel != null) {
+ lockChannel.close();
+ lockChannel = null;
+ }
+ } catch (final IOException ignored) {
+ // Best effort during failure/close.
+ }
+ }
+
+ private void writeFully(final ByteBuffer buffer) throws IOException {
+ while (buffer.hasRemaining()) {
+ channel.write(buffer);
+ }
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed || channel == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB);
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ try {
+ if (channel != null) {
+ if (!streamEnded) {
+ writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), "");
+ }
+ sealSegment();
+ }
+ manifest.status = manifest.recovered ? "RECOVERED" : "SEALED";
+ manifest.closedAt = Instant.now().toString();
+ manifest.lastDurableSequence = manifest.lastSequence;
+ writeManifest();
+ } finally {
+ if (channel != null) {
+ channel.close();
+ channel = null;
+ }
+ closed = true;
+ releaseLock();
+ }
+ }
+
+ private void rollBeforeEventIfNecessary(
+ final List requests, final String metadata) throws IOException {
+ final long metadataBytes = metadata.getBytes(StandardCharsets.UTF_8).length;
+ long requiredBytes = frameSize(metadataBytes, 0) * 2;
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ requiredBytes += frameSize(metadataBytes, request.getBody().length);
+ }
+ if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ > segmentSizeBytes) {
+ sealSegment();
+ segmentId++;
+ openNewSegment();
+ }
+ }
+
+ private void rollBeforeControlIfNecessary(final String metadata) throws IOException {
+ final long requiredBytes = frameSize(metadata.getBytes(StandardCharsets.UTF_8).length, 0);
+ if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ > segmentSizeBytes) {
+ sealSegment();
+ segmentId++;
+ openNewSegment();
+ }
+ }
+
+ private static long frameSize(final long metadataBytes, final long payloadBytes) {
+ return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES;
+ }
+
+ private static String computeEventDigest(final List requests)
+ throws IOException {
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ updateEventDigest(digest, request.getVersion(), request.getType(), request.getBody());
+ }
+ return LogicalBackupFormat.toHex(digest.digest());
+ }
+
+ private static void updateEventDigest(
+ final MessageDigest digest,
+ final byte requestVersion,
+ final short requestType,
+ final byte[] body) {
+ digest.update(requestVersion);
+ digest.update(
+ ByteBuffer.allocate(Short.BYTES).order(ByteOrder.BIG_ENDIAN).putShort(requestType).array());
+ digest.update(body);
+ }
+
+ private void rollbackTo(final long position) throws IOException {
+ final Path segmentPath = currentSegmentPath();
+ channel.truncate(position);
+ channel.force(true);
+ channel.close();
+ channel = null;
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ openExistingSegment(segmentPath, scan);
+ rebuildManifestFromSegments();
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ manifest.lastDurableSequence = manifest.lastSequence;
+ operationsSinceFsync = 0;
+ lastFsyncAt = System.currentTimeMillis();
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+
+ private void rebuildManifestFromSegments() throws IOException {
+ manifest.firstSequence = -1;
+ manifest.lastSequence = -1;
+ manifest.lastEventGroupId = null;
+ manifest.lastEventDigest = null;
+ manifest.lastEventFirstSequence = -1;
+ manifest.skippedEventCount = 0;
+ manifest.operationCounts = new LinkedHashMap<>();
+
+ long expectedSequence = -1;
+ long expectedSegmentId = -1;
+ for (int segmentIndex = 0; segmentIndex < manifest.segments.size(); segmentIndex++) {
+ final LogicalBackupManifest.Segment segment = manifest.segments.get(segmentIndex);
+ final Path segmentPath = resolveExistingSegmentPath(segment.file);
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ if (scan.getSegmentId() != segment.segmentId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ if ((expectedSegmentId >= 0 && scan.getSegmentId() != expectedSegmentId)
+ || (!scan.isSealed() && segmentIndex != manifest.segments.size() - 1)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ expectedSegmentId = scan.getSegmentId() + 1;
+ if (scan.isSealed() && "SEALED".equals(segment.status)) {
+ final LogicalBackupSegmentReader.Footer footer = scan.getFooter();
+ if (segment.firstSequence != footer.getFirstSequence()
+ || segment.lastSequence != footer.getLastSequence()
+ || segment.recordCount != footer.getRecordCount()
+ || segment.sizeBytes != scan.getValidLength()
+ || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3,
+ segmentPath));
+ }
+ }
+
+ segment.firstSequence = -1;
+ segment.lastSequence = -1;
+ segment.recordCount = scan.getRecords().size();
+ segment.sizeBytes = scan.getValidLength();
+ segment.status = scan.isSealed() ? "SEALED" : "ACTIVE";
+ segment.sha256 =
+ scan.isSealed() ? LogicalBackupFormat.toHex(scan.getFooter().getDigest()) : null;
+
+ MessageDigest eventDigest = null;
+ UUID eventGroupId = null;
+ long eventFirstSequence = -1;
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ if (expectedSequence >= 0 && record.getSequence() != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149,
+ segmentPath));
+ }
+ expectedSequence = record.getSequence() + 1;
+ segment.firstSequence =
+ segment.firstSequence < 0 ? record.getSequence() : segment.firstSequence;
+ segment.lastSequence = record.getSequence();
+ manifest.firstSequence =
+ manifest.firstSequence < 0 ? record.getSequence() : manifest.firstSequence;
+ manifest.lastSequence = record.getSequence();
+ manifest.operationCounts.merge(record.getRecordType().name(), 1L, Long::sum);
+ if (record.getRecordType() == LogicalBackupRecordType.SKIPPED_EVENT) {
+ manifest.skippedEventCount++;
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ eventDigest = LogicalBackupFormat.newSha256();
+ eventGroupId = record.getEventGroupId();
+ eventFirstSequence = record.getSequence();
+ } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) {
+ updateEventDigest(
+ eventDigest,
+ record.getRequestVersion(),
+ record.getRequestType(),
+ record.getPayload());
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) {
+ manifest.lastEventGroupId = eventGroupId.toString();
+ manifest.lastEventDigest = LogicalBackupFormat.toHex(eventDigest.digest());
+ manifest.lastEventFirstSequence = eventFirstSequence;
+ eventDigest = null;
+ eventGroupId = null;
+ eventFirstSequence = -1;
+ }
+ }
+ }
+ nextSequence = Math.max(0, manifest.lastSequence + 1);
+ streamEnded =
+ manifest.lastSequence >= 0
+ && !manifest.segments.isEmpty()
+ && isLastRecordStreamEnd(reader.scan(currentSegmentPath(), false).getRecords());
+ }
+
+ private static boolean isLastRecordStreamEnd(final List records) {
+ return !records.isEmpty()
+ && records.get(records.size() - 1).getRecordType() == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private Path currentSegmentPath() throws IOException {
+ if (manifest.segments.isEmpty()) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15);
+ }
+ return resolveSegmentPath(manifest.segments.get(manifest.segments.size() - 1).file);
+ }
+
+ private Path resolveSegmentPath(final String file) throws IOException {
+ final Path relativePath;
+ try {
+ relativePath = file == null ? null : Path.of(file);
+ } catch (final RuntimeException e) {
+ throw invalidSegmentPath(file, e);
+ }
+ final Path normalizedDirectory = directory.toAbsolutePath().normalize();
+ if (relativePath == null || relativePath.isAbsolute() || relativePath.getNameCount() != 1) {
+ throw invalidSegmentPath(file, null);
+ }
+ final Path resolved = normalizedDirectory.resolve(relativePath).normalize();
+ if (!resolved.startsWith(normalizedDirectory)
+ || !normalizedDirectory.equals(resolved.getParent())) {
+ throw invalidSegmentPath(file, null);
+ }
+ return resolved;
+ }
+
+ private Path resolveExistingSegmentPath(final String file) throws IOException {
+ final Path resolved = resolveSegmentPath(file);
+ if (Files.isSymbolicLink(resolved)
+ || !Files.isRegularFile(resolved, LinkOption.NOFOLLOW_LINKS)) {
+ throw invalidSegmentPath(file, null);
+ }
+ return resolved;
+ }
+
+ private static IOException invalidSegmentPath(final String file, final Throwable cause) {
+ final IOException exception =
+ new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ file));
+ if (cause != null) {
+ exception.initCause(cause);
+ }
+ return exception;
+ }
+
+ private void updateActiveSegmentManifest() throws IOException {
+ if (channel == null || manifest.segments.isEmpty()) {
+ return;
+ }
+ final LogicalBackupManifest.Segment segment =
+ manifest.segments.get(manifest.segments.size() - 1);
+ segment.firstSequence = segmentFirstSequence;
+ segment.lastSequence = segmentLastSequence;
+ segment.recordCount = segmentRecordCount;
+ segment.sizeBytes = channel.position();
+ segment.status = "ACTIVE";
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
new file mode 100644
index 0000000000000..05f6298e95fac
--- /dev/null
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupArchiveReaderTest {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final int MAX_RECORD_BYTES = 1024;
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data-stream", "data", 4096);
+ writeStream(root.resolve("schema"), "schema-stream", "schema", 4096);
+
+ final List streams = new LogicalBackupArchiveReader().read(root, false);
+
+ Assert.assertEquals(2, streams.size());
+ Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId);
+ Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId);
+ Assert.assertEquals(1, streams.get(0).getEventGroups().size());
+ Assert.assertEquals(4, streams.get(0).getRecords().size());
+ Assert.assertEquals(
+ request(1),
+ streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq());
+ }
+
+ @Test
+ public void testManifestCounterMismatchIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.operationCounts.clear();
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testUnlistedSegmentIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ Files.copy(onlySegment(directory), directory.resolve("unlisted.pwal"));
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ }
+
+ assertReadFails(directory, false);
+ final List streams = new LogicalBackupArchiveReader().read(directory, true);
+ Assert.assertEquals(1, streams.size());
+ Assert.assertEquals(1, streams.get(0).getEventGroups().size());
+ }
+
+ @Test
+ public void testUnsealedNonLastSegmentIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 500);
+ final List segments = segments(directory);
+ Assert.assertTrue(segments.size() > 1);
+ try (final RandomAccessFile file = new RandomAccessFile(segments.get(0).toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ }
+
+ assertReadFails(directory, true);
+ }
+
+ @Test
+ public void testParentSegmentPathIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.segments.get(0).file = "../outside.pwal";
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testSkippedEventRequiresExplicitIncompleteOptIn() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = manifest("stream", "data");
+ try (final LogicalBackupWriter writer = writer(directory, manifest, 4096)) {
+ writer.recordSkippedEvent(1, "unsupported-event");
+ }
+
+ assertReadFails(directory, false);
+ Assert.assertEquals(1, new LogicalBackupArchiveReader().read(directory, true).size());
+
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest persistedManifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ persistedManifest.skippedEventCount = 0;
+ Files.writeString(manifestPath, GSON.toJson(persistedManifest), StandardCharsets.UTF_8);
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testMixedBackupSessionsAreRejected() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data-stream", "data", 4096);
+ writeStream(root.resolve("schema"), "schema-stream", "schema", 4096);
+ final Path manifestPath =
+ root.resolve("schema").resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.backupId = "another-backup";
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(root, false);
+ }
+
+ @Test
+ public void testNonReplayableRequestTypeIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) {
+ writer.writeEvent(
+ UUID.randomUUID(),
+ 1,
+ Collections.singletonList(
+ new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 200)
+ .setBody(ByteBuffer.wrap(new byte[] {1}))),
+ "metadata");
+ }
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testDuplicateEventGroupIdIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventGroupId = UUID.randomUUID();
+ try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) {
+ writer.writeEvent(eventGroupId, 1, Collections.singletonList(request(1)), "metadata");
+ writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "metadata");
+ writer.writeEvent(eventGroupId, 3, Collections.singletonList(request(1)), "metadata");
+ }
+
+ assertReadFails(directory, false);
+ }
+
+ private static void writeStream(
+ final Path directory, final String streamId, final String streamType, final long segmentSize)
+ throws Exception {
+ try (final LogicalBackupWriter writer =
+ writer(directory, manifest(streamId, streamType), segmentSize)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata");
+ if (segmentSize < 1000) {
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "metadata");
+ }
+ }
+ }
+
+ private static LogicalBackupManifest manifest(final String streamId, final String streamType) {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = streamId;
+ manifest.streamType = streamType;
+ manifest.timestampPrecision = "ms";
+ return manifest;
+ }
+
+ private static LogicalBackupWriter writer(
+ final Path directory, final LogicalBackupManifest manifest, final long segmentSize)
+ throws IOException {
+ return new LogicalBackupWriter(
+ directory,
+ manifest,
+ segmentSize,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ false);
+ }
+
+ private static TPipeTransferReq request(final int value) {
+ return request(new byte[] {(byte) value});
+ }
+
+ private static TPipeTransferReq request(final byte[] body) {
+ return new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 10)
+ .setBody(ByteBuffer.wrap(body));
+ }
+
+ private static Path onlySegment(final Path directory) throws IOException {
+ final List segments = segments(directory);
+ Assert.assertEquals(1, segments.size());
+ return segments.get(0);
+ }
+
+ private static List segments(final Path directory) throws IOException {
+ try (final java.util.stream.Stream files = Files.list(directory)) {
+ return files
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .sorted()
+ .collect(java.util.stream.Collectors.toList());
+ }
+ }
+
+ private static void assertReadFails(final Path source, final boolean allowIncomplete)
+ throws IOException {
+ try {
+ new LogicalBackupArchiveReader().read(source, allowIncomplete);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
new file mode 100644
index 0000000000000..ec7d5d63ed9fc
--- /dev/null
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
@@ -0,0 +1,618 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupWriterTest {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final int MAX_RECORD_BYTES = 1024;
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testRoundTripAndDuplicateEvent() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventId = UUID.randomUUID();
+ final TPipeTransferReq request = request((byte) 1, (short) 2, new byte[] {3, 4, 5});
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m"));
+ Assert.assertEquals(
+ 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m"));
+ }
+
+ final LogicalBackupSegmentReader.ScanResult result = scanOnlySegment(directory);
+ Assert.assertTrue(result.isSealed());
+ Assert.assertEquals(4, result.getRecords().size());
+ Assert.assertEquals(
+ LogicalBackupRecordType.EVENT_BEGIN, result.getRecords().get(0).getRecordType());
+ Assert.assertEquals(
+ LogicalBackupRecordType.PIPE_REQUEST, result.getRecords().get(1).getRecordType());
+ Assert.assertEquals(request, result.getRecords().get(1).toTPipeTransferReq());
+ Assert.assertEquals(
+ LogicalBackupRecordType.EVENT_COMMIT, result.getRecords().get(2).getRecordType());
+ Assert.assertEquals(
+ LogicalBackupRecordType.STREAM_END, result.getRecords().get(3).getRecordType());
+ Assert.assertEquals(0, result.getFooter().getFirstSequence());
+ Assert.assertEquals(3, result.getFooter().getLastSequence());
+ }
+
+ @Test
+ public void testDuplicateEventWithDifferentDigestFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventId = UUID.randomUUID();
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(eventId, 1, Collections.singletonList(request(1)), "");
+ try {
+ writer.writeEvent(eventId, 1, Collections.singletonList(request(2)), "");
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains(eventId.toString()));
+ }
+ }
+ }
+
+ @Test
+ public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID firstEventId = UUID.randomUUID();
+ final TPipeTransferReq firstRequest = replayableRequest(1);
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first"));
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first"));
+ Assert.assertEquals(
+ 4,
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(replayableRequest(2)), "second"));
+ }
+
+ final List streams =
+ new LogicalBackupArchiveReader().read(directory, false);
+ Assert.assertEquals(1, streams.size());
+ Assert.assertEquals(2, streams.get(0).getEventGroups().size());
+ }
+
+ @Test
+ public void testOnlineRollbackAfterPartialEvent() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ final List requests =
+ new AbstractList() {
+ private int accesses;
+
+ @Override
+ public TPipeTransferReq get(final int index) {
+ if (++accesses == 6) {
+ throw new IllegalStateException();
+ }
+ return request(index + 1);
+ }
+
+ @Override
+ public int size() {
+ return 2;
+ }
+ };
+ try {
+ writer.writeEvent(UUID.randomUUID(), 1, requests, "");
+ Assert.fail();
+ } catch (final IllegalStateException expected) {
+ // Expected.
+ }
+ Assert.assertEquals(
+ 0,
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(3)), "after-rollback"));
+ }
+
+ final List records = scanOnlySegment(directory).getRecords();
+ Assert.assertEquals(4, records.size());
+ Assert.assertEquals(0, records.get(0).getSequence());
+ Assert.assertEquals("after-rollback", records.get(0).getMetadata());
+ }
+
+ @Test
+ public void testRecoverIncompleteEventAndRebuildCounts() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ final long beginAndRequestLength =
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE + frameSize(0, 0) + frameSize(0, 1);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(beginAndRequestLength);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_BEGIN"));
+ Assert.assertEquals(
+ Long.valueOf(1), writer.getManifest().operationCounts.get("PIPE_REQUEST"));
+ Assert.assertEquals(
+ Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_COMMIT"));
+ }
+ }
+
+ @Test
+ public void testRecoverIncompleteFrame() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 2);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(2, writer.getManifest().lastSequence);
+ }
+ }
+
+ @Test
+ public void testRecoverPartialFooter() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE + 8);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(
+ 4, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ }
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES)
+ .scan(segments(directory).get(segments(directory).size() - 1), false)
+ .isSealed());
+ }
+
+ @Test
+ public void testRecoverCorruptFooterWhenManifestStillMarksSegmentActive() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long lastByteOffset = file.length() - 1;
+ file.seek(lastByteOffset);
+ final int lastByte = file.readUnsignedByte();
+ file.seek(lastByteOffset);
+ file.writeByte(lastByte ^ 1);
+ }
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class);
+ manifest.status = "WRITING";
+ manifest.closedAt = null;
+ manifest.segments.get(0).status = "ACTIVE";
+ manifest.segments.get(0).sizeBytes -= LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ manifest.segments.get(0).sha256 = null;
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ }
+ Assert.assertTrue(scanOnlySegment(directory).isSealed());
+ }
+
+ @Test
+ public void testChecksumDamageFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long payloadOffset =
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ + frameSize("metadata".length(), 0)
+ + LogicalBackupFormat.RECORD_HEADER_SIZE
+ + "metadata".length();
+ file.seek(payloadOffset);
+ file.writeByte(file.readByte() ^ 1);
+ }
+ try {
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("CRC"));
+ }
+ }
+
+ @Test
+ public void testFooterSequenceDamageFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long footerOffset = file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ final byte[] footer = new byte[LogicalBackupFormat.SEGMENT_FOOTER_SIZE];
+ file.seek(footerOffset);
+ file.readFully(footer);
+ ByteBuffer.wrap(footer).order(ByteOrder.BIG_ENDIAN).putLong(Integer.BYTES + Long.BYTES, 99);
+ ByteBuffer.wrap(footer)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES,
+ LogicalBackupFormat.crc32c(
+ footer, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES));
+ file.seek(footerOffset);
+ file.write(footer);
+ }
+ try {
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("metadata"));
+ }
+ }
+
+ @Test
+ public void testSegmentRollover() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 500, false)) {
+ writer.writeEvent(
+ UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m");
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m");
+ }
+ final List segments = segments(directory);
+ Assert.assertEquals(2, segments.size());
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(0), false).isSealed());
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed());
+ }
+
+ @Test
+ public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = manifest();
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 500,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.NONE,
+ 1,
+ 1,
+ false)) {
+ writer.writeEvent(
+ UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m");
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m");
+ Assert.assertEquals(2, writer.getManifest().lastDurableSequence);
+ }
+ }
+
+ @Test
+ public void testControlRecordRollover() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 258, false)) {
+ writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first");
+ writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second");
+ Assert.assertEquals(2, writer.getManifest().segments.size());
+ Assert.assertEquals(1, writer.getManifest().lastSequence);
+ }
+ }
+
+ @Test
+ public void testDirectoryLock() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("locked"));
+ }
+ }
+ }
+
+ @Test
+ public void testFailIfDirectoryExists() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath();
+ try {
+ writer(directory, 4096, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains(directory.toString()));
+ }
+ }
+
+ @Test
+ public void testAppendRejectsConfigurationMismatch() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create the backup.
+ }
+ try {
+ writer(directory, 8192, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("segmentSizeBytes"));
+ }
+ }
+
+ @Test
+ public void testAppendRejectsParentSegmentPath() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ final Path directory = root.resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create the backup.
+ }
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class);
+ final String originalSegment = manifest.segments.get(0).file;
+ Files.copy(directory.resolve(originalSegment), root.resolve("outside.pwal"));
+ manifest.segments.get(0).file = "../outside.pwal";
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("segment path"));
+ }
+
+ manifest.segments.get(0).file = originalSegment;
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, true)) {
+ // A failed append must release the directory lock.
+ }
+ }
+
+ @Test
+ public void testAppendAdoptsOrphanHeaderOnlySegmentAndClearsClosedState() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path orphan = directory.resolve(segmentFileName(1));
+ Files.write(orphan, segmentHeader(1, 123));
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals("WRITING", writer.getManifest().status);
+ Assert.assertNull(writer.getManifest().closedAt);
+ Assert.assertEquals(2, writer.getManifest().segments.size());
+ Assert.assertEquals(
+ 1, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ }
+
+ final LogicalBackupSegmentReader.ScanResult scan =
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(orphan, false);
+ Assert.assertTrue(scan.isSealed());
+ Assert.assertEquals(123, scan.getCreatedAt());
+ }
+
+ @Test
+ public void testAppendRejectsNonEmptyOrphanSegment() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path orphan = directory.resolve(segmentFileName(1));
+ final byte[] invalidOrphan =
+ Arrays.copyOf(segmentHeader(1, 123), LogicalBackupFormat.SEGMENT_HEADER_SIZE + 1);
+ Files.write(orphan, invalidOrphan);
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // The extra byte is an incomplete record tail and must never be adopted.
+ }
+ }
+
+ @Test
+ public void testAppendRejectsUnexpectedUnlistedSegment() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path unexpected = directory.resolve(segmentFileName(99));
+ Files.write(unexpected, segmentHeader(99, 123));
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("Unlisted"));
+ }
+ }
+
+ @Test
+ public void testPeriodicFsyncAdvancesOnHeartbeat() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "stream";
+ manifest.streamType = "data";
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 4096,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.PERIODIC,
+ 1000,
+ 100,
+ false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ Assert.assertEquals(-1, writer.getManifest().lastDurableSequence);
+ Thread.sleep(150);
+ writer.heartbeat();
+ Assert.assertEquals(
+ writer.getManifest().lastSequence, writer.getManifest().lastDurableSequence);
+ }
+ }
+
+ @Test
+ public void testMaxRecordBytesHasHardLimit() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ try {
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 4096,
+ LogicalBackupFormat.MAX_RECORD_BYTES + 1,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+
+ private static LogicalBackupWriter writer(
+ final Path directory, final long segmentSize, final boolean append) throws IOException {
+ return new LogicalBackupWriter(
+ directory,
+ manifest(),
+ segmentSize,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ append);
+ }
+
+ private static LogicalBackupManifest manifest() {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "stream";
+ manifest.streamType = "data";
+ manifest.timestampPrecision = "ms";
+ return manifest;
+ }
+
+ private static TPipeTransferReq request(final int value) {
+ return request(new byte[] {(byte) value});
+ }
+
+ private static TPipeTransferReq request(final byte[] body) {
+ return request((byte) 1, (short) 2, body);
+ }
+
+ private static TPipeTransferReq replayableRequest(final int value) {
+ return request((byte) 1, (short) 10, new byte[] {(byte) value});
+ }
+
+ private static TPipeTransferReq request(final byte version, final short type, final byte[] body) {
+ return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body));
+ }
+
+ private static LogicalBackupSegmentReader.ScanResult scanOnlySegment(final Path directory)
+ throws IOException {
+ return new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(onlySegment(directory), false);
+ }
+
+ private static Path onlySegment(final Path directory) throws IOException {
+ final List segments = segments(directory);
+ Assert.assertEquals(1, segments.size());
+ return segments.get(0);
+ }
+
+ private static List segments(final Path directory) throws IOException {
+ try (final java.util.stream.Stream files = Files.list(directory)) {
+ final Path[] segments =
+ files
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .sorted()
+ .toArray(Path[]::new);
+ return Arrays.asList(segments);
+ }
+ }
+
+ private static long frameSize(final int metadataBytes, final int payloadBytes) {
+ return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES;
+ }
+
+ private static String segmentFileName(final long segmentId) {
+ return String.format(java.util.Locale.ROOT, "segment-%020d.pwal", segmentId);
+ }
+
+ private static byte[] segmentHeader(final long segmentId, final long createdAt) {
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putLong(LogicalBackupFormat.SEGMENT_MAGIC);
+ header.putShort(LogicalBackupFormat.MAJOR_VERSION);
+ header.putShort(LogicalBackupFormat.MINOR_VERSION);
+ header.putLong(segmentId);
+ header.putLong(createdAt);
+ header.putInt(0);
+ final byte[] bytes = header.array();
+ ByteBuffer.wrap(bytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES,
+ LogicalBackupFormat.crc32c(
+ bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES));
+ return bytes;
+ }
+}
diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-pipe-logical-backup.sh
new file mode 100644
index 0000000000000..8a0a4f8a1834b
--- /dev/null
+++ b/scripts/tools/export-pipe-logical-backup.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@"
diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-pipe-logical-backup.sh
new file mode 100644
index 0000000000000..6e4e069210ded
--- /dev/null
+++ b/scripts/tools/import-pipe-logical-backup.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@"
diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/pipe-logical-backup.sh
new file mode 100644
index 0000000000000..a0f6a3529f746
--- /dev/null
+++ b/scripts/tools/pipe-logical-backup.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -n "${IOTDB_INCLUDE}" ] && [ -r "${IOTDB_INCLUDE}" ]; then
+ . "${IOTDB_INCLUDE}"
+fi
+
+if [ -z "${IOTDB_HOME}" ]; then
+ IOTDB_HOME="$(cd "$(dirname "$0")/.."; pwd)"
+ export IOTDB_HOME
+fi
+
+if [ -n "${JAVA_HOME}" ] && [ -x "${JAVA_HOME}/bin/java" ]; then
+ JAVA="${JAVA_HOME}/bin/java"
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for jar in "${IOTDB_HOME}"/lib/*.jar; do
+ CLASSPATH="${CLASSPATH}:${jar}"
+done
+
+exec "${JAVA}" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 \
+ -DIOTDB_HOME="${IOTDB_HOME}" -cp "${CLASSPATH}" \
+ org.apache.iotdb.tool.pipe.PipeLogicalBackupTool "$@"
diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-pipe-logical-backup.bat
new file mode 100644
index 0000000000000..42e1df8afb274
--- /dev/null
+++ b/scripts/tools/windows/export-pipe-logical-backup.bat
@@ -0,0 +1,21 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@call "%~dp0pipe-logical-backup.bat" export %*
+@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-pipe-logical-backup.bat
new file mode 100644
index 0000000000000..0fba332d5a2e5
--- /dev/null
+++ b/scripts/tools/windows/import-pipe-logical-backup.bat
@@ -0,0 +1,21 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@call "%~dp0pipe-logical-backup.bat" import %*
+@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/pipe-logical-backup.bat
new file mode 100644
index 0000000000000..14e3180e46994
--- /dev/null
+++ b/scripts/tools/windows/pipe-logical-backup.bat
@@ -0,0 +1,40 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..\..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED JAVA_HOME goto :no_java
+set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*"
+"%JAVA_HOME%\bin\java" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 ^
+ -DIOTDB_HOME="%IOTDB_HOME%" -cp %CLASSPATH% ^
+ org.apache.iotdb.tool.pipe.PipeLogicalBackupTool %*
+set ret_code=%ERRORLEVEL%
+goto :finally
+
+:no_java
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+
+:finally
+endlocal & exit /B %ret_code%