From c5c6829b73d8a7d017fdea893a056a92309d32c8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:39:44 +0800 Subject: [PATCH 1/3] Add Pipe logical backup sink and restore tooling --- distribution/src/assembly/cli.xml | 2 + iotdb-client/cli/pom.xml | 5 + .../apache/iotdb/cli/i18n/CliMessages.java | 57 + .../apache/iotdb/cli/i18n/CliMessages.java | 57 + .../tool/pipe/PipeLogicalBackupTool.java | 817 ++++++++++++ .../tool/pipe/PipeLogicalBackupToolTest.java | 166 +++ .../iotdb/db/i18n/DataNodePipeMessages.java | 11 + .../iotdb/db/i18n/DataNodePipeMessages.java | 11 + .../PipeDataRegionSinkConstructor.java | 7 + .../PipeSchemaRegionSinkConstructor.java | 7 + .../logicalbackup/LogicalBackupSink.java | 591 +++++++++ iotdb-core/node-commons/pom.xml | 4 + .../commons/i18n/LogicalBackupMessages.java | 151 +++ .../commons/i18n/LogicalBackupMessages.java | 151 +++ .../plugin/builtin/BuiltinPipePlugin.java | 4 + .../sink/logicalbackup/LogicalBackupSink.java | 24 + .../config/constant/PipeSinkConstant.java | 49 + .../LogicalBackupArchiveReader.java | 481 +++++++ .../logicalbackup/LogicalBackupFormat.java | 62 + .../logicalbackup/LogicalBackupManifest.java | 68 + .../logicalbackup/LogicalBackupRecord.java | 136 ++ .../LogicalBackupRecordType.java | 54 + .../LogicalBackupSegmentReader.java | 541 ++++++++ .../logicalbackup/LogicalBackupWriter.java | 1162 +++++++++++++++++ .../LogicalBackupArchiveReaderTest.java | 272 ++++ .../LogicalBackupWriterTest.java | 588 +++++++++ scripts/tools/export-pipe-logical-backup.sh | 21 + scripts/tools/import-pipe-logical-backup.sh | 21 + scripts/tools/pipe-logical-backup.sh | 43 + .../windows/export-pipe-logical-backup.bat | 21 + .../windows/import-pipe-logical-backup.bat | 21 + scripts/tools/windows/pipe-logical-backup.bat | 40 + 32 files changed, 5645 insertions(+) create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java create mode 100644 iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java create mode 100644 iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java create mode 100644 scripts/tools/export-pipe-logical-backup.sh create mode 100644 scripts/tools/import-pipe-logical-backup.sh create mode 100644 scripts/tools/pipe-logical-backup.sh create mode 100644 scripts/tools/windows/export-pipe-logical-backup.bat create mode 100644 scripts/tools/windows/import-pipe-logical-backup.bat create mode 100644 scripts/tools/windows/pipe-logical-backup.bat diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml index 6f4eed7f7c259..7e60322b49870 100644 --- a/distribution/src/assembly/cli.xml +++ b/distribution/src/assembly/cli.xml @@ -54,9 +54,11 @@ *data.* *schema.* *tsfile.* + *logical-backup* **/*data.* **/*schema.* **/*tsfile.* + **/*logical-backup* 0755 diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index 5de247a084a09..c9824ef4aec11 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -124,6 +124,10 @@ commons-cli commons-cli + + com.google.code.gson + gson + org.jline jline @@ -204,6 +208,7 @@ org.apache.iotdb:iotdb-server org/apache/iotdb/db/utils/* + org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.class diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 0329812f2d4d3..77888e65768a0 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -96,4 +96,61 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "Input time format {} is not supported, "; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n"; + // Pipe logical backup tool + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "Logical backup command failed: %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "Unknown logical backup command: %s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "--output is required for logical backup export"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "Stream %s: %d records, %d committed event groups"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "Logical backup verified: %d streams, %d records, %d committed event groups"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "Logical backup export target must not be inside source"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "Logical backup exported from %s to %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "Dry run completed; no data was written"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "Logical backup import completed: %d event groups, checkpoint %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "Logical backup option --%s is required"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "Logical backup password environment variable is not set"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "Logical backup handshake failed: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "Logical backup request type %d failed: %s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = + "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; + public static final String + LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = + "pipe-logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "Logical backup archive entry is unsafe: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "Logical backup archive exceeds safety limit"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "Logical backup export source must be a directory or manifest: %s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "Specify exactly one of --password-stdin and --password-env for logical backup import"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "No password was read from standard input"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "Logical backup checkpoint does not match the source or target: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "Logical backup checkpoint is invalid: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "Unsupported logical backup export format: %s"; + } diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 61a14d94d5884..530b504b29ce0 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -92,4 +92,61 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "不支持输入时间格式 {}，"; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "请输入类似 yyyy-MM-dd\\ HH:mm:ss.SSS 或 yyyy-MM-dd'T'HH:mm:ss.SSS 的格式%n"; + // Pipe 逻辑备份工具 + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "逻辑备份命令执行失败：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "未知的逻辑备份命令：%s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "逻辑备份导出必须指定 --output"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "流 %s：%d 条记录，%d 个已提交事件组"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "逻辑备份校验通过：%d 个流，%d 条记录，%d 个已提交事件组"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "逻辑备份导出目标不能位于源目录内"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "逻辑备份已从 %s 导出到 %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "试运行完成，未写入数据"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "逻辑备份导入完成：%d 个事件组，checkpoint 为 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "逻辑备份选项 --%s 为必填项"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "未设置逻辑备份密码环境变量"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "逻辑备份握手失败：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "逻辑备份请求类型 %d 执行失败：%s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = + "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; + public static final String + LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = + "pipe-logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "逻辑备份归档项路径不安全：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "逻辑备份归档超过安全限制"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "逻辑备份导出源必须是目录或 manifest：%s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "逻辑备份导入必须且只能指定 --password-stdin 或 --password-env 中的一项"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "未能从标准输入读取密码"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "逻辑备份 checkpoint 与源备份或目标实例不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "逻辑备份 checkpoint 无效：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "不支持的逻辑备份导出格式：%s"; + } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java new file mode 100644 index 0000000000000..5d80243f3adf5 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -0,0 +1,817 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.cli.i18n.CliMessages; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.thrift.TException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */ +public final class PipeLogicalBackupTool { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final String COMMAND_INSPECT = "inspect"; + private static final String COMMAND_VERIFY = "verify"; + private static final String COMMAND_EXPORT = "export"; + private static final String COMMAND_IMPORT = "import"; + private static final String COMMAND_RESTORE = "restore"; + private static final String COMMAND_STATS = "stats"; + private static final String OPTION_INPUT = "input"; + private static final String OPTION_OUTPUT = "output"; + private static final String OPTION_RESUME = "resume"; + private static final String OPTION_FORMAT = "format"; + private static final String OPTION_DEEP = "deep"; + // Compatibility aliases for the initial command-line prototype. + private static final String OPTION_SOURCE = "source"; + private static final String OPTION_TARGET = "target"; + private static final String OPTION_HOST = "host"; + private static final String OPTION_PORT = "port"; + private static final String OPTION_USER = "user"; + private static final String OPTION_PASSWORD_STDIN = "password-stdin"; + private static final String OPTION_PASSWORD_ENV = "password-env"; + private static final String OPTION_CHECKPOINT = "checkpoint"; + private static final String OPTION_DRY_RUN = "dry-run"; + private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete"; + private static final String OPTION_JSON = "json"; + private static final int MAX_ARCHIVE_ENTRIES = 1_000_000; + private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024; + private static final String CHECKPOINT_FORMAT_NAME = + "iotdb-pipe-logical-backup-import-checkpoint"; + private static final String CHECKPOINT_FORMAT_VERSION = "1.0"; + + private PipeLogicalBackupTool() {} + + public static void main(final String[] args) { + int exitCode = 0; + try { + exitCode = run(args); + } catch (final Exception e) { + System.err.println( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0, e.getMessage())); + exitCode = 1; + } + if (exitCode != 0) { + System.exit(exitCode); + } + } + + static int run(final String[] args) throws Exception { + if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) { + printUsage(); + return 0; + } + final String command = args[0].toLowerCase(java.util.Locale.ROOT); + final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length)); + switch (command) { + case COMMAND_INSPECT: + return inspect(line); + case COMMAND_VERIFY: + return verify(line); + case COMMAND_EXPORT: + return export(line); + case COMMAND_IMPORT: + case COMMAND_RESTORE: + return importBackup(line); + case COMMAND_STATS: + return inspect(line); + default: + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command)); + } + } + + private static CommandLine parse(final String command, final String[] args) + throws ParseException { + final Options options = new Options(); + addOption(options, OPTION_INPUT, true, false); + addOption(options, OPTION_OUTPUT, true, false); + addOption(options, OPTION_RESUME, true, false); + addOption(options, OPTION_FORMAT, true, false); + addOption(options, OPTION_DEEP, false, false); + addOption(options, OPTION_SOURCE, true, false); + addOption(options, OPTION_TARGET, true, false); + addOption(options, OPTION_HOST, true, false); + addOption(options, OPTION_PORT, true, false); + addOption(options, OPTION_USER, true, false); + addOption(options, OPTION_PASSWORD_STDIN, false, false); + addOption(options, OPTION_PASSWORD_ENV, true, false); + addOption(options, OPTION_CHECKPOINT, true, false); + addOption(options, OPTION_DRY_RUN, false, false); + addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); + addOption(options, OPTION_JSON, false, false); + final CommandLine line = new DefaultParser().parse(options, args); + final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE); + if (input == null || input.isBlank()) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET); + if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) { + throw new ParseException( + CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1); + } + final String format = line.getOptionValue(OPTION_FORMAT, "binary"); + if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format)); + } + return line; + } + + private static void addOption( + final Options options, final String name, final boolean hasArg, final boolean required) { + options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build()); + } + + private static int inspect(final CommandLine line) throws IOException { + final List streams = read(line); + if (line.hasOption(OPTION_JSON)) { + final List> output = new ArrayList<>(); + for (final BackupStream stream : streams) { + final Map item = new HashMap<>(); + item.put("manifest", stream.getManifest()); + item.put("recordCount", stream.getRecords().size()); + item.put("eventGroupCount", stream.getEventGroups().size()); + output.add(item); + } + System.out.println(GSON.toJson(output)); + return 0; + } + for (final BackupStream stream : streams) { + System.out.println( + String.format( + CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, + stream.getManifest().streamId, + stream.getRecords().size(), + stream.getEventGroups().size())); + } + return 0; + } + + private static int verify(final CommandLine line) throws IOException { + final List streams = read(line); + long recordCount = 0; + long eventCount = 0; + for (final BackupStream stream : streams) { + recordCount += stream.getRecords().size(); + eventCount += stream.getEventGroups().size(); + } + System.out.println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B, + streams.size(), + recordCount, + eventCount)); + return 0; + } + + private static int export(final CommandLine line) throws IOException { + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + final Path target = + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); + final List streams = read(line); + final Path root; + if (Files.isDirectory(source)) { + root = source; + } else if (Files.isRegularFile(source) + && "manifest.json".equals(source.getFileName().toString())) { + root = source.getParent(); + } else { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + source)); + } + if (target.startsWith(root)) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994); + } + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new java.nio.file.FileAlreadyExistsException(target.toString()); + } + final Path parent = target.toAbsolutePath().normalize().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp"); + boolean published = false; + try { + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) { + for (final Path file : exportFiles(root, streams)) { + final String entryName = root.relativize(file).toString().replace('\\', '/'); + zip.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zip); + zip.closeEntry(); + } + } + try (final java.nio.channels.FileChannel archiveChannel = + java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) { + archiveChannel.force(true); + } + try { + Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, target); + } + published = true; + } finally { + if (!published) { + Files.deleteIfExists(temporary); + } + } + System.out.println( + String.format( + CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target)); + return 0; + } + + private static List exportFiles(final Path root, final List streams) + throws IOException { + final Set files = new LinkedHashSet<>(); + for (final BackupStream stream : streams) { + final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize(); + if (!manifestPath.startsWith(root)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + root)); + } + files.add(manifestPath); + final Path streamDirectory = manifestPath.getParent(); + for (final LogicalBackupManifest.Segment segment : stream.getManifest().segments) { + files.add(streamDirectory.resolve(segment.file).normalize()); + } + } + final List sortedFiles = new ArrayList<>(files); + sortedFiles.sort(Comparator.comparing(Path::toString)); + return sortedFiles; + } + + private static int importBackup(final CommandLine line) throws Exception { + final List streams = read(line); + final boolean dryRun = line.hasOption(OPTION_DRY_RUN); + if (dryRun) { + System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B); + for (final BackupStream stream : streams) { + System.out.println( + String.format( + CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, + stream.getManifest().streamId, + stream.getRecords().size(), + stream.getEventGroups().size())); + } + return 0; + } + final String host = required(line, OPTION_HOST); + final int port = Integer.parseInt(required(line, OPTION_PORT)); + final String user = line.getOptionValue(OPTION_USER, "root"); + final String password = password(line); + final Path checkpoint = checkpointPath(line); + final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user); + try (final IoTDBSyncClient client = + new IoTDBSyncClient( + new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(), + host, + port, + false, + "", + "")) { + handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + long importedGroups = 0; + for (final BackupStream stream : streams) { + final String streamId = stream.getManifest().streamId; + final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); + for (final EventGroup group : stream.getEventGroups()) { + if (group.getLastSequence() <= lastApplied) { + continue; + } + for (final LogicalBackupRecord record : group.getRequests()) { + transfer(client, record.toTPipeTransferReq()); + } + checkpointState.appliedSequences.put(streamId, group.getLastSequence()); + writeCheckpoint(checkpoint, checkpointState); + importedGroups++; + } + } + writeCheckpoint(checkpoint, checkpointState); + System.out.println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D, + importedGroups, + checkpoint)); + } + return 0; + } + + private static List read(final CommandLine line) throws IOException { + final String source = input(line); + try (final BackupInput input = BackupInput.open(Path.of(source))) { + return new LogicalBackupArchiveReader() + .read(input.getPath(), line.hasOption(OPTION_ALLOW_INCOMPLETE)); + } + } + + private static String input(final CommandLine line) throws IOException { + final String value = optionValue(line, OPTION_INPUT, OPTION_SOURCE); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + return value; + } + + private static String optionValue( + final CommandLine line, final String preferredOption, final String compatibilityOption) { + return line.hasOption(preferredOption) + ? line.getOptionValue(preferredOption) + : line.getOptionValue(compatibilityOption); + } + + private static String required(final CommandLine line, final String option) throws IOException { + final String value = line.getOptionValue(option); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, option)); + } + return value; + } + + private static String password(final CommandLine line) throws IOException { + final boolean fromStandardInput = line.hasOption(OPTION_PASSWORD_STDIN); + final boolean fromEnvironment = line.hasOption(OPTION_PASSWORD_ENV); + if (fromStandardInput == fromEnvironment) { + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + if (fromStandardInput) { + final BufferedReader reader = + new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + final String value = reader.readLine(); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E); + } + return value; + } + if (fromEnvironment) { + final String value = System.getenv(line.getOptionValue(OPTION_PASSWORD_ENV)); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2); + } + return value; + } + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + + private static void handshake( + final IoTDBSyncClient client, + final String timestampPrecision, + final String user, + final String password) + throws IOException, TException { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"); + final TPipeTransferResp response = + client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params)); + if (response == null + || response.getStatus() == null + || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697, + response == null ? "null" : response.getStatus())); + } + } + + private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) + throws TException, IOException { + final TPipeTransferResp response = client.pipeTransfer(request); + if (response == null + || response.getStatus() == null + || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && response.getStatus().getCode() + != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, + request.getType(), + response == null ? "null" : response.getStatus())); + } + } + + private static Path checkpointPath(final CommandLine line) throws IOException { + final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT); + if (configuredCheckpoint != null) { + return Path.of(configuredCheckpoint); + } + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + return source.resolveSibling(source.getFileName() + ".import.checkpoint.json"); + } + + private static ImportCheckpoint readCheckpoint( + final Path checkpoint, + final List streams, + final String host, + final int port, + final String user) + throws IOException { + final Map sourceStreams = sourceStreamIdentities(streams); + if (!Files.exists(checkpoint)) { + final ImportCheckpoint state = new ImportCheckpoint(); + state.formatName = CHECKPOINT_FORMAT_NAME; + state.formatVersion = CHECKPOINT_FORMAT_VERSION; + state.sourceStreams = sourceStreams; + state.targetHost = host; + state.targetPort = port; + state.targetUser = user; + return state; + } + final ImportCheckpoint state; + try { + state = + GSON.fromJson( + Files.readString(checkpoint, StandardCharsets.UTF_8), ImportCheckpoint.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint), + e); + } + if (state == null + || !CHECKPOINT_FORMAT_NAME.equals(state.formatName) + || !CHECKPOINT_FORMAT_VERSION.equals(state.formatVersion) + || state.sourceStreams == null + || state.appliedSequences == null) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); + } + if (!sourceStreams.equals(state.sourceStreams) + || !host.equals(state.targetHost) + || port != state.targetPort + || !user.equals(state.targetUser) + || !checkpointSequencesAreValid(streams, state.appliedSequences)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D, + checkpoint)); + } + return state; + } + + static Map sourceStreamIdentities(final List streams) { + final Map identities = new LinkedHashMap<>(); + for (final BackupStream stream : streams) { + final LogicalBackupManifest manifest = stream.getManifest(); + final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, manifest.backupId); + updateDigest(digest, manifest.streamId); + updateDigest(digest, manifest.pipeName); + updateDigest(digest, manifest.pipeCreationTime); + updateDigest(digest, manifest.sourceClusterId); + updateDigest(digest, manifest.regionId); + updateDigest(digest, manifest.streamType); + updateDigest(digest, manifest.segments.size()); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + updateDigest(digest, segment.file); + updateDigest(digest, segment.segmentId); + updateDigest(digest, segment.firstSequence); + updateDigest(digest, segment.lastSequence); + updateDigest(digest, segment.recordCount); + updateDigest(digest, segment.sizeBytes); + updateDigest(digest, segment.sha256); + updateDigest(digest, segment.status); + } + updateDigest(digest, stream.getRecords().size()); + for (final LogicalBackupRecord record : stream.getRecords()) { + digest.update(record.getRecordType().getCode()); + digest.update( + ByteBuffer.allocate(Long.BYTES * 4 + Integer.BYTES + Byte.BYTES + Short.BYTES) + .putLong(record.getSequence()) + .putLong(record.getEventGroupId().getMostSignificantBits()) + .putLong(record.getEventGroupId().getLeastSignificantBits()) + .putInt(record.getOperationIndex()) + .putLong(record.getEventTime()) + .put(record.getRequestVersion()) + .putShort(record.getRequestType()) + .array()); + updateDigest(digest, record.getMetadata()); + updateDigest(digest, record.getPayload()); + } + identities.put(manifest.streamId, LogicalBackupFormat.toHex(digest.digest())); + } + return identities; + } + + private static void updateDigest(final MessageDigest digest, final String value) { + if (value == null) { + digest.update((byte) 0); + return; + } + digest.update((byte) 1); + updateDigest(digest, value.getBytes(StandardCharsets.UTF_8)); + } + + private static void updateDigest(final MessageDigest digest, final byte[] value) { + updateDigest(digest, value.length); + digest.update(value); + } + + private static void updateDigest(final MessageDigest digest, final int value) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } + + private static void updateDigest(final MessageDigest digest, final long value) { + digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array()); + } + + private static boolean checkpointSequencesAreValid( + final List streams, final Map appliedSequences) { + if (appliedSequences.size() > streams.size()) { + return false; + } + for (final BackupStream stream : streams) { + if (!appliedSequences.containsKey(stream.getManifest().streamId)) { + continue; + } + final Long applied = appliedSequences.get(stream.getManifest().streamId); + if (applied == null) { + return false; + } + if (applied == -1) { + continue; + } + if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) { + return false; + } + } + return appliedSequences.keySet().stream() + .allMatch( + streamId -> + streams.stream() + .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + } + + private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) + throws IOException { + final Path parent = checkpoint.toAbsolutePath().normalize().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp"); + Files.deleteIfExists(temporary); + final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState)); + try (final java.nio.channels.FileChannel checkpointChannel = + java.nio.channels.FileChannel.open( + temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + while (checkpointBytes.hasRemaining()) { + checkpointChannel.write(checkpointBytes); + } + checkpointChannel.force(true); + } + try { + Files.move( + temporary, + checkpoint, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static String getClusterId() { + final byte[] bytes = new byte[16]; + new SecureRandom().nextBytes(bytes); + return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes); + } + + private static void printUsage() { + final HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( + new PrintWriter(System.out), + 120, + CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2, + CliMessages + .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E, + optionsForHelp(), + 2, + 2, + "", + true); + } + + private static Options optionsForHelp() { + final Options options = new Options(); + addOption(options, OPTION_INPUT, true, false); + addOption(options, OPTION_OUTPUT, true, false); + addOption(options, OPTION_RESUME, true, false); + addOption(options, OPTION_FORMAT, true, false); + addOption(options, OPTION_DEEP, false, false); + addOption(options, OPTION_HOST, true, false); + addOption(options, OPTION_PORT, true, false); + addOption(options, OPTION_USER, true, false); + addOption(options, OPTION_PASSWORD_STDIN, false, false); + addOption(options, OPTION_PASSWORD_ENV, true, false); + addOption(options, OPTION_CHECKPOINT, true, false); + addOption(options, OPTION_DRY_RUN, false, false); + addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); + addOption(options, OPTION_JSON, false, false); + return options; + } + + private static class ImportCheckpoint { + private String formatName; + private String formatVersion; + private Map sourceStreams = new LinkedHashMap<>(); + private String targetHost; + private int targetPort; + private String targetUser; + private Map appliedSequences = new LinkedHashMap<>(); + } + + private static class BackupInput implements AutoCloseable { + private final Path path; + private final Path temporaryDirectory; + + private BackupInput(final Path path, final Path temporaryDirectory) { + this.path = path; + this.temporaryDirectory = temporaryDirectory; + } + + private static BackupInput open(final Path source) throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + if (!Files.isRegularFile(normalizedSource) + || "manifest.json".equals(normalizedSource.getFileName().toString())) { + return new BackupInput(normalizedSource, null); + } + final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-"); + try { + extract(normalizedSource, temporaryDirectory); + return new BackupInput(temporaryDirectory, temporaryDirectory); + } catch (final IOException | RuntimeException e) { + deleteRecursively(temporaryDirectory); + throw e; + } + } + + private static void extract(final Path archive, final Path target) throws IOException { + int entryCount = 0; + long totalBytes = 0; + final byte[] buffer = new byte[64 * 1024]; + try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (++entryCount > MAX_ARCHIVE_ENTRIES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + final Path output = target.resolve(entry.getName()).normalize(); + if (!output.startsWith(target)) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152, + entry.getName())); + } + if (entry.isDirectory()) { + Files.createDirectories(output); + } else { + final Path parent = output.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (final OutputStream stream = + Files.newOutputStream( + output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + int length; + while ((length = zip.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + totalBytes += length; + if (totalBytes > MAX_ARCHIVE_BYTES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + stream.write(buffer, 0, length); + } + } + } + zip.closeEntry(); + } + } + } + + private Path getPath() { + return path; + } + + @Override + public void close() throws IOException { + if (temporaryDirectory != null) { + deleteRecursively(temporaryDirectory); + } + } + + private static void deleteRecursively(final Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (final java.util.stream.Stream paths = Files.walk(directory)) { + for (final Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java new file mode 100644 index 0000000000000..0fe45b22d8970 --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.commons.cli.ParseException; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +public class PipeLogicalBackupToolTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testDirectoryExportZipVerifyAndDryRun() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source); + + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run(new String[] {"verify", "--input", source.toString(), "--deep"})); + + final Path archive = root.resolve("backup.zip"); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] { + "export", + "--input", + source.toString(), + "--output", + archive.toString(), + "--format=binary" + })); + Assert.assertTrue(Files.isRegularFile(archive)); + try (final ZipFile zip = new ZipFile(archive.toFile())) { + Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); + Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal")); + Assert.assertNull(zip.getEntry(".backup.lock")); + Assert.assertEquals(2, zip.size()); + } + + Assert.assertEquals( + 0, PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"import", "--input", archive.toString(), "--dry-run"})); + + final byte[] originalArchive = Files.readAllBytes(archive); + try { + PipeLogicalBackupTool.run( + new String[] {"export", "--input", source.toString(), "--output", archive.toString()}); + Assert.fail(); + } catch (final java.nio.file.FileAlreadyExistsException expected) { + Assert.assertArrayEquals(originalArchive, Files.readAllBytes(archive)); + } + } + + @Test + public void testCheckpointSourceIdentityIncludesContent() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Map before = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + streams.get(0).getManifest().segments.get(0).sha256 = "different-content"; + final Map after = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + Assert.assertNotEquals(before, after); + } + + @Test + public void testZipSlipIsRejected() throws Exception { + final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip"); + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive))) { + zip.putNextEntry(new ZipEntry("../outside")); + zip.write(1); + zip.closeEntry(); + } + + try { + PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()}); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + @Test + public void testPlaintextPasswordOptionIsRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + try { + PipeLogicalBackupTool.run( + new String[] { + "import", "--source", source.toString(), "--dry-run", "--password", "secret" + }); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + + private static void writeBackup(final Path directory) throws Exception { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "data-1"; + manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Collections.singletonList( + new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(new byte[] {3}))), + "metadata"); + } + } +} diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index b818a52e83877..d530fc7cf1bd3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2643,4 +2643,15 @@ private DataNodePipeMessages() {} "Interrupted while reading OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "Failed to read OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "Unsupported logical backup event policy: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "Unsupported logical backup resume policy: %s"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "Unsupported Pipe event for logical backup: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Logical backup ID must resolve below the configured backup directory: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Logical backup directory must not be empty"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index fe768a1f0181c..49575e46abb91 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2467,4 +2467,15 @@ private DataNodePipeMessages() {} "读取 OPC UA 服务器操作限制时被中断，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "读取 OPC UA 服务器操作限制失败，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "不支持的逻辑备份事件策略：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "不支持的逻辑备份恢复策略：%s"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "逻辑备份不支持 Pipe 事件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Pipe 逻辑备份 ID 必须解析到配置的备份目录下：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Pipe 逻辑备份目录不能为空"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java index c8e87890afd69..08061c954afcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink; import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; @@ -74,6 +75,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_CONNECTOR.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBDataRegionAsyncSink::new); @@ -96,6 +100,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_SINK.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.SUBSCRIPTION_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java index 160ecf54c0158..a4eba6400c178 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink; import org.apache.iotdb.pipe.api.PipeConnector; @@ -46,6 +47,9 @@ protected void initConstructors() { IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("schema")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new); @@ -59,6 +63,9 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("schema")); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..5050107c05f98 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,591 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.pipe.api.PipeConnector; +import org.apache.iotdb.pipe.api.annotation.TableModel; +import org.apache.iotdb.pipe.api.annotation.TreeModel; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.write.record.Tablet; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +@TreeModel +@TableModel +public class LogicalBackupSink implements PipeConnector { + + private static final int FILE_PIECE_BYTES = 64 * 1024; + private LogicalBackupWriter writer; + private String pipeName; + private long creationTime; + private int regionId; + private boolean includeHeartbeat; + private String unsupportedEventPolicy; + private final String streamType; + + public LogicalBackupSink() { + this("unknown"); + } + + public LogicalBackupSink(final String streamType) { + this.streamType = streamType; + } + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + final PipeParameters parameters = validator.getParameters(); + validator.validateSynonymAttributes( + Collections.singletonList(PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY), + Collections.singletonList(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY), + true); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + final String policy = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + if (!PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL.equals(policy) + && !PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(policy)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405, + policy)); + } + } + + private static void validateSynonymAttributes( + final PipeParameterValidator validator, + final String connectorAttribute, + final String sinkAttribute) { + validator.validateSynonymAttributes( + Collections.singletonList(connectorAttribute), + Collections.singletonList(sinkAttribute), + false); + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + pipeName = configuration.getRuntimeEnvironment().getPipeName(); + creationTime = configuration.getRuntimeEnvironment().getCreationTime(); + regionId = configuration.getRuntimeEnvironment().getRegionId(); + final String root = + parameters.getStringByKeys( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY); + if (root == null || root.isBlank()) { + throw new PipeParameterNotValidException( + DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB); + } + final String backupId = + parameters.getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY), + pipeName + "-" + creationTime); + if (backupId == null || backupId.isBlank()) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId; + final Path backupRoot = Paths.get(root).toAbsolutePath().normalize(); + final Path backupIdPath; + try { + backupIdPath = Path.of(backupId); + } catch (final RuntimeException e) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + if (backupIdPath.isAbsolute() + || backupIdPath.getNameCount() != 1 + || ".".equals(backupId) + || "..".equals(backupId)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize(); + if (!directory.startsWith(backupRoot)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + final String resume = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), + PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); + if (!append + && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) + && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9, + resume)); + } + if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) { + directory = + directory.resolveSibling( + directory.getFileName() + + "-new-" + + System.currentTimeMillis() + + "-" + + UUID.randomUUID().toString().substring(0, 8)); + } + final long segmentSize = + parameters.getLongOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY), + PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE); + final int maxRecord = + parameters.getIntOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY), + PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE); + final String fsync = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE) + .toUpperCase(Locale.ROOT); + final LogicalBackupWriter.FsyncPolicy fsyncPolicy = + LogicalBackupWriter.FsyncPolicy.valueOf(fsync); + final int fsyncBatch = + parameters.getIntOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE); + final long fsyncPeriod = + parameters.getLongOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE); + includeHeartbeat = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE); + unsupportedEventPolicy = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = backupId; + manifest.pipeName = pipeName; + manifest.pipeCreationTime = creationTime; + manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId(); + manifest.sourceVersion = IoTDBConstant.VERSION; + manifest.timestampPrecision = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + manifest.streamId = streamId; + manifest.regionId = regionId; + manifest.streamType = streamType; + manifest.sinkTaskId = streamId; + writer = + new LogicalBackupWriter( + directory, + manifest, + segmentSize, + maxRecord, + fsyncPolicy, + fsyncBatch, + fsyncPeriod, + append); + } + + @Override + public void handshake() {} + + @Override + public void heartbeat() throws Exception { + if (writer != null) { + writer.heartbeat(); + } + } + + @Override + public void transfer(final TabletInsertionEvent event) throws Exception { + if (event instanceof PipeInsertNodeTabletInsertionEvent) { + final PipeInsertNodeTabletInsertionEvent insert = (PipeInsertNodeTabletInsertionEvent) event; + transferEnriched(insert, () -> requests(insert)); + } else if (event instanceof PipeRawTabletInsertionEvent) { + final PipeRawTabletInsertionEvent raw = (PipeRawTabletInsertionEvent) event; + transferEnriched( + raw, + () -> + Collections.singletonList( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + raw.convertToTablet(), + raw.isAligned(), + raw.isTableModelEvent() + ? raw.getTableModelDatabaseName() + : raw.getTreeModelDatabaseName()))); + } else { + handleUnsupported(event); + } + } + + @Override + public void transfer(final TsFileInsertionEvent event) throws Exception { + if (!(event instanceof PipeTsFileInsertionEvent)) { + handleUnsupported(event); + return; + } + final PipeTsFileInsertionEvent tsFile = (PipeTsFileInsertionEvent) event; + final List requests = new ArrayList<>(); + if (!tsFile.increaseReferenceCount(getClass().getName())) { + return; + } + try { + tsFile.consumeTabletInsertionEventsWithRetry( + tablet -> { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablet.convertToTablet(), + tablet.isAligned(), + tablet.isTableModelEvent() + ? tablet.getTableModelDatabaseName() + : tablet.getTreeModelDatabaseName())); + }, + getClass().getName()); + if (!requests.isEmpty()) { + writer.writeEvent(eventId(tsFile), System.currentTimeMillis(), requests, metadata(tsFile)); + } + } finally { + event.close(); + tsFile.decreaseReferenceCount(getClass().getName(), false); + } + } + + @Override + public void transfer(final Event event) throws Exception { + if (event instanceof PipeDeleteDataNodeEvent) { + final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event; + transferEnriched( + deletion, + () -> + Collections.singletonList( + PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode()))); + } else if (event instanceof PipeSchemaRegionWritePlanEvent) { + final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event; + transferEnriched( + schema, + () -> + Collections.singletonList( + PipeTransferPlanNodeReq.toTPipeTransferReq(schema.getPlanNode()))); + } else if (event instanceof PipeSchemaRegionSnapshotEvent) { + transferSnapshot((PipeSchemaRegionSnapshotEvent) event); + } else if (event instanceof PipeHeartbeatEvent) { + if (includeHeartbeat) { + writer.writeControl( + LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event)); + } + } else if (event instanceof PipeTerminateEvent) { + writer.writeControl( + LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event)); + } else { + handleUnsupported(event); + } + } + + private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + final List requests = new ArrayList<>(); + final File[] files = + new File[] { + event.getMTreeSnapshotFile(), + event.getTagLogSnapshotFile(), + event.getAttributeSnapshotFile() + }; + for (final File file : files) { + if (file == null) { + continue; + } + try (final FileInputStream input = new FileInputStream(file)) { + final byte[] piece = new byte[FILE_PIECE_BYTES]; + long offset = 0; + int length; + while ((length = input.read(piece)) >= 0) { + if (length == 0) { + continue; + } + requests.add( + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq( + file.getName(), offset, Arrays.copyOf(piece, length))); + offset += length; + } + } + } + requests.add( + PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( + event.getTreePattern().getPattern(), + event.getTablePattern().getDatabasePattern(), + event.getTablePattern().getTablePattern(), + event.getTreePattern().isTreeModelDataAllowedToBeCaptured(), + event.getTablePattern().isTableModelDataAllowedToBeCaptured(), + files[0] == null ? null : files[0].getName(), + files[0] == null ? 0 : files[0].length(), + files[1] == null ? null : files[1].getName(), + files[1] == null ? 0 : files[1].length(), + files[2] == null ? null : files[2].getName(), + files[2] == null ? 0 : files[2].length(), + event.getDatabaseName(), + event.toSealTypeString())); + writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event)); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + private List requests(final PipeInsertNodeTabletInsertionEvent event) + throws IOException { + final String databaseName = + event.isTableModelEvent() + ? event.getTableModelDatabaseName() + : event.getTreeModelDatabaseName(); + if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) { + return Collections.singletonList( + PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq( + event.getInsertNode(), databaseName)); + } + + final List tablets = event.convertToTablets(); + final List requests = new ArrayList<>(tablets.size()); + for (int index = 0; index < tablets.size(); index++) { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablets.get(index), event.isAligned(index), databaseName)); + } + return requests; + } + + private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) + throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + writer.writeEvent( + eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event)); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + @FunctionalInterface + private interface RequestSupplier { + List get() throws Exception; + } + + private void handleUnsupported(final Event event) throws Exception { + if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) { + writer.recordSkippedEvent( + System.currentTimeMillis(), event == null ? "null" : event.getClass().getName()); + return; + } + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97, + event == null ? "null" : event.getClass().getName())); + } + + private UUID eventId(final EnrichedEvent event) { + return UUID.nameUUIDFromBytes( + (event.getClass().getName() + + '\0' + + event.getPipeName() + + '\0' + + event.getCreationTime() + + '\0' + + event.getRegionId() + + '\0' + + commitIds(event)) + .getBytes(StandardCharsets.UTF_8)); + } + + private String commitIds(final EnrichedEvent event) { + try { + return String.valueOf(event.getCommitIds()); + } catch (final UnsupportedOperationException ignored) { + return Long.toString(event.getCommitId()); + } + } + + private String metadata(final Event event) { + if (event instanceof EnrichedEvent) { + final EnrichedEvent enriched = (EnrichedEvent) event; + return "pipe=" + + enriched.getPipeName() + + ";creationTime=" + + enriched.getCreationTime() + + ";regionId=" + + enriched.getRegionId() + + ";commitIds=" + + commitIds(enriched) + + ";eventType=" + + event.getClass().getName(); + } + return "eventType=" + (event == null ? "null" : event.getClass().getName()); + } + + @Override + public void close() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3bb68a29562c3..a632dbff5ddef 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -139,6 +139,10 @@ com.google.guava guava + + com.google.code.gson + gson + com.google.code.findbugs jsr305 diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..1629e2dd00028 --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "directory"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "manifest"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsyncPolicy"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "eventGroupId"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Logical backup directory already exists: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Logical backup event must contain at least one request"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Logical backup request body must not be null"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Logical backup record exceeds max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "Unsupported logical backup manifest format: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Logical backup manifest does not match %s: expected %s, found %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Logical backup directory is locked: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Logical backup writer is closed"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Incomplete logical backup segment header: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Invalid logical backup segment footer size: %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "Invalid logical backup record magic at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "Logical backup record header CRC mismatch at offset %d in %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "Invalid logical backup record length at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "Logical backup record payload CRC mismatch at offset %d in %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "Unknown logical backup record type %d in %s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Nested logical backup event groups in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Logical backup commit without begin in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "Logical backup operation index mismatch at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "Logical backup sequence mismatch at offset %d in %s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Incomplete logical backup record tail: %s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "Sealed logical backup segment contains an open event group: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Invalid logical backup segment magic: %s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "Unsupported logical backup segment major version %d: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Logical backup segment header CRC mismatch: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Invalid logical backup segment footer magic: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Logical backup segment footer ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Logical backup segment digest mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Logical backup segment footer CRC mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Logical backup segment footer metadata mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Logical backup event ID %s was already written with a different digest"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Invalid logical backup writer configuration: segment-size, max-record, fsync-batch and" + + " fsync-period must be positive"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "Logical backup request outside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "Logical backup control record inside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Logical backup segment ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Logical backup sequence gap in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Logical backup has no segments"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Invalid logical backup segment path: %s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "No logical backup manifest found under %s"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Duplicate logical backup stream ID: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Logical backup manifest has no segments: %s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "Unlisted logical backup segment %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Logical backup segment metadata mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Logical backup segment is not sealed: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Logical backup manifest counters do not match segment contents: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Logical backup sequence is not continuous across segments: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Logical backup manifest is invalid: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Logical backup contains skipped events: %s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "Logical backup request version %d is not supported"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "Logical backup request type %d is not allowed"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "Duplicate logical backup event group ID: %s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "Symbolic links are not allowed in logical backup directory paths: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "Logical backup directory is unavailable: %s"; + + private LogicalBackupMessages() {} +} diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..bfc2bed98fed1 --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "目录"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "清单"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsync 策略"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "事件组 ID"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Pipe 逻辑备份目录已存在：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Pipe 逻辑备份事件必须至少包含一个请求"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Pipe 逻辑备份请求体不能为空"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Pipe 逻辑备份记录超过 max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "不支持的 Pipe 逻辑备份清单格式：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Pipe 逻辑备份清单的 %s 不匹配：预期 %s，实际 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Pipe 逻辑备份目录已被锁定：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Pipe 逻辑备份写入器已关闭"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Pipe 逻辑备份 segment 文件头不完整：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Pipe 逻辑备份 segment 文件尾大小无效：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "偏移 %d 处的 Pipe 逻辑备份记录魔数无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "偏移 %d 处的 Pipe 逻辑备份记录头 CRC 不匹配，文件：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "偏移 %d 处的 Pipe 逻辑备份记录长度无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "偏移 %d 处的 Pipe 逻辑备份记录载荷 CRC 不匹配，文件：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "未知的 Pipe 逻辑备份记录类型 %d，文件：%s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Pipe 逻辑备份事件组发生嵌套，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Pipe 逻辑备份事件组缺少开始记录，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "偏移 %d 处的 Pipe 逻辑备份操作序号不匹配，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "偏移 %d 处的 Pipe 逻辑备份 sequence 不连续，文件：%s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Pipe 逻辑备份记录尾部不完整：%s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "已封存的 Pipe 逻辑备份 segment 包含未提交事件组：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Pipe 逻辑备份 segment 魔数无效：%s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "不支持的 Pipe 逻辑备份 segment 主版本 %d：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Pipe 逻辑备份 segment 文件头 CRC 不匹配：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Pipe 逻辑备份 segment 文件尾魔数无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Pipe 逻辑备份 segment 文件尾 ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Pipe 逻辑备份 segment 摘要不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Pipe 逻辑备份 segment 文件尾 CRC 不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Pipe 逻辑备份 segment 文件尾元数据不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Pipe 逻辑备份事件 ID %s 已使用不同摘要写入"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Pipe 逻辑备份写入器配置无效：segment-size、max-record、fsync-batch 和 fsync-period" + + " 必须为正数"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "偏移 %d 处的 Pipe 逻辑备份请求不在事件组中，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "偏移 %d 处的 Pipe 逻辑备份控制记录位于事件组内，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Pipe 逻辑备份 segment ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Pipe 逻辑备份 sequence 在 %s 中不连续"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Pipe 逻辑备份不包含 segment"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Pipe 逻辑备份 segment 路径无效：%s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "在 %s 下未找到 Pipe 逻辑备份 manifest"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Pipe 逻辑备份 stream ID 重复：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Pipe 逻辑备份 manifest 不包含 segment：%s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "未在 manifest 中列出 Pipe 逻辑备份 segment：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Pipe 逻辑备份 segment 元数据不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Pipe 逻辑备份 segment 尚未封存：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Pipe 逻辑备份 manifest 计数与 segment 内容不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Pipe 逻辑备份 sequence 在 segment 之间不连续：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Pipe 逻辑备份 manifest 无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Pipe 逻辑备份包含已跳过的事件：%s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "不支持逻辑备份请求版本 %d"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "不允许逻辑备份请求类型 %d"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "逻辑备份事件组 ID 重复：%s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "逻辑备份目录路径中不允许使用符号链接：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "逻辑备份目录不可用：%s"; + + private LogicalBackupMessages() {} +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 76b45d1e79895..2a10db843618c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; @@ -95,6 +96,7 @@ public enum BuiltinPipePlugin { OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class), OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class), WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class), + LOGICAL_BACKUP_CONNECTOR("logical-backup-connector", LogicalBackupSink.class), DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class), IOTDB_THRIFT_SINK("iotdb-thrift-sink", IoTDBThriftSink.class), @@ -107,6 +109,7 @@ public enum BuiltinPipePlugin { OPC_UA_SINK("opc-ua-sink", OpcUaSink.class), OPC_DA_SINK("opc-da-sink", OpcDaSink.class), WRITE_BACK_SINK("write-back-sink", WriteBackSink.class), + LOGICAL_BACKUP_SINK("logical-backup-sink", LogicalBackupSink.class), SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class), IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class), // Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename. @@ -178,6 +181,7 @@ public String getClassName() { OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(), OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(), WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(), + LOGICAL_BACKUP_CONNECTOR.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), // Sinks diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..d5fb9b22021ef --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.PlaceholderSink; + +public class LogicalBackupSink extends PlaceholderSink {} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java index f2d9ea283c823..6d5dac18c3d9c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java @@ -80,6 +80,55 @@ public class PipeSinkConstant { public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first"; public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true; + public static final String CONNECTOR_LOGICAL_BACKUP_DIR_KEY = "connector.dir"; + public static final String SINK_LOGICAL_BACKUP_DIR_KEY = "sink.dir"; + public static final String CONNECTOR_LOGICAL_BACKUP_ID_KEY = "connector.backup-id"; + public static final String SINK_LOGICAL_BACKUP_ID_KEY = "sink.backup-id"; + public static final String CONNECTOR_LOGICAL_BACKUP_RESUME_KEY = "connector.resume"; + public static final String SINK_LOGICAL_BACKUP_RESUME_KEY = "sink.resume"; + public static final String LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS = "fail-if-exists"; + public static final String LOGICAL_BACKUP_RESUME_APPEND = "append"; + public static final String LOGICAL_BACKUP_RESUME_NEW = "new"; + public static final String LOGICAL_BACKUP_RESUME_DEFAULT_VALUE = + LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS; + public static final String CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = + "connector.segment-size-bytes"; + public static final String SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = "sink.segment-size-bytes"; + public static final long LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE = 256L * MB; + public static final String CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = + "connector.max-record-bytes"; + public static final String SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = "sink.max-record-bytes"; + public static final int LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE = 64 * 1024 * 1024; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "connector.fsync-policy"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "sink.fsync-policy"; + public static final String LOGICAL_BACKUP_FSYNC_ALWAYS = "always"; + public static final String LOGICAL_BACKUP_FSYNC_BATCH = "batch"; + public static final String LOGICAL_BACKUP_FSYNC_PERIODIC = "periodic"; + public static final String LOGICAL_BACKUP_FSYNC_NONE = "none"; + public static final String LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE = + LOGICAL_BACKUP_FSYNC_ALWAYS; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "connector.fsync-batch-operations"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "sink.fsync-batch-operations"; + public static final int LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = + "connector.fsync-period-ms"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = "sink.fsync-period-ms"; + public static final long LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "connector.on-unsupported-event"; + public static final String SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "sink.on-unsupported-event"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL = "fail"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP = "skip"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE = + LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL; + public static final String CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = + "connector.include-heartbeat"; + public static final String SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = "sink.include-heartbeat"; + public static final boolean LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE = false; + public static final String CONNECTOR_SERIALIZE_BY_REGION_KEY = "connector.serialize-by-region"; public static final String SINK_SERIALIZE_BY_REGION_KEY = "sink.serialize-by-region"; public static final boolean CONNECTOR_SERIALIZE_BY_REGION_DEFAULT_VALUE = false; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java new file mode 100644 index 0000000000000..6ac9ca23f67bd --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; + +import com.google.gson.Gson; +import com.google.gson.JsonParseException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LogicalBackupArchiveReader { + + private static final Gson GSON = new Gson(); + private static final Set REPLAYABLE_REQUEST_TYPES = + Set.of( + PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType(), + PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), + PipeRequestType.TRANSFER_PLAN_NODE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType()); + + public List read(final Path source, final boolean allowIncomplete) + throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + final List manifests = discoverManifests(normalizedSource); + if (manifests.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E, + source)); + } + final List streams = new ArrayList<>(); + final Set streamIds = new HashSet<>(); + LogicalBackupManifest archiveIdentity = null; + for (final Path manifest : manifests) { + final BackupStream stream = readStream(manifest, allowIncomplete); + if (archiveIdentity == null) { + archiveIdentity = stream.getManifest(); + } else { + validateArchiveIdentity(archiveIdentity, stream.getManifest()); + } + if (!streamIds.add(stream.getManifest().streamId)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE, + stream.getManifest().streamId)); + } + streams.add(stream); + } + streams.sort( + Comparator.comparingInt( + (BackupStream stream) -> + "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1) + .thenComparing((BackupStream stream) -> stream.getManifest().streamId)); + return Collections.unmodifiableList(streams); + } + + private static void validateArchiveIdentity( + final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException { + validateArchiveIdentityField("backupId", expected.backupId, actual.backupId); + validateArchiveIdentityField("pipeName", expected.pipeName, actual.pipeName); + validateArchiveIdentityField( + "pipeCreationTime", expected.pipeCreationTime, actual.pipeCreationTime); + validateArchiveIdentityField( + "sourceClusterId", expected.sourceClusterId, actual.sourceClusterId); + validateArchiveIdentityField("sourceVersion", expected.sourceVersion, actual.sourceVersion); + validateArchiveIdentityField( + "timestampPrecision", expected.timestampPrecision, actual.timestampPrecision); + } + + private static void validateArchiveIdentityField( + final String field, final Object expected, final Object actual) throws IOException { + if (!java.util.Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete) + throws IOException { + final LogicalBackupManifest manifest; + try { + manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + validateManifest(manifest, manifestPath); + if (!allowIncomplete && manifest.skippedEventCount > 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + + final Path streamDirectory = manifestPath.getParent().toAbsolutePath().normalize(); + final LogicalBackupSegmentReader segmentReader = + new LogicalBackupSegmentReader(manifest.maxRecordBytes); + final List records = new ArrayList<>(); + final Map operationCounts = new LinkedHashMap<>(); + final Set listedSegments = new HashSet<>(); + long expectedSequence = -1; + long expectedSegmentId = 0; + + for (int index = 0; index < manifest.segments.size(); index++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(index); + if (segment == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + "null")); + } + final Path segmentPath = resolveSegmentPath(streamDirectory, segment.file); + if (segmentPath == null + || Files.isSymbolicLink(segmentPath) + || !Files.isRegularFile(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + segment.file)); + } + listedSegments.add(segmentPath); + final boolean lastSegment = index == manifest.segments.size() - 1; + final LogicalBackupSegmentReader.ScanResult scan = + segmentReader.scan(segmentPath, allowIncomplete && lastSegment); + if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF, + segmentPath)); + } + if (scan.isSealed()) { + validateSegmentManifest(segment, scan, segmentPath); + } + for (final LogicalBackupRecord record : scan.getRecords()) { + validateReplayableRequest(record); + } + + final List committedRecords = + scan.hasOpenEventGroup() ? removeOpenEventGroup(scan.getRecords()) : scan.getRecords(); + for (final LogicalBackupRecord record : committedRecords) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E, + manifestPath)); + } + expectedSequence = record.getSequence() + 1; + records.add(record); + operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + } + } + + if (!allowIncomplete) { + validateUnlistedSegments(streamDirectory, listedSegments); + final long skippedEventCount = + operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L); + if (skippedEventCount > 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + if (skippedEventCount != manifest.skippedEventCount + || !operationCounts.equals(manifest.operationCounts) + || (records.isEmpty() + ? manifest.firstSequence != -1 || manifest.lastSequence != -1 + : records.get(0).getSequence() != manifest.firstSequence + || records.get(records.size() - 1).getSequence() != manifest.lastSequence)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB, + manifestPath)); + } + } + return new BackupStream(manifestPath, manifest, records, toEventGroups(records)); + } + + private static void validateReplayableRequest(final LogicalBackupRecord record) + throws IOException { + if (record.getRecordType() != LogicalBackupRecordType.PIPE_REQUEST) { + return; + } + if (record.getRequestVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion()) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7, + record.getRequestVersion())); + } + if (!REPLAYABLE_REQUEST_TYPES.contains(record.getRequestType())) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38, + record.getRequestType())); + } + } + + private static List discoverManifests(final Path source) throws IOException { + if (!Files.isSymbolicLink(source) + && Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + && LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())) { + return Collections.singletonList(source); + } + if (!Files.isDirectory(source)) { + return Collections.emptyList(); + } + try (final Stream paths = Files.walk(source)) { + return paths + .filter(path -> !Files.isSymbolicLink(path)) + .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + .filter( + path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString())) + .sorted() + .collect(Collectors.toList()); + } + } + + private static Path resolveSegmentPath(final Path streamDirectory, final String segmentFile) { + if (segmentFile == null) { + return null; + } + final Path relativePath; + try { + relativePath = Path.of(segmentFile); + } catch (final RuntimeException e) { + return null; + } + if (relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + return null; + } + final Path segmentPath = streamDirectory.resolve(relativePath).normalize(); + return segmentPath.startsWith(streamDirectory) + && segmentPath.getParent().equals(streamDirectory) + ? segmentPath + : null; + } + + private static void validateManifest( + final LogicalBackupManifest manifest, final Path manifestPath) throws IOException { + if (manifest == null + || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion) + || manifest.streamId == null + || manifest.backupId == null + || manifest.timestampPrecision == null + || manifest.maxRecordBytes <= 0 + || manifest.maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || manifest.skippedEventCount < 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (manifest.segments == null || manifest.segments.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138, + manifestPath)); + } + if (manifest.operationCounts == null) { + manifest.operationCounts = new LinkedHashMap<>(); + } + } + + private static void validateSegmentManifest( + final LogicalBackupManifest.Segment segment, + final LogicalBackupSegmentReader.ScanResult scan, + final Path segmentPath) + throws IOException { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (!"SEALED".equals(segment.status) + || segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + private static void validateUnlistedSegments( + final Path streamDirectory, final Set listedSegments) throws IOException { + try (final Stream paths = Files.list(streamDirectory)) { + final Path unlisted = + paths + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .map(path -> path.toAbsolutePath().normalize()) + .filter(path -> !listedSegments.contains(path)) + .findFirst() + .orElse(null); + if (unlisted != null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + unlisted)); + } + } + } + + private static List removeOpenEventGroup( + final List records) { + for (int i = records.size() - 1; i >= 0; i--) { + if (records.get(i).getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + return records.subList(0, i); + } + } + return records; + } + + private static List toEventGroups(final List records) + throws IOException { + final List groups = new ArrayList<>(); + final Set eventGroupIds = new HashSet<>(); + UUID eventGroupId = null; + long firstSequence = -1; + String metadata = ""; + final List requests = new ArrayList<>(); + for (final LogicalBackupRecord record : records) { + if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventGroupId = record.getEventGroupId(); + if (!eventGroupIds.add(eventGroupId)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89, + eventGroupId)); + } + firstSequence = record.getSequence(); + metadata = record.getMetadata(); + requests.clear(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + requests.add(record); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + groups.add( + new EventGroup( + eventGroupId, + firstSequence, + record.getSequence(), + metadata, + new ArrayList<>(requests))); + eventGroupId = null; + requests.clear(); + } + } + return Collections.unmodifiableList(groups); + } + + public static class BackupStream { + private final Path manifestPath; + private final LogicalBackupManifest manifest; + private final List records; + private final List eventGroups; + + private BackupStream( + final Path manifestPath, + final LogicalBackupManifest manifest, + final List records, + final List eventGroups) { + this.manifestPath = manifestPath; + this.manifest = manifest; + this.records = Collections.unmodifiableList(records); + this.eventGroups = eventGroups; + } + + public Path getManifestPath() { + return manifestPath; + } + + public LogicalBackupManifest getManifest() { + return manifest; + } + + public List getRecords() { + return records; + } + + public List getEventGroups() { + return eventGroups; + } + } + + public static class EventGroup { + private final UUID eventGroupId; + private final long firstSequence; + private final long lastSequence; + private final String metadata; + private final List requests; + + private EventGroup( + final UUID eventGroupId, + final long firstSequence, + final long lastSequence, + final String metadata, + final List requests) { + this.eventGroupId = eventGroupId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.metadata = metadata; + this.requests = Collections.unmodifiableList(requests); + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public String getMetadata() { + return metadata; + } + + public List getRequests() { + return requests; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java new file mode 100644 index 0000000000000..6309595202b52 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.zip.CRC32C; + +public final class LogicalBackupFormat { + + public static final String FORMAT_NAME = "iotdb-pipe-logical-backup"; + public static final String FORMAT_VERSION = "1.0"; + public static final short MAJOR_VERSION = 1; + public static final short MINOR_VERSION = 0; + public static final long SEGMENT_MAGIC = 0x494F545057414C31L; + public static final int RECORD_MAGIC = 0x50575231; + public static final int FOOTER_MAGIC = 0x50575346; + public static final int SEGMENT_HEADER_SIZE = 32; + public static final int RECORD_HEADER_SIZE = 56; + public static final int SEGMENT_FOOTER_SIZE = 96; + public static final int SHA256_SIZE = 32; + public static final int MAX_RECORD_BYTES = 256 * 1024 * 1024; + public static final String MANIFEST_FILE_NAME = "manifest.json"; + + private LogicalBackupFormat() {} + + public static int crc32c(final byte[] bytes, final int offset, final int length) { + final CRC32C crc32c = new CRC32C(); + crc32c.update(bytes, offset, length); + return (int) crc32c.getValue(); + } + + public static MessageDigest newSha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + public static String toHex(final byte[] bytes) { + return HexFormat.of().formatHex(bytes); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java new file mode 100644 index 0000000000000..3742ca88d8016 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class LogicalBackupManifest { + + public String formatName = LogicalBackupFormat.FORMAT_NAME; + public String formatVersion = LogicalBackupFormat.FORMAT_VERSION; + public String backupId; + public String status; + public String pipeName; + public long pipeCreationTime; + public String sourceClusterId; + public String sourceVersion; + public String timestampPrecision; + public String createdAt; + public String closedAt; + public String streamId; + public int regionId; + public String sinkTaskId; + public String streamType; + public String fsyncPolicy; + public long segmentSizeBytes; + public int maxRecordBytes; + public long firstSequence = -1; + public long lastSequence = -1; + public long lastDurableSequence = -1; + public String lastEventGroupId; + public String lastEventDigest; + public long lastEventFirstSequence = -1; + public long skippedEventCount; + public boolean recovered; + public Map operationCounts = new LinkedHashMap<>(); + public List segments = new ArrayList<>(); + + public static class Segment { + public String file; + public long segmentId; + public long firstSequence = -1; + public long lastSequence = -1; + public long recordCount; + public long sizeBytes; + public String sha256; + public String status; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java new file mode 100644 index 0000000000000..b779c4fc4aa3e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.UUID; + +public class LogicalBackupRecord { + + private final LogicalBackupRecordType recordType; + private final long sequence; + private final UUID eventGroupId; + private final int operationIndex; + private final long eventTime; + private final byte requestVersion; + private final short requestType; + private final String metadata; + private final byte[] payload; + + public LogicalBackupRecord( + final LogicalBackupRecordType recordType, + final long sequence, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final byte requestVersion, + final short requestType, + final String metadata, + final byte[] payload) { + this.recordType = recordType; + this.sequence = sequence; + this.eventGroupId = eventGroupId; + this.operationIndex = operationIndex; + this.eventTime = eventTime; + this.requestVersion = requestVersion; + this.requestType = requestType; + this.metadata = metadata; + this.payload = payload; + } + + public LogicalBackupRecordType getRecordType() { + return recordType; + } + + public long getSequence() { + return sequence; + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public int getOperationIndex() { + return operationIndex; + } + + public long getEventTime() { + return eventTime; + } + + public byte getRequestVersion() { + return requestVersion; + } + + public short getRequestType() { + return requestType; + } + + public String getMetadata() { + return metadata; + } + + public byte[] getPayload() { + return payload; + } + + public TPipeTransferReq toTPipeTransferReq() { + return new TPipeTransferReq() + .setVersion(requestVersion) + .setType(requestType) + .setBody(ByteBuffer.wrap(payload)); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LogicalBackupRecord)) { + return false; + } + final LogicalBackupRecord that = (LogicalBackupRecord) obj; + return sequence == that.sequence + && operationIndex == that.operationIndex + && eventTime == that.eventTime + && requestVersion == that.requestVersion + && requestType == that.requestType + && recordType == that.recordType + && eventGroupId.equals(that.eventGroupId) + && metadata.equals(that.metadata) + && Arrays.equals(payload, that.payload); + } + + @Override + public int hashCode() { + int result = eventGroupId.hashCode(); + result = 31 * result + recordType.hashCode(); + result = 31 * result + Long.hashCode(sequence); + result = 31 * result + operationIndex; + result = 31 * result + Long.hashCode(eventTime); + result = 31 * result + requestVersion; + result = 31 * result + requestType; + result = 31 * result + metadata.hashCode(); + return 31 * result + Arrays.hashCode(payload); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java new file mode 100644 index 0000000000000..e188669548ca6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.HashMap; +import java.util.Map; + +public enum LogicalBackupRecordType { + EVENT_BEGIN((byte) 1), + PIPE_REQUEST((byte) 2), + EVENT_COMMIT((byte) 3), + HEARTBEAT((byte) 4), + SKIPPED_EVENT((byte) 5), + STREAM_END((byte) 6); + + private static final Map TYPE_MAP = new HashMap<>(); + + static { + for (final LogicalBackupRecordType type : values()) { + TYPE_MAP.put(type.code, type); + } + } + + private final byte code; + + LogicalBackupRecordType(final byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + public static LogicalBackupRecordType valueOf(final byte code) { + return TYPE_MAP.get(code); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java new file mode 100644 index 0000000000000..e400d6133c27d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java @@ -0,0 +1,541 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; + +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupSegmentReader { + + private final int maxRecordBytes; + + public LogicalBackupSegmentReader(final int maxRecordBytes) { + this.maxRecordBytes = maxRecordBytes; + } + + public ScanResult scan(final Path segment, final boolean allowIncompleteTail) throws IOException { + try (final RandomAccessFile reader = new RandomAccessFile(segment.toFile(), "r")) { + final long fileLength = reader.length(); + if (fileLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new EOFException( + String.format( + LogicalBackupMessages + .EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A, + segment)); + } + + final byte[] segmentHeader = readBytes(reader, LogicalBackupFormat.SEGMENT_HEADER_SIZE); + validateSegmentHeader(segmentHeader, segment); + final ByteBuffer segmentHeaderBuffer = wrap(segmentHeader); + segmentHeaderBuffer.position(Long.BYTES + Short.BYTES * 2); + final long segmentId = segmentHeaderBuffer.getLong(); + final long createdAt = segmentHeaderBuffer.getLong(); + + final MessageDigest digest = LogicalBackupFormat.newSha256(); + digest.update(segmentHeader); + final List records = new ArrayList<>(); + long validLength = LogicalBackupFormat.SEGMENT_HEADER_SIZE; + long lastCommittedLength = validLength; + boolean openEventGroup = false; + UUID openEventGroupId = null; + int nextOperationIndex = 0; + long expectedSequence = -1; + boolean incompleteTail = false; + Footer footer = null; + + while (reader.getFilePointer() < fileLength) { + final long frameOffset = reader.getFilePointer(); + if (fileLength - frameOffset < Integer.BYTES) { + incompleteTail = true; + break; + } + + final int magic = reader.readInt(); + reader.seek(frameOffset); + if (magic == LogicalBackupFormat.FOOTER_MAGIC) { + if (fileLength - frameOffset < LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + incompleteTail = true; + break; + } + if (fileLength - frameOffset > LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278, + segment)); + } + final byte[] footerBytes = readBytes(reader, LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + footer = + parseAndValidateFooter( + footerBytes, + segmentId, + createdAt, + records.isEmpty() ? -1 : records.get(0).getSequence(), + records.isEmpty() ? -1 : records.get(records.size() - 1).getSequence(), + frameOffset, + records.size(), + digest.digest(), + segment); + validLength = reader.getFilePointer(); + lastCommittedLength = validLength; + break; + } + if (magic != LogicalBackupFormat.RECORD_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5, + frameOffset, + segment)); + } + if (fileLength - frameOffset < LogicalBackupFormat.RECORD_HEADER_SIZE) { + incompleteTail = true; + break; + } + + final byte[] header = readBytes(reader, LogicalBackupFormat.RECORD_HEADER_SIZE); + final ByteBuffer headerBuffer = wrap(header); + headerBuffer.getInt(); + final byte recordTypeCode = headerBuffer.get(); + final long sequence = headerBuffer.getLong(); + final UUID eventGroupId = new UUID(headerBuffer.getLong(), headerBuffer.getLong()); + final int operationIndex = headerBuffer.getInt(); + final long eventTime = headerBuffer.getLong(); + final byte requestVersion = headerBuffer.get(); + final short requestType = headerBuffer.getShort(); + final int metadataLength = headerBuffer.getInt(); + final int payloadLength = headerBuffer.getInt(); + final int headerCrc = headerBuffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79, + frameOffset, + segment)); + } + if (metadataLength < 0 + || payloadLength < 0 + || metadataLength > maxRecordBytes + || payloadLength > maxRecordBytes + || (long) metadataLength + payloadLength > maxRecordBytes) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E, + frameOffset, + segment)); + } + final long remainingLength = (long) metadataLength + payloadLength + Integer.BYTES; + if (fileLength - reader.getFilePointer() < remainingLength) { + incompleteTail = true; + break; + } + + final byte[] metadataBytes = readBytes(reader, metadataLength); + final byte[] payload = readBytes(reader, payloadLength); + final int frameCrc = reader.readInt(); + final byte[] frameContent = new byte[metadataLength + payloadLength]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataLength); + System.arraycopy(payload, 0, frameContent, metadataLength, payloadLength); + if (frameCrc != LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552, + frameOffset, + segment)); + } + + final LogicalBackupRecordType recordType = LogicalBackupRecordType.valueOf(recordTypeCode); + if (recordType == null) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D, + recordTypeCode, + segment)); + } + if (expectedSequence >= 0 && sequence != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B, + frameOffset, + segment)); + } + final LogicalBackupRecord record = + new LogicalBackupRecord( + recordType, + sequence, + eventGroupId, + operationIndex, + eventTime, + requestVersion, + requestType, + new String(metadataBytes, StandardCharsets.UTF_8), + payload); + records.add(record); + digest.update(header); + digest.update(frameContent); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(frameCrc).array()); + validLength = reader.getFilePointer(); + if (recordType == LogicalBackupRecordType.EVENT_BEGIN) { + if (openEventGroup || operationIndex != -1) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A, + segment)); + } + openEventGroup = true; + openEventGroupId = eventGroupId; + nextOperationIndex = 0; + } else if (recordType == LogicalBackupRecordType.EVENT_COMMIT) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + openEventGroup = false; + openEventGroupId = null; + lastCommittedLength = validLength; + } else if (recordType == LogicalBackupRecordType.PIPE_REQUEST) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F, + frameOffset, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + nextOperationIndex++; + } else if (openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B, + frameOffset, + segment)); + } else if (!openEventGroup) { + lastCommittedLength = validLength; + } + expectedSequence = sequence + 1; + } + + if (incompleteTail && !allowIncompleteTail) { + throw new EOFException( + String.format( + LogicalBackupMessages.EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7, + segment)); + } + if (footer != null && openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8, + segment)); + } + return new ScanResult( + segmentId, + createdAt, + records, + footer, + validLength, + lastCommittedLength, + incompleteTail, + openEventGroup); + } + } + + private static void validateSegmentHeader(final byte[] header, final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(header); + if (buffer.getLong() != LogicalBackupFormat.SEGMENT_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73, + segment)); + } + final short majorVersion = buffer.getShort(); + buffer.getShort(); + if (majorVersion != LogicalBackupFormat.MAJOR_VERSION) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB, + majorVersion, + segment)); + } + buffer.position(LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES); + final int headerCrc = buffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55, + segment)); + } + } + + private static Footer parseAndValidateFooter( + final byte[] bytes, + final long expectedSegmentId, + final long expectedCreatedAt, + final long expectedFirstSequence, + final long expectedLastSequence, + final long expectedValidBytes, + final long expectedRecordCount, + final byte[] expectedDigest, + final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(bytes); + if (buffer.getInt() != LogicalBackupFormat.FOOTER_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4, + segment)); + } + final long segmentId = buffer.getLong(); + final long firstSequence = buffer.getLong(); + final long lastSequence = buffer.getLong(); + final long recordCount = buffer.getLong(); + final long validBytes = buffer.getLong(); + final long createdAt = buffer.getLong(); + final long sealedAt = buffer.getLong(); + final byte[] digest = new byte[LogicalBackupFormat.SHA256_SIZE]; + buffer.get(digest); + final int footerCrc = buffer.getInt(); + if (segmentId != expectedSegmentId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D, + segment)); + } + if (createdAt != expectedCreatedAt + || firstSequence != expectedFirstSequence + || lastSequence != expectedLastSequence + || validBytes != expectedValidBytes + || recordCount != expectedRecordCount) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB, + segment)); + } + if (!Arrays.equals(digest, expectedDigest)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116, + segment)); + } + if (footerCrc + != LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91, + segment)); + } + return new Footer( + segmentId, + firstSequence, + lastSequence, + recordCount, + validBytes, + createdAt, + sealedAt, + digest); + } + + private static byte[] readBytes(final RandomAccessFile reader, final int length) + throws IOException { + final byte[] bytes = new byte[length]; + reader.readFully(bytes); + return bytes; + } + + private static ByteBuffer wrap(final byte[] bytes) { + return ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + } + + public static class ScanResult { + private final long segmentId; + private final long createdAt; + private final List records; + private final Footer footer; + private final long validLength; + private final long lastCommittedLength; + private final boolean incompleteTail; + private final boolean openEventGroup; + + private ScanResult( + final long segmentId, + final long createdAt, + final List records, + final Footer footer, + final long validLength, + final long lastCommittedLength, + final boolean incompleteTail, + final boolean openEventGroup) { + this.segmentId = segmentId; + this.createdAt = createdAt; + this.records = Collections.unmodifiableList(records); + this.footer = footer; + this.validLength = validLength; + this.lastCommittedLength = lastCommittedLength; + this.incompleteTail = incompleteTail; + this.openEventGroup = openEventGroup; + } + + public long getSegmentId() { + return segmentId; + } + + public long getCreatedAt() { + return createdAt; + } + + public List getRecords() { + return records; + } + + public Footer getFooter() { + return footer; + } + + public long getValidLength() { + return validLength; + } + + public long getLastCommittedLength() { + return lastCommittedLength; + } + + public boolean hasIncompleteTail() { + return incompleteTail; + } + + public boolean hasOpenEventGroup() { + return openEventGroup; + } + + public boolean isSealed() { + return footer != null; + } + } + + public static class Footer { + private final long segmentId; + private final long firstSequence; + private final long lastSequence; + private final long recordCount; + private final long validBytes; + private final long createdAt; + private final long sealedAt; + private final byte[] digest; + + private Footer( + final long segmentId, + final long firstSequence, + final long lastSequence, + final long recordCount, + final long validBytes, + final long createdAt, + final long sealedAt, + final byte[] digest) { + this.segmentId = segmentId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.recordCount = recordCount; + this.validBytes = validBytes; + this.createdAt = createdAt; + this.sealedAt = sealedAt; + this.digest = digest; + } + + public long getSegmentId() { + return segmentId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public long getRecordCount() { + return recordCount; + } + + public long getValidBytes() { + return validBytes; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getSealedAt() { + return sealedAt; + } + + public byte[] getDigest() { + return digest; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java new file mode 100644 index 0000000000000..9c8ba6f9a9a63 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -0,0 +1,1162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +public class LogicalBackupWriter implements Closeable { + + public enum FsyncPolicy { + ALWAYS, + BATCH, + PERIODIC, + NONE + } + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private final Path directory; + private final Path manifestPath; + private final Path lockPath; + private final int maxRecordBytes; + private final long segmentSizeBytes; + private final FsyncPolicy fsyncPolicy; + private final int fsyncBatchOperations; + private final long fsyncPeriodMs; + private final LogicalBackupManifest manifest; + private final LogicalBackupSegmentReader reader; + + private FileChannel channel; + private FileChannel lockChannel; + private FileLock directoryLock; + private long segmentId; + private long segmentCreatedAt; + private long segmentFirstSequence = -1; + private long segmentLastSequence = -1; + private long segmentRecordCount; + private MessageDigest segmentDigest; + private long nextSequence; + private int operationsSinceFsync; + private long lastFsyncAt; + private boolean closed; + private boolean streamEnded; + private boolean recoveredDuringOpen; + + public LogicalBackupWriter( + final Path directory, + final LogicalBackupManifest manifest, + final long segmentSizeBytes, + final int maxRecordBytes, + final FsyncPolicy fsyncPolicy, + final int fsyncBatchOperations, + final long fsyncPeriodMs, + final boolean append) + throws IOException { + this.directory = + Objects.requireNonNull(directory, LogicalBackupMessages.EXCEPTION_DIRECTORY_5F8F22B8) + .toAbsolutePath() + .normalize(); + this.manifest = + Objects.requireNonNull(manifest, LogicalBackupMessages.EXCEPTION_MANIFEST_7F5CB74A); + this.segmentSizeBytes = segmentSizeBytes; + this.maxRecordBytes = maxRecordBytes; + this.fsyncPolicy = + Objects.requireNonNull(fsyncPolicy, LogicalBackupMessages.EXCEPTION_FSYNC_POLICY_6D493614); + if (segmentSizeBytes + <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + || maxRecordBytes <= 0 + || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || fsyncBatchOperations <= 0 + || fsyncPeriodMs <= 0) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99); + } + this.fsyncBatchOperations = fsyncBatchOperations; + this.fsyncPeriodMs = fsyncPeriodMs; + this.manifestPath = this.directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + this.lockPath = this.directory.resolve(".backup.lock"); + this.reader = new LogicalBackupSegmentReader(maxRecordBytes); + try { + prepareDirectory(append); + acquireLock(); + if (Files.isSymbolicLink(manifestPath)) { + throw symbolicLinkException(manifestPath); + } + if (Files.exists(manifestPath)) { + if (!append) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory)); + } + loadExistingManifest(); + } else { + manifest.status = "WRITING"; + manifest.createdAt = Instant.now().toString(); + manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT); + manifest.segmentSizeBytes = segmentSizeBytes; + manifest.maxRecordBytes = maxRecordBytes; + writeManifest(); + } + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + nextSequence = Math.max(0, manifest.lastSequence + 1); + openOrCreateSegment(); + rebuildManifestFromSegments(); + if (recoveredDuringOpen) { + manifest.lastDurableSequence = manifest.lastSequence; + } else { + manifest.lastDurableSequence = + Math.max(-1, Math.min(manifest.lastDurableSequence, manifest.lastSequence)); + } + updateActiveSegmentManifest(); + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + writeManifest(); + lastFsyncAt = System.currentTimeMillis(); + } catch (final IOException | RuntimeException e) { + if (channel != null) { + try { + channel.close(); + } catch (final IOException closeException) { + e.addSuppressed(closeException); + } + channel = null; + } + releaseLock(); + throw e; + } + } + + private void prepareDirectory(final boolean append) throws IOException { + rejectSymbolicLinksInDirectoryPath(); + if (append) { + Files.createDirectories(directory); + rejectSymbolicLinksInDirectoryPath(); + return; + } + final Path parent = directory.getParent(); + if (parent != null) { + Files.createDirectories(parent); + rejectSymbolicLinksInDirectoryPath(); + } + try { + Files.createDirectory(directory); + } catch (final FileAlreadyExistsException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory), + e); + } + rejectSymbolicLinksInDirectoryPath(); + } + + private void rejectSymbolicLinksInDirectoryPath() throws IOException { + Path current = directory.getRoot(); + for (final Path part : directory) { + current = current == null ? part : current.resolve(part); + if (Files.isSymbolicLink(current)) { + throw symbolicLinkException(current); + } + } + } + + private static IOException symbolicLinkException(final Path path) { + return new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569, + path)); + } + + public synchronized long writeEvent( + final UUID eventGroupId, + final long eventTime, + final List requests, + final String metadata) + throws IOException { + ensureOpen(); + if (requests == null || requests.isEmpty()) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC); + } + Objects.requireNonNull(eventGroupId, LogicalBackupMessages.EXCEPTION_EVENT_GROUP_ID_C6F6268A); + final String safeMetadata = metadata == null ? "" : metadata; + final String eventDigest = computeEventDigest(requests); + if (eventGroupId.toString().equals(manifest.lastEventGroupId)) { + if (eventDigest.equals(manifest.lastEventDigest)) { + updateActiveSegmentManifest(); + writeManifest(); + return manifest.lastEventFirstSequence; + } + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330, + eventGroupId)); + } + rollBeforeEventIfNecessary(requests, safeMetadata); + final long eventStartPosition = channel.position(); + final long firstSequence = nextSequence; + try { + appendRecord( + LogicalBackupRecordType.EVENT_BEGIN, eventGroupId, -1, eventTime, null, safeMetadata); + int operationIndex = 0; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + appendRecord( + LogicalBackupRecordType.PIPE_REQUEST, + eventGroupId, + operationIndex++, + eventTime, + request, + safeMetadata); + } + appendRecord( + LogicalBackupRecordType.EVENT_COMMIT, + eventGroupId, + operationIndex, + eventTime, + null, + safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(eventStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = eventDigest; + manifest.lastEventFirstSequence = firstSequence; + writeManifest(); + return firstSequence; + } + + public synchronized void writeControl( + final LogicalBackupRecordType recordType, final long eventTime, final String metadata) + throws IOException { + ensureOpen(); + final String safeMetadata = metadata == null ? "" : metadata; + rollBeforeControlIfNecessary(safeMetadata); + final long recordStartPosition = channel.position(); + try { + appendRecord(recordType, new UUID(0, 0), -1, eventTime, null, safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(recordStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + writeManifest(); + } + + public synchronized void recordSkippedEvent(final long eventTime, final String metadata) + throws IOException { + writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata); + manifest.skippedEventCount++; + writeManifest(); + } + + public synchronized LogicalBackupManifest getManifest() { + return manifest; + } + + public synchronized void heartbeat() throws IOException { + ensureOpen(); + if (!Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) + || directoryLock == null + || !directoryLock.isValid()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD, + directory)); + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + updateActiveSegmentManifest(); + writeManifest(); + } + } + + private void appendRecord( + final LogicalBackupRecordType recordType, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final TPipeTransferReq request, + final String metadata) + throws IOException { + final byte[] metadataBytes = metadata.getBytes(StandardCharsets.UTF_8); + final byte[] payload = + request == null + ? new byte[0] + : java.util.Arrays.copyOf(request.getBody(), request.getBody().length); + if ((long) metadataBytes.length + payload.length > maxRecordBytes) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996); + } + final long requiredBytes = + LogicalBackupFormat.RECORD_HEADER_SIZE + + metadataBytes.length + + payload.length + + Integer.BYTES; + final long sequence = nextSequence++; + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.RECORD_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putInt(LogicalBackupFormat.RECORD_MAGIC); + header.put(recordType.getCode()); + header.putLong(sequence); + header.putLong(eventGroupId.getMostSignificantBits()); + header.putLong(eventGroupId.getLeastSignificantBits()); + header.putInt(operationIndex); + header.putLong(eventTime); + header.put(request == null ? (byte) 0 : request.getVersion()); + header.putShort(request == null ? (short) 0 : request.getType()); + header.putInt(metadataBytes.length); + header.putInt(payload.length); + header.putInt(0); + final byte[] headerBytes = header.array(); + final int headerCrc = + LogicalBackupFormat.crc32c(headerBytes, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4); + ByteBuffer.wrap(headerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt(LogicalBackupFormat.RECORD_HEADER_SIZE - 4, headerCrc); + final byte[] frameContent = new byte[metadataBytes.length + payload.length]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataBytes.length); + System.arraycopy(payload, 0, frameContent, metadataBytes.length, payload.length); + final int frameCrc = LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length); + + writeFully(ByteBuffer.wrap(headerBytes)); + writeFully(ByteBuffer.wrap(frameContent)); + writeFully( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).flip()); + segmentDigest.update(headerBytes); + segmentDigest.update(frameContent); + segmentDigest.update( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).array()); + if (segmentFirstSequence < 0) { + segmentFirstSequence = sequence; + } + segmentLastSequence = sequence; + segmentRecordCount++; + manifest.firstSequence = manifest.firstSequence < 0 ? sequence : manifest.firstSequence; + manifest.lastSequence = sequence; + final String countKey = recordType.name(); + manifest.operationCounts.put(countKey, manifest.operationCounts.getOrDefault(countKey, 0L) + 1); + operationsSinceFsync++; + streamEnded = recordType == LogicalBackupRecordType.STREAM_END; + } + + private void openOrCreateSegment() throws IOException { + if (manifest.segments.isEmpty()) { + segmentId = 0; + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + final LogicalBackupManifest.Segment last = manifest.segments.get(manifest.segments.size() - 1); + final Path segmentPath = resolveExistingSegmentPath(last.file); + LogicalBackupSegmentReader.ScanResult scan = scanLastSegmentForAppend(segmentPath, last); + if (scan.isSealed()) { + if (!"SEALED".equals(last.status)) { + manifest.recovered = true; + recoveredDuringOpen = true; + } + segmentId = scan.getSegmentId() + 1; + // Validate every existing sealed segment before creating a new file in the backup. + rebuildManifestFromSegments(); + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + validateSegmentFiles(null); + if (scan.hasIncompleteTail() || scan.hasOpenEventGroup()) { + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(scan.getLastCommittedLength()); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + scan = reader.scan(segmentPath, false); + } + if (!scan.getRecords().isEmpty()) { + nextSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence() + 1; + manifest.lastSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence(); + } + segmentId = scan.getSegmentId(); + openExistingSegment(segmentPath, scan); + } + + private LogicalBackupSegmentReader.ScanResult scanLastSegmentForAppend( + final Path segmentPath, final LogicalBackupManifest.Segment segment) throws IOException { + try { + return reader.scan(segmentPath, true); + } catch (final IOException scanException) { + final long recoverableLength = + "SEALED".equals(segment.status) + ? -1 + : findRecoverableTailStart(segmentPath, segment.sizeBytes); + if (recoverableLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw scanException; + } + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(recoverableLength); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + return reader.scan(segmentPath, false); + } + } + + private static long findRecoverableTailStart( + final Path segmentPath, final long manifestSegmentSize) throws IOException { + final long fileSize = Files.size(segmentPath); + if (fileSize + >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + final ByteBuffer magic = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN); + try (final FileChannel readChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + readChannel.position(fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + while (magic.hasRemaining()) { + if (readChannel.read(magic) < 0) { + break; + } + } + } + if (!magic.hasRemaining() && magic.flip().getInt() == LogicalBackupFormat.FOOTER_MAGIC) { + return fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + } + } + if (manifestSegmentSize >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + && manifestSegmentSize < fileSize + && fileSize - manifestSegmentSize <= LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + return manifestSegmentSize; + } + return -1; + } + + private void openExistingSegment( + final Path segmentPath, final LogicalBackupSegmentReader.ScanResult scan) throws IOException { + channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ); + channel.position(channel.size()); + segmentCreatedAt = scan.getCreatedAt(); + segmentDigest = LogicalBackupFormat.newSha256(); + try (final FileChannel digestChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + final ByteBuffer digestBuffer = ByteBuffer.allocate(64 * 1024); + long remaining = scan.getValidLength(); + while (remaining > 0) { + digestBuffer.clear(); + digestBuffer.limit((int) Math.min(digestBuffer.capacity(), remaining)); + final int read = digestChannel.read(digestBuffer); + if (read < 0) { + break; + } + remaining -= read; + segmentDigest.update(digestBuffer.array(), 0, read); + } + } + segmentFirstSequence = -1; + segmentLastSequence = -1; + segmentRecordCount = 0; + for (final LogicalBackupRecord record : scan.getRecords()) { + segmentFirstSequence = segmentFirstSequence < 0 ? record.getSequence() : segmentFirstSequence; + segmentLastSequence = record.getSequence(); + segmentRecordCount++; + } + streamEnded = + !scan.getRecords().isEmpty() + && scan.getRecords().get(scan.getRecords().size() - 1).getRecordType() + == LogicalBackupRecordType.STREAM_END; + } + + private void openNewSegment() throws IOException { + final String fileName = segmentFileName(segmentId); + final Path segmentPath = resolveSegmentPath(fileName); + final Path temporarySegmentPath = resolveSegmentPath(fileName + ".tmp"); + if (Files.exists(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + adoptOrphanSegment(resolveExistingSegmentPath(fileName), fileName); + return; + } + + // A temporary segment is never exposed through the manifest and contains only a header, so it + // is safe to discard after an interrupted creation attempt. + Files.deleteIfExists(temporarySegmentPath); + segmentCreatedAt = System.currentTimeMillis(); + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(segmentCreatedAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4, + LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)); + try (final FileChannel temporaryChannel = + FileChannel.open( + temporarySegmentPath, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + StandardOpenOption.READ)) { + final ByteBuffer headerBuffer = ByteBuffer.wrap(bytes); + while (headerBuffer.hasRemaining()) { + temporaryChannel.write(headerBuffer); + } + temporaryChannel.force(true); + } + try { + Files.move(temporarySegmentPath, segmentPath, StandardCopyOption.ATOMIC_MOVE); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporarySegmentPath, segmentPath); + } + + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void adoptOrphanSegment(final Path segmentPath, final String fileName) + throws IOException { + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segmentId + || scan.isSealed() + || !scan.getRecords().isEmpty() + || scan.getValidLength() != LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentPath)); + } + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void addActiveSegmentToManifest(final String fileName) throws IOException { + final LogicalBackupManifest.Segment segment = new LogicalBackupManifest.Segment(); + segment.file = fileName; + segment.segmentId = segmentId; + segment.status = "ACTIVE"; + manifest.segments.add(segment); + writeManifest(); + } + + private void validateSegmentFiles(final String allowedOrphanFile) throws IOException { + final Set listedFiles = new HashSet<>(); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + listedFiles.add(segment.file); + } + try (final DirectoryStream segmentFiles = Files.newDirectoryStream(directory, "*.pwal")) { + for (final Path segmentFile : segmentFiles) { + final String fileName = segmentFile.getFileName().toString(); + if (!listedFiles.contains(fileName) && !fileName.equals(allowedOrphanFile)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentFile)); + } + } + } + } + + private static String segmentFileName(final long id) { + return String.format(Locale.ROOT, "segment-%020d.pwal", id); + } + + private void sealSegment() throws IOException { + if (channel == null || segmentRecordCount == 0) { + return; + } + final byte[] digest = segmentDigest.digest(); + final ByteBuffer footer = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN); + footer.putInt(LogicalBackupFormat.FOOTER_MAGIC); + footer.putLong(segmentId); + footer.putLong(segmentFirstSequence); + footer.putLong(segmentLastSequence); + footer.putLong(segmentRecordCount); + footer.putLong(channel.position()); + footer.putLong(segmentCreatedAt); + footer.putLong(System.currentTimeMillis()); + footer.put(digest); + footer.putInt(0); + final byte[] footerBytes = footer.array(); + ByteBuffer.wrap(footerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4, + LogicalBackupFormat.crc32c( + footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)); + writeFully(ByteBuffer.wrap(footerBytes)); + channel.force(true); + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.sha256 = LogicalBackupFormat.toHex(digest); + segment.status = "SEALED"; + manifest.lastDurableSequence = segmentLastSequence; + channel.close(); + channel = null; + writeManifest(); + } + + private boolean forceIfNeeded() throws IOException { + if (channel == null) { + return false; + } + final long now = System.currentTimeMillis(); + final boolean shouldForce = + fsyncPolicy == FsyncPolicy.ALWAYS + || (fsyncPolicy == FsyncPolicy.BATCH && operationsSinceFsync >= fsyncBatchOperations) + || (fsyncPolicy == FsyncPolicy.PERIODIC && now - lastFsyncAt >= fsyncPeriodMs); + if (shouldForce) { + channel.force(true); + operationsSinceFsync = 0; + lastFsyncAt = now; + } + return shouldForce; + } + + private void writeManifest() throws IOException { + final Path temporary = + manifestPath.resolveSibling(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp"); + final byte[] manifestBytes = GSON.toJson(manifest).getBytes(StandardCharsets.UTF_8); + Files.deleteIfExists(temporary); + try (final FileChannel manifestChannel = + FileChannel.open(temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + final ByteBuffer manifestBuffer = ByteBuffer.wrap(manifestBytes); + while (manifestBuffer.hasRemaining()) { + manifestChannel.write(manifestBuffer); + } + manifestChannel.force(true); + } + try { + Files.move( + temporary, + manifestPath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporary, manifestPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + private void loadExistingManifest() throws IOException { + final LogicalBackupManifest existing; + try { + existing = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + if (existing == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (!LogicalBackupFormat.FORMAT_NAME.equals(existing.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(existing.formatVersion)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E, + manifestPath)); + } + validateManifestIdentity("backupId", manifest.backupId, existing.backupId); + validateManifestIdentity("streamId", manifest.streamId, existing.streamId); + validateManifestIdentity("pipeName", manifest.pipeName, existing.pipeName); + validateManifestIdentity("streamType", manifest.streamType, existing.streamType); + validateManifestIdentity("sourceClusterId", manifest.sourceClusterId, existing.sourceClusterId); + validateManifestIdentity("sourceVersion", manifest.sourceVersion, existing.sourceVersion); + validateManifestIdentity( + "timestampPrecision", manifest.timestampPrecision, existing.timestampPrecision); + validateManifestIdentity("sinkTaskId", manifest.sinkTaskId, existing.sinkTaskId); + validateManifestIdentity( + "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy); + validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes); + validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes); + if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "pipeCreationTime", + manifest.pipeCreationTime, + existing.pipeCreationTime)); + } + if (manifest.regionId != existing.regionId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "regionId", + manifest.regionId, + existing.regionId)); + } + manifest.status = existing.status; + manifest.backupId = existing.backupId; + manifest.pipeName = existing.pipeName; + manifest.pipeCreationTime = existing.pipeCreationTime; + manifest.sourceClusterId = existing.sourceClusterId; + manifest.sourceVersion = existing.sourceVersion; + manifest.timestampPrecision = existing.timestampPrecision; + manifest.streamId = existing.streamId; + manifest.regionId = existing.regionId; + manifest.sinkTaskId = existing.sinkTaskId; + manifest.streamType = existing.streamType; + manifest.firstSequence = existing.firstSequence; + manifest.lastSequence = existing.lastSequence; + manifest.lastDurableSequence = existing.lastDurableSequence; + manifest.recovered = existing.recovered; + manifest.segments = existing.segments == null ? new ArrayList<>() : existing.segments; + manifest.operationCounts = + existing.operationCounts == null + ? new java.util.LinkedHashMap<>() + : existing.operationCounts; + manifest.createdAt = existing.createdAt; + manifest.closedAt = existing.closedAt; + manifest.fsyncPolicy = existing.fsyncPolicy; + manifest.segmentSizeBytes = existing.segmentSizeBytes; + manifest.maxRecordBytes = existing.maxRecordBytes; + manifest.lastEventGroupId = existing.lastEventGroupId; + manifest.lastEventDigest = existing.lastEventDigest; + manifest.lastEventFirstSequence = existing.lastEventFirstSequence; + manifest.skippedEventCount = existing.skippedEventCount; + } + + private void validateManifestIdentity( + final String field, final String expected, final String actual) throws IOException { + if (expected != null && !Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void validateManifestConfiguration( + final String field, final long expected, final long actual) throws IOException { + if (expected != actual) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void acquireLock() throws IOException { + try { + if (Files.isSymbolicLink(lockPath)) { + throw symbolicLinkException(lockPath); + } + lockChannel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + directoryLock = lockChannel.tryLock(); + if (directoryLock == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory)); + } + lockChannel.truncate(0); + lockChannel.write( + StandardCharsets.UTF_8.encode(Long.toString(ProcessHandle.current().pid()))); + lockChannel.force(true); + } catch (final OverlappingFileLockException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory), + e); + } + } + + private void releaseLock() { + try { + if (directoryLock != null) { + directoryLock.release(); + directoryLock = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + try { + if (lockChannel != null) { + lockChannel.close(); + lockChannel = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + } + + private void writeFully(final ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } + + private void ensureOpen() throws IOException { + if (closed || channel == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB); + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + try { + if (channel != null) { + if (!streamEnded) { + writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), ""); + } + sealSegment(); + } + manifest.status = manifest.recovered ? "RECOVERED" : "SEALED"; + manifest.closedAt = Instant.now().toString(); + manifest.lastDurableSequence = manifest.lastSequence; + writeManifest(); + } finally { + if (channel != null) { + channel.close(); + channel = null; + } + closed = true; + releaseLock(); + } + } + + private void rollBeforeEventIfNecessary( + final List requests, final String metadata) throws IOException { + final long metadataBytes = metadata.getBytes(StandardCharsets.UTF_8).length; + long requiredBytes = frameSize(metadataBytes, 0) * 2; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + requiredBytes += frameSize(metadataBytes, request.getBody().length); + } + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private void rollBeforeControlIfNecessary(final String metadata) throws IOException { + final long requiredBytes = frameSize(metadata.getBytes(StandardCharsets.UTF_8).length, 0); + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private static long frameSize(final long metadataBytes, final long payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String computeEventDigest(final List requests) + throws IOException { + final MessageDigest digest = LogicalBackupFormat.newSha256(); + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + updateEventDigest(digest, request.getVersion(), request.getType(), request.getBody()); + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + private static void updateEventDigest( + final MessageDigest digest, + final byte requestVersion, + final short requestType, + final byte[] body) { + digest.update(requestVersion); + digest.update( + ByteBuffer.allocate(Short.BYTES).order(ByteOrder.BIG_ENDIAN).putShort(requestType).array()); + digest.update(body); + } + + private void rollbackTo(final long position) throws IOException { + final Path segmentPath = currentSegmentPath(); + channel.truncate(position); + channel.force(true); + channel.close(); + channel = null; + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + rebuildManifestFromSegments(); + manifest.recovered = true; + manifest.status = "RECOVERED"; + manifest.lastDurableSequence = manifest.lastSequence; + operationsSinceFsync = 0; + lastFsyncAt = System.currentTimeMillis(); + updateActiveSegmentManifest(); + writeManifest(); + } + + private void rebuildManifestFromSegments() throws IOException { + manifest.firstSequence = -1; + manifest.lastSequence = -1; + manifest.lastEventGroupId = null; + manifest.lastEventDigest = null; + manifest.lastEventFirstSequence = -1; + manifest.skippedEventCount = 0; + manifest.operationCounts = new LinkedHashMap<>(); + + long expectedSequence = -1; + long expectedSegmentId = -1; + for (int segmentIndex = 0; segmentIndex < manifest.segments.size(); segmentIndex++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(segmentIndex); + final Path segmentPath = resolveExistingSegmentPath(segment.file); + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segment.segmentId) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if ((expectedSegmentId >= 0 && scan.getSegmentId() != expectedSegmentId) + || (!scan.isSealed() && segmentIndex != manifest.segments.size() - 1)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + expectedSegmentId = scan.getSegmentId() + 1; + if (scan.isSealed() && "SEALED".equals(segment.status)) { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + segment.firstSequence = -1; + segment.lastSequence = -1; + segment.recordCount = scan.getRecords().size(); + segment.sizeBytes = scan.getValidLength(); + segment.status = scan.isSealed() ? "SEALED" : "ACTIVE"; + segment.sha256 = + scan.isSealed() ? LogicalBackupFormat.toHex(scan.getFooter().getDigest()) : null; + + MessageDigest eventDigest = null; + UUID eventGroupId = null; + long eventFirstSequence = -1; + for (final LogicalBackupRecord record : scan.getRecords()) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149, + segmentPath)); + } + expectedSequence = record.getSequence() + 1; + segment.firstSequence = + segment.firstSequence < 0 ? record.getSequence() : segment.firstSequence; + segment.lastSequence = record.getSequence(); + manifest.firstSequence = + manifest.firstSequence < 0 ? record.getSequence() : manifest.firstSequence; + manifest.lastSequence = record.getSequence(); + manifest.operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + if (record.getRecordType() == LogicalBackupRecordType.SKIPPED_EVENT) { + manifest.skippedEventCount++; + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventDigest = LogicalBackupFormat.newSha256(); + eventGroupId = record.getEventGroupId(); + eventFirstSequence = record.getSequence(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + updateEventDigest( + eventDigest, + record.getRequestVersion(), + record.getRequestType(), + record.getPayload()); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = LogicalBackupFormat.toHex(eventDigest.digest()); + manifest.lastEventFirstSequence = eventFirstSequence; + eventDigest = null; + eventGroupId = null; + eventFirstSequence = -1; + } + } + } + nextSequence = Math.max(0, manifest.lastSequence + 1); + streamEnded = + manifest.lastSequence >= 0 + && !manifest.segments.isEmpty() + && isLastRecordStreamEnd(reader.scan(currentSegmentPath(), false).getRecords()); + } + + private static boolean isLastRecordStreamEnd(final List records) { + return !records.isEmpty() + && records.get(records.size() - 1).getRecordType() == LogicalBackupRecordType.STREAM_END; + } + + private Path currentSegmentPath() throws IOException { + if (manifest.segments.isEmpty()) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15); + } + return resolveSegmentPath(manifest.segments.get(manifest.segments.size() - 1).file); + } + + private Path resolveSegmentPath(final String file) throws IOException { + final Path relativePath; + try { + relativePath = file == null ? null : Path.of(file); + } catch (final RuntimeException e) { + throw invalidSegmentPath(file, e); + } + final Path normalizedDirectory = directory.toAbsolutePath().normalize(); + if (relativePath == null || relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + throw invalidSegmentPath(file, null); + } + final Path resolved = normalizedDirectory.resolve(relativePath).normalize(); + if (!resolved.startsWith(normalizedDirectory) + || !normalizedDirectory.equals(resolved.getParent())) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private Path resolveExistingSegmentPath(final String file) throws IOException { + final Path resolved = resolveSegmentPath(file); + if (Files.isSymbolicLink(resolved) + || !Files.isRegularFile(resolved, LinkOption.NOFOLLOW_LINKS)) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private static IOException invalidSegmentPath(final String file, final Throwable cause) { + final IOException exception = + new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + file)); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + + private void updateActiveSegmentManifest() throws IOException { + if (channel == null || manifest.segments.isEmpty()) { + return; + } + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.status = "ACTIVE"; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java new file mode 100644 index 0000000000000..05f6298e95fac --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupArchiveReaderTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + + final List streams = new LogicalBackupArchiveReader().read(root, false); + + Assert.assertEquals(2, streams.size()); + Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId); + Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + Assert.assertEquals(4, streams.get(0).getRecords().size()); + Assert.assertEquals( + request(1), + streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq()); + } + + @Test + public void testManifestCounterMismatchIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.operationCounts.clear(); + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testUnlistedSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + Files.copy(onlySegment(directory), directory.resolve("unlisted.pwal")); + + assertReadFails(directory, false); + } + + @Test + public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, false); + final List streams = new LogicalBackupArchiveReader().read(directory, true); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + } + + @Test + public void testUnsealedNonLastSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 500); + final List segments = segments(directory); + Assert.assertTrue(segments.size() > 1); + try (final RandomAccessFile file = new RandomAccessFile(segments.get(0).toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, true); + } + + @Test + public void testParentSegmentPathIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testSkippedEventRequiresExplicitIncompleteOptIn() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest("stream", "data"); + try (final LogicalBackupWriter writer = writer(directory, manifest, 4096)) { + writer.recordSkippedEvent(1, "unsupported-event"); + } + + assertReadFails(directory, false); + Assert.assertEquals(1, new LogicalBackupArchiveReader().read(directory, true).size()); + + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest persistedManifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + persistedManifest.skippedEventCount = 0; + Files.writeString(manifestPath, GSON.toJson(persistedManifest), StandardCharsets.UTF_8); + assertReadFails(directory, false); + } + + @Test + public void testMixedBackupSessionsAreRejected() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + final Path manifestPath = + root.resolve("schema").resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.backupId = "another-backup"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(root, false); + } + + @Test + public void testNonReplayableRequestTypeIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Collections.singletonList( + new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 200) + .setBody(ByteBuffer.wrap(new byte[] {1}))), + "metadata"); + } + + assertReadFails(directory, false); + } + + @Test + public void testDuplicateEventGroupIdIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventGroupId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent(eventGroupId, 1, Collections.singletonList(request(1)), "metadata"); + writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "metadata"); + writer.writeEvent(eventGroupId, 3, Collections.singletonList(request(1)), "metadata"); + } + + assertReadFails(directory, false); + } + + private static void writeStream( + final Path directory, final String streamId, final String streamType, final long segmentSize) + throws Exception { + try (final LogicalBackupWriter writer = + writer(directory, manifest(streamId, streamType), segmentSize)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + if (segmentSize < 1000) { + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "metadata"); + } + } + } + + private static LogicalBackupManifest manifest(final String streamId, final String streamType) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = streamId; + manifest.streamType = streamType; + manifest.timestampPrecision = "ms"; + return manifest; + } + + private static LogicalBackupWriter writer( + final Path directory, final LogicalBackupManifest manifest, final long segmentSize) + throws IOException { + return new LogicalBackupWriter( + directory, + manifest, + segmentSize, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(body)); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + return files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .collect(java.util.stream.Collectors.toList()); + } + } + + private static void assertReadFails(final Path source, final boolean allowIncomplete) + throws IOException { + try { + new LogicalBackupArchiveReader().read(source, allowIncomplete); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java new file mode 100644 index 0000000000000..b6806f3407be0 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -0,0 +1,588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.AbstractList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupWriterTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testRoundTripAndDuplicateEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + final TPipeTransferReq request = request((byte) 1, (short) 2, new byte[] {3, 4, 5}); + + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + } + + final LogicalBackupSegmentReader.ScanResult result = scanOnlySegment(directory); + Assert.assertTrue(result.isSealed()); + Assert.assertEquals(4, result.getRecords().size()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_BEGIN, result.getRecords().get(0).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.PIPE_REQUEST, result.getRecords().get(1).getRecordType()); + Assert.assertEquals(request, result.getRecords().get(1).toTPipeTransferReq()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_COMMIT, result.getRecords().get(2).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.STREAM_END, result.getRecords().get(3).getRecordType()); + Assert.assertEquals(0, result.getFooter().getFirstSequence()); + Assert.assertEquals(3, result.getFooter().getLastSequence()); + } + + @Test + public void testDuplicateEventWithDifferentDigestFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(eventId, 1, Collections.singletonList(request(1)), ""); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(request(2)), ""); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(eventId.toString())); + } + } + } + + @Test + public void testOnlineRollbackAfterPartialEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final List requests = + new AbstractList() { + private int accesses; + + @Override + public TPipeTransferReq get(final int index) { + if (++accesses == 6) { + throw new IllegalStateException(); + } + return request(index + 1); + } + + @Override + public int size() { + return 2; + } + }; + try { + writer.writeEvent(UUID.randomUUID(), 1, requests, ""); + Assert.fail(); + } catch (final IllegalStateException expected) { + // Expected. + } + Assert.assertEquals( + 0, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(3)), "after-rollback")); + } + + final List records = scanOnlySegment(directory).getRecords(); + Assert.assertEquals(4, records.size()); + Assert.assertEquals(0, records.get(0).getSequence()); + Assert.assertEquals("after-rollback", records.get(0).getMetadata()); + } + + @Test + public void testRecoverIncompleteEventAndRebuildCounts() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + final long beginAndRequestLength = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + frameSize(0, 0) + frameSize(0, 1); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(beginAndRequestLength); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_BEGIN")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("PIPE_REQUEST")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_COMMIT")); + } + } + + @Test + public void testRecoverIncompleteFrame() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 2); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(2, writer.getManifest().lastSequence); + } + } + + @Test + public void testRecoverPartialFooter() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE + 8); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals( + 4, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES) + .scan(segments(directory).get(segments(directory).size() - 1), false) + .isSealed()); + } + + @Test + public void testRecoverCorruptFooterWhenManifestStillMarksSegmentActive() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long lastByteOffset = file.length() - 1; + file.seek(lastByteOffset); + final int lastByte = file.readUnsignedByte(); + file.seek(lastByteOffset); + file.writeByte(lastByte ^ 1); + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + manifest.status = "WRITING"; + manifest.closedAt = null; + manifest.segments.get(0).status = "ACTIVE"; + manifest.segments.get(0).sizeBytes -= LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + manifest.segments.get(0).sha256 = null; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + } + Assert.assertTrue(scanOnlySegment(directory).isSealed()); + } + + @Test + public void testChecksumDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long payloadOffset = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + + frameSize("metadata".length(), 0) + + LogicalBackupFormat.RECORD_HEADER_SIZE + + "metadata".length(); + file.seek(payloadOffset); + file.writeByte(file.readByte() ^ 1); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("CRC")); + } + } + + @Test + public void testFooterSequenceDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long footerOffset = file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + final byte[] footer = new byte[LogicalBackupFormat.SEGMENT_FOOTER_SIZE]; + file.seek(footerOffset); + file.readFully(footer); + ByteBuffer.wrap(footer).order(ByteOrder.BIG_ENDIAN).putLong(Integer.BYTES + Long.BYTES, 99); + ByteBuffer.wrap(footer) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + footer, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES)); + file.seek(footerOffset); + file.write(footer); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("metadata")); + } + } + + @Test + public void testSegmentRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + } + final List segments = segments(directory); + Assert.assertEquals(2, segments.size()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(0), false).isSealed()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed()); + } + + @Test + public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest(); + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest, + 500, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.NONE, + 1, + 1, + false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + Assert.assertEquals(2, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testControlRecordRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 258, false)) { + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first"); + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second"); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals(1, writer.getManifest().lastSequence); + } + } + + @Test + public void testDirectoryLock() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("locked")); + } + } + } + + @Test + public void testFailIfDirectoryExists() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath(); + try { + writer(directory, 4096, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(directory.toString())); + } + } + + @Test + public void testAppendRejectsConfigurationMismatch() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + try { + writer(directory, 8192, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segmentSizeBytes")); + } + } + + @Test + public void testAppendRejectsParentSegmentPath() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + final Path directory = root.resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + final String originalSegment = manifest.segments.get(0).file; + Files.copy(directory.resolve(originalSegment), root.resolve("outside.pwal")); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segment path")); + } + + manifest.segments.get(0).file = originalSegment; + Files.writeString(manifestPath, GSON.toJson(manifest)); + try (final LogicalBackupWriter ignored = writer(directory, 4096, true)) { + // A failed append must release the directory lock. + } + } + + @Test + public void testAppendAdoptsOrphanHeaderOnlySegmentAndClearsClosedState() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + Files.write(orphan, segmentHeader(1, 123)); + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals("WRITING", writer.getManifest().status); + Assert.assertNull(writer.getManifest().closedAt); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals( + 1, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + + final LogicalBackupSegmentReader.ScanResult scan = + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(orphan, false); + Assert.assertTrue(scan.isSealed()); + Assert.assertEquals(123, scan.getCreatedAt()); + } + + @Test + public void testAppendRejectsNonEmptyOrphanSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + final byte[] invalidOrphan = + Arrays.copyOf(segmentHeader(1, 123), LogicalBackupFormat.SEGMENT_HEADER_SIZE + 1); + Files.write(orphan, invalidOrphan); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + // The extra byte is an incomplete record tail and must never be adopted. + } + } + + @Test + public void testAppendRejectsUnexpectedUnlistedSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path unexpected = directory.resolve(segmentFileName(99)); + Files.write(unexpected, segmentHeader(99, 123)); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("Unlisted")); + } + } + + @Test + public void testPeriodicFsyncAdvancesOnHeartbeat() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest, + 4096, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.PERIODIC, + 1000, + 100, + false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + Assert.assertEquals(-1, writer.getManifest().lastDurableSequence); + Thread.sleep(150); + writer.heartbeat(); + Assert.assertEquals( + writer.getManifest().lastSequence, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testMaxRecordBytesHasHardLimit() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + try { + new LogicalBackupWriter( + directory, + manifest, + 4096, + LogicalBackupFormat.MAX_RECORD_BYTES + 1, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + private static LogicalBackupWriter writer( + final Path directory, final long segmentSize, final boolean append) throws IOException { + return new LogicalBackupWriter( + directory, + manifest(), + segmentSize, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + append); + } + + private static LogicalBackupManifest manifest() { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + return manifest; + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return request((byte) 1, (short) 2, body); + } + + private static TPipeTransferReq request(final byte version, final short type, final byte[] body) { + return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body)); + } + + private static LogicalBackupSegmentReader.ScanResult scanOnlySegment(final Path directory) + throws IOException { + return new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(onlySegment(directory), false); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + final Path[] segments = + files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .toArray(Path[]::new); + return Arrays.asList(segments); + } + } + + private static long frameSize(final int metadataBytes, final int payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String segmentFileName(final long segmentId) { + return String.format(java.util.Locale.ROOT, "segment-%020d.pwal", segmentId); + } + + private static byte[] segmentHeader(final long segmentId, final long createdAt) { + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(createdAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES)); + return bytes; + } +} diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-pipe-logical-backup.sh new file mode 100644 index 0000000000000..8a0a4f8a1834b --- /dev/null +++ b/scripts/tools/export-pipe-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@" diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-pipe-logical-backup.sh new file mode 100644 index 0000000000000..6e4e069210ded --- /dev/null +++ b/scripts/tools/import-pipe-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@" diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/pipe-logical-backup.sh new file mode 100644 index 0000000000000..a0f6a3529f746 --- /dev/null +++ b/scripts/tools/pipe-logical-backup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [ -n "${IOTDB_INCLUDE}" ] && [ -r "${IOTDB_INCLUDE}" ]; then + . "${IOTDB_INCLUDE}" +fi + +if [ -z "${IOTDB_HOME}" ]; then + IOTDB_HOME="$(cd "$(dirname "$0")/.."; pwd)" + export IOTDB_HOME +fi + +if [ -n "${JAVA_HOME}" ] && [ -x "${JAVA_HOME}/bin/java" ]; then + JAVA="${JAVA_HOME}/bin/java" +else + JAVA=java +fi + +CLASSPATH="" +for jar in "${IOTDB_HOME}"/lib/*.jar; do + CLASSPATH="${CLASSPATH}:${jar}" +done + +exec "${JAVA}" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 \ + -DIOTDB_HOME="${IOTDB_HOME}" -cp "${CLASSPATH}" \ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool "$@" diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-pipe-logical-backup.bat new file mode 100644 index 0000000000000..42e1df8afb274 --- /dev/null +++ b/scripts/tools/windows/export-pipe-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0pipe-logical-backup.bat" export %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-pipe-logical-backup.bat new file mode 100644 index 0000000000000..0fba332d5a2e5 --- /dev/null +++ b/scripts/tools/windows/import-pipe-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0pipe-logical-backup.bat" import %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/pipe-logical-backup.bat new file mode 100644 index 0000000000000..14e3180e46994 --- /dev/null +++ b/scripts/tools/windows/pipe-logical-backup.bat @@ -0,0 +1,40 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0..\.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED JAVA_HOME goto :no_java +set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +"%JAVA_HOME%\bin\java" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 ^ + -DIOTDB_HOME="%IOTDB_HOME%" -cp %CLASSPATH% ^ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool %* +set ret_code=%ERRORLEVEL% +goto :finally + +:no_java +echo JAVA_HOME environment variable must be set! +set ret_code=1 + +:finally +endlocal & exit /B %ret_code% From ce4b30d0ea26d433ca780e03fb691100ab15c268 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:03:37 +0800 Subject: [PATCH 2/3] Fix logical backup tablet serialization error handling --- .../logicalbackup/LogicalBackupSink.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 5050107c05f98..594d49922f7e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -51,6 +51,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -389,13 +390,17 @@ public void transfer(final TsFileInsertionEvent event) throws Exception { try { tsFile.consumeTabletInsertionEventsWithRetry( tablet -> { - requests.add( - PipeTransferTabletRawReqV2.toTPipeTransferReq( - tablet.convertToTablet(), - tablet.isAligned(), - tablet.isTableModelEvent() - ? tablet.getTableModelDatabaseName() - : tablet.getTreeModelDatabaseName())); + try { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablet.convertToTablet(), + tablet.isAligned(), + tablet.isTableModelEvent() + ? tablet.getTableModelDatabaseName() + : tablet.getTreeModelDatabaseName())); + } catch (final IOException e) { + throw new PipeException(e.getMessage(), e); + } }, getClass().getName()); if (!requests.isEmpty()) { From 218dcb43001c4f99f5905d0cba032ee7e62478c1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:06:00 +0800 Subject: [PATCH 3/3] Harden logical backup failover and import recovery --- .../apache/iotdb/cli/i18n/CliMessages.java | 5 +- .../apache/iotdb/cli/i18n/CliMessages.java | 5 +- .../tool/pipe/PipeLogicalBackupTool.java | 191 +++++++++++++----- .../tool/pipe/PipeLogicalBackupToolTest.java | 111 +++++++++- .../iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../logicalbackup/LogicalBackupSink.java | 25 +++ .../logicalbackup/LogicalBackupSinkTest.java | 79 ++++++++ .../LogicalBackupWriterTest.java | 30 +++ 9 files changed, 390 insertions(+), 62 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 77888e65768a0..8aa2feccc907a 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -130,9 +130,8 @@ private CliMessages() {} public static final String LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; - public static final String - LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = - "pipe-logical-backup "; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "Logical backup archive entry is unsafe: %s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 530b504b29ce0..35346133f19e5 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -126,9 +126,8 @@ private CliMessages() {} public static final String LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; - public static final String - LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = - "pipe-logical-backup "; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "逻辑备份归档项路径不安全：%s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 5d80243f3adf5..1d2315f17c04a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -80,8 +80,6 @@ public final class PipeLogicalBackupTool { private static final String COMMAND_VERIFY = "verify"; private static final String COMMAND_EXPORT = "export"; private static final String COMMAND_IMPORT = "import"; - private static final String COMMAND_RESTORE = "restore"; - private static final String COMMAND_STATS = "stats"; private static final String OPTION_INPUT = "input"; private static final String OPTION_OUTPUT = "output"; private static final String OPTION_RESUME = "resume"; @@ -137,10 +135,7 @@ static int run(final String[] args) throws Exception { case COMMAND_EXPORT: return export(line); case COMMAND_IMPORT: - case COMMAND_RESTORE: return importBackup(line); - case COMMAND_STATS: - return inspect(line); default: throw new ParseException( String.format( @@ -352,19 +347,13 @@ private static int importBackup(final CommandLine line) throws Exception { handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); long importedGroups = 0; for (final BackupStream stream : streams) { - final String streamId = stream.getManifest().streamId; - final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); - for (final EventGroup group : stream.getEventGroups()) { - if (group.getLastSequence() <= lastApplied) { - continue; - } - for (final LogicalBackupRecord record : group.getRequests()) { - transfer(client, record.toTPipeTransferReq()); - } - checkpointState.appliedSequences.put(streamId, group.getLastSequence()); - writeCheckpoint(checkpoint, checkpointState); - importedGroups++; - } + importedGroups += + importStream( + checkpoint, + checkpointState, + stream, + request -> transfer(client, request), + state -> writeCheckpoint(checkpoint, state)); } writeCheckpoint(checkpoint, checkpointState); System.out.println( @@ -377,6 +366,49 @@ private static int importBackup(final CommandLine line) throws Exception { return 0; } + static long importStream( + final Path checkpoint, + final ImportCheckpoint checkpointState, + final BackupStream stream, + final RequestTransfer requestTransfer, + final CheckpointPersister checkpointPersister) + throws Exception { + final String streamId = stream.getManifest().streamId; + final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); + long importedGroups = 0; + for (final EventGroup group : stream.getEventGroups()) { + if (group.getLastSequence() <= lastApplied) { + continue; + } + + int nextRequestIndex = 0; + if (checkpointState.inProgress == null) { + checkpointState.inProgress = ImportProgress.start(streamId, group); + checkpointPersister.persist(checkpointState); + } else if (checkpointState.inProgress.matches(streamId, group)) { + nextRequestIndex = checkpointState.inProgress.nextRequestIndex; + } else { + throw invalidCheckpoint(checkpoint); + } + + final List requests = group.getRequests(); + if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) { + throw invalidCheckpoint(checkpoint); + } + for (int index = nextRequestIndex; index < requests.size(); index++) { + requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + checkpointState.inProgress.nextRequestIndex = index + 1; + checkpointPersister.persist(checkpointState); + } + + checkpointState.appliedSequences.put(streamId, group.getLastSequence()); + checkpointState.inProgress = null; + checkpointPersister.persist(checkpointState); + importedGroups++; + } + return importedGroups; + } + private static List read(final CommandLine line) throws IOException { final String source = input(line); try (final BackupInput input = BackupInput.open(Path.of(source))) { @@ -472,14 +504,15 @@ private static void handshake( } } - private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) + static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) throws TException, IOException { final TPipeTransferResp response = client.pipeTransfer(request); if (response == null || response.getStatus() == null || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() && response.getStatus().getCode() - != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) { + != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { throw new IOException( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, @@ -497,7 +530,7 @@ private static Path checkpointPath(final CommandLine line) throws IOException { return source.resolveSibling(source.getFileName() + ".import.checkpoint.json"); } - private static ImportCheckpoint readCheckpoint( + static ImportCheckpoint readCheckpoint( final Path checkpoint, final List streams, final String host, @@ -539,7 +572,7 @@ private static ImportCheckpoint readCheckpoint( || !host.equals(state.targetHost) || port != state.targetPort || !user.equals(state.targetUser) - || !checkpointSequencesAreValid(streams, state.appliedSequences)) { + || !checkpointStateIsValid(streams, state)) { throw new IOException( String.format( CliMessages @@ -615,34 +648,66 @@ private static void updateDigest(final MessageDigest digest, final long value) { digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array()); } - private static boolean checkpointSequencesAreValid( - final List streams, final Map appliedSequences) { + private static boolean checkpointStateIsValid( + final List streams, final ImportCheckpoint state) { + final Map appliedSequences = state.appliedSequences; if (appliedSequences.size() > streams.size()) { return false; } + boolean foundIncompleteStream = false; + boolean matchedInProgress = false; for (final BackupStream stream : streams) { - if (!appliedSequences.containsKey(stream.getManifest().streamId)) { - continue; - } - final Long applied = appliedSequences.get(stream.getManifest().streamId); - if (applied == null) { + final String streamId = stream.getManifest().streamId; + final Long applied = appliedSequences.getOrDefault(streamId, -1L); + if (applied == null || applied < -1) { return false; } - if (applied == -1) { - continue; + + int appliedGroupIndex = -1; + if (applied != -1) { + for (int index = 0; index < stream.getEventGroups().size(); index++) { + if (stream.getEventGroups().get(index).getLastSequence() == applied) { + appliedGroupIndex = index; + break; + } + } + if (appliedGroupIndex < 0) { + return false; + } } - if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) { + + final boolean streamComplete = appliedGroupIndex == stream.getEventGroups().size() - 1; + if (foundIncompleteStream && appliedGroupIndex >= 0) { return false; } + if (!streamComplete && !foundIncompleteStream) { + foundIncompleteStream = true; + if (state.inProgress != null) { + final EventGroup nextGroup = stream.getEventGroups().get(appliedGroupIndex + 1); + if (!state.inProgress.matches(streamId, nextGroup) + || state.inProgress.nextRequestIndex < 0 + || state.inProgress.nextRequestIndex > nextGroup.getRequests().size()) { + return false; + } + matchedInProgress = true; + } + } } - return appliedSequences.keySet().stream() - .allMatch( - streamId -> - streams.stream() - .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + return (state.inProgress == null || matchedInProgress) + && appliedSequences.keySet().stream() + .allMatch( + streamId -> + streams.stream() + .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + } + + private static IOException invalidCheckpoint(final Path checkpoint) { + return new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); } - private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) + static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) throws IOException { final Path parent = checkpoint.toAbsolutePath().normalize().getParent(); if (parent != null) { @@ -681,7 +746,7 @@ private static void printUsage() { formatter.printHelp( new PrintWriter(System.out), 120, - CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2, + CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE, CliMessages .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E, optionsForHelp(), @@ -710,14 +775,46 @@ private static Options optionsForHelp() { return options; } - private static class ImportCheckpoint { - private String formatName; - private String formatVersion; - private Map sourceStreams = new LinkedHashMap<>(); - private String targetHost; - private int targetPort; - private String targetUser; - private Map appliedSequences = new LinkedHashMap<>(); + static class ImportCheckpoint { + String formatName; + String formatVersion; + Map sourceStreams = new LinkedHashMap<>(); + String targetHost; + int targetPort; + String targetUser; + Map appliedSequences = new LinkedHashMap<>(); + ImportProgress inProgress; + } + + static class ImportProgress { + String streamId; + String eventGroupId; + long lastSequence; + int nextRequestIndex; + + static ImportProgress start(final String streamId, final EventGroup group) { + final ImportProgress progress = new ImportProgress(); + progress.streamId = streamId; + progress.eventGroupId = group.getEventGroupId().toString(); + progress.lastSequence = group.getLastSequence(); + return progress; + } + + boolean matches(final String candidateStreamId, final EventGroup group) { + return candidateStreamId.equals(streamId) + && group.getEventGroupId().toString().equals(eventGroupId) + && group.getLastSequence() == lastSequence; + } + } + + @FunctionalInterface + interface RequestTransfer { + void transfer(TPipeTransferReq request) throws Exception; + } + + @FunctionalInterface + interface CheckpointPersister { + void persist(ImportCheckpoint state) throws IOException; } private static class BackupInput implements AutoCloseable { diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index 0fe45b22d8970..6e7af8726ab35 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -19,27 +19,35 @@ package org.apache.iotdb.tool.pipe; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; import org.apache.commons.cli.ParseException; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; @@ -141,7 +149,93 @@ public void testPlaintextPasswordOptionIsRejected() throws Exception { } } + @Test + public void testRemovedCommandAliasesAreRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + for (final String command : Arrays.asList("restore", "stats")) { + try { + PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()}); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + } + + @Test + public void testTransferAcceptsIdempotentConflict() throws Exception { + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + final TPipeTransferReq request = request(1); + Mockito.when(client.pipeTransfer(request)) + .thenReturn( + new TPipeTransferResp( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); + + PipeLogicalBackupTool.transfer(client, request); + } + + @Test + public void testImportCheckpointResumesAtRequestGranularity() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source, Arrays.asList(request(1), request(2))); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final BackupStream stream = streams.get(0); + final Path checkpoint = root.resolve("import.checkpoint.json"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final List transferredBodies = new ArrayList<>(); + final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true); + + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> { + if (checkpointState.inProgress != null + && checkpointState.inProgress.nextRequestIndex == 1 + && failCheckpointOnce.getAndSet(false)) { + throw new IOException("simulated checkpoint failure"); + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); + }); + Assert.fail(); + } catch (final IOException expected) { + // The request succeeded, but advancing its checkpoint was interrupted. + } + + Assert.assertEquals(Collections.singletonList(1), transferredBodies); + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertNotNull(durableState.inProgress); + Assert.assertEquals(0, durableState.inProgress.nextRequestIndex); + + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies); + Assert.assertNull(durableState.inProgress); + Assert.assertEquals( + Long.valueOf(stream.getEventGroups().get(0).getLastSequence()), + durableState.appliedSequences.get(stream.getManifest().streamId)); + } + private static void writeBackup(final Path directory) throws Exception { + writeBackup(directory, Collections.singletonList(request(3))); + } + + private static void writeBackup(final Path directory, final List requests) + throws Exception { final LogicalBackupManifest manifest = new LogicalBackupManifest(); manifest.backupId = "backup"; manifest.pipeName = "pipe"; @@ -152,15 +246,14 @@ private static void writeBackup(final Path directory) throws Exception { try (final LogicalBackupWriter writer = new LogicalBackupWriter( directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { - writer.writeEvent( - UUID.randomUUID(), - 1, - Collections.singletonList( - new TPipeTransferReq() - .setVersion((byte) 1) - .setType((short) 10) - .setBody(ByteBuffer.wrap(new byte[] {3}))), - "metadata"); + writer.writeEvent(UUID.randomUUID(), 1, requests, "metadata"); } } + + private static TPipeTransferReq request(final int value) { + return new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(new byte[] {(byte) value})); + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index d530fc7cf1bd3..3a914bc80c763 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2647,6 +2647,9 @@ private DataNodePipeMessages() {} "Unsupported logical backup event policy: %s"; public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "Unsupported logical backup resume policy: %s"; + public static final String + EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = + "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "Unsupported Pipe event for logical backup: %s"; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 49575e46abb91..08803638ea60a 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2471,6 +2471,9 @@ private DataNodePipeMessages() {} "不支持的逻辑备份事件策略：%s"; public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "不支持的逻辑备份恢复策略：%s"; + public static final String + EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = + "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "逻辑备份不支持 Pipe 事件：%s"; public static final String diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 594d49922f7e0..ef152b58b7c34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -160,6 +160,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validateDataRegionResumePolicy(parameters); final String policy = parameters .getStringOrDefault( @@ -187,6 +188,29 @@ private static void validateSynonymAttributes( false); } + private void validateDataRegionResumePolicy(final PipeParameters parameters) + throws PipeParameterNotValidException { + if (!"data".equals(streamType)) { + return; + } + final boolean explicitlyConfigured = + parameters.hasAnyAttributes( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + final String resume = + parameters.getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), + PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE); + if (!explicitlyConfigured + || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) { + throw new PipeParameterNotValidException( + DataNodePipeMessages + .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8); + } + } + @Override public void customize( final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) @@ -253,6 +277,7 @@ public void customize( PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) .toLowerCase(Locale.ROOT); + validateDataRegionResumePolicy(parameters); final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); if (!append && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java new file mode 100644 index 0000000000000..37c3d622b4f55 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class LogicalBackupSinkTest { + + @Test + public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS); + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + + attributes.remove(PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + attributes.put( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + @Test + public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + new LogicalBackupSink("schema") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + private static void assertInvalid( + final LogicalBackupSink sink, final Map attributes) throws Exception { + try { + sink.validate(new PipeParameterValidator(new PipeParameters(new HashMap<>(attributes)))); + Assert.fail(); + } catch (final PipeParameterNotValidException expected) { + Assert.assertTrue(expected.getMessage().contains("append")); + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java index b6806f3407be0..ec7d5d63ed9fc 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -91,6 +91,31 @@ public void testDuplicateEventWithDifferentDigestFails() throws Exception { } } + @Test + public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID firstEventId = UUID.randomUUID(); + final TPipeTransferReq firstRequest = replayableRequest(1); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + Assert.assertEquals( + 4, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(replayableRequest(2)), "second")); + } + + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(2, streams.get(0).getEventGroups().size()); + } + @Test public void testOnlineRollbackAfterPartialEvent() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -522,6 +547,7 @@ private static LogicalBackupManifest manifest() { manifest.pipeCreationTime = 1; manifest.streamId = "stream"; manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; return manifest; } @@ -533,6 +559,10 @@ private static TPipeTransferReq request(final byte[] body) { return request((byte) 1, (short) 2, body); } + private static TPipeTransferReq replayableRequest(final int value) { + return request((byte) 1, (short) 10, new byte[] {(byte) value}); + } + private static TPipeTransferReq request(final byte version, final short type, final byte[] body) { return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body)); }