diff --git a/core/src/main/java/google/registry/config/files/premium/example.txt b/core/src/main/java/google/registry/config/files/premium/example.txt index 349c92e7ed2..ec9d45719a9 100644 --- a/core/src/main/java/google/registry/config/files/premium/example.txt +++ b/core/src/main/java/google/registry/config/files/premium/example.txt @@ -4,4 +4,4 @@ # These are manipulated using the "nomulus" tool # {create,update,delete,list}_premium_list commands. foo,USD 100 # comment after the item -bar,JPY 14 +bar,USD 14 diff --git a/core/src/main/java/google/registry/model/tld/label/PremiumList.java b/core/src/main/java/google/registry/model/tld/label/PremiumList.java index 52b94c2d237..9ba24591c85 100644 --- a/core/src/main/java/google/registry/model/tld/label/PremiumList.java +++ b/core/src/main/java/google/registry/model/tld/label/PremiumList.java @@ -200,16 +200,23 @@ PremiumEntry createFromLine(String originalLine) { List parts = Splitter.on(',').trimResults().splitToList(line); checkArgument(parts.size() == 2, "Could not parse line in premium list: %s", originalLine); List moneyParts = Splitter.on(' ').trimResults().splitToList(parts.get(1)); - if (moneyParts.size() == 2 && this.currency != null) { - if (!Money.parse(parts.get(1)).getCurrencyUnit().equals(this.currency)) { + BigDecimal price; + if (moneyParts.size() == 2) { + Money money = Money.parse(parts.get(1)); + // If no currency was explicitly set on the PremiumList prior to parsing (e.g., when creating + // a new list via `update_premium_list --upsert` or validating a raw file), infer the list's + // currency from the first entry that includes a currency code and enforce that all + // subsequent entries use the same currency. + if (this.currency == null) { + this.currency = money.getCurrencyUnit(); + } else if (!money.getCurrencyUnit().equals(this.currency)) { throw new IllegalArgumentException( String.format("The currency unit must be %s", this.currency.getCode())); } + price = money.getAmount(); + } else { + price = new BigDecimal(parts.get(1)); } - BigDecimal price = - moneyParts.size() == 2 - ? Money.parse(parts.get(1)).getAmount() - : new BigDecimal(parts.get(1)); return new PremiumEntry.Builder() .setLabel(parts.get(0)) .setPrice(price) diff --git a/core/src/main/java/google/registry/model/tld/label/PremiumListUtils.java b/core/src/main/java/google/registry/model/tld/label/PremiumListUtils.java index d578077224d..9e243f909a9 100644 --- a/core/src/main/java/google/registry/model/tld/label/PremiumListUtils.java +++ b/core/src/main/java/google/registry/model/tld/label/PremiumListUtils.java @@ -14,6 +14,8 @@ package google.registry.model.tld.label; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import google.registry.model.tld.label.PremiumList.PremiumEntry; @@ -21,13 +23,32 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; import org.joda.money.CurrencyUnit; /** Static utility methods for {@link PremiumList}. */ public class PremiumListUtils { + /** + * Parses the given CSV input lines into a {@link PremiumList}. + * + *

The {@code currencyUnit} is explicitly passed in when creating a list via {@code + * create_premium_list} (which requires the {@code --currency} flag) or when updating an existing + * list in the database (to ensure the updated entries match the existing list's persisted + * currency, and to support bare-number price entries that omit the currency code). + * + *

When {@code currencyUnit} is {@code null} (such as when creating a new list via {@code + * update_premium_list --upsert}, which does not take a {@code --currency} flag and has no + * existing database revision to consult), {@link PremiumList#createFromLine} infers the list's + * currency from the first input line that specifies a currency code and validates that all + * subsequent lines match it. If {@code currencyUnit} is {@code null} and no input line specifies + * a currency code, an {@link IllegalArgumentException} is thrown. + */ public static PremiumList parseToPremiumList( - String name, CurrencyUnit currencyUnit, List inputData, Instant creationTime) { + String name, + @Nullable CurrencyUnit currencyUnit, + List inputData, + Instant creationTime) { PremiumList partialPremiumList = new PremiumList.Builder() .setName(name) @@ -35,6 +56,9 @@ public static PremiumList parseToPremiumList( .setCreationTimestamp(creationTime) .build(); ImmutableMap prices = partialPremiumList.parse(inputData); + checkArgument( + partialPremiumList.getCurrency() != null, + "Could not determine currency for premium list from input file"); Map priceAmounts = Maps.transformValues(prices, PremiumEntry::getValue); return partialPremiumList.asBuilder().setLabelsToPrices(priceAmounts).build(); } diff --git a/core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java b/core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java index 249dd83f3ca..371a69f0eab 100644 --- a/core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java +++ b/core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java @@ -72,14 +72,9 @@ abstract class CreateOrUpdatePremiumListCommand extends ConfirmingCommand { public String execute() throws Exception { String message = String.format("Saved premium list %s with %d entries.", name, inputData.size()); - try { - logger.atInfo().log("Saving premium list for TLD %s.", name); - tm().transact(() -> PremiumListDao.save(name, currency, inputData)); - logger.atInfo().log(message); - } catch (Throwable e) { - message = "Unexpected error saving premium list from nomulus tool command."; - logger.atSevere().withCause(e).log(message); - } + logger.atInfo().log("Saving premium list for TLD %s.", name); + tm().transact(() -> PremiumListDao.save(name, currency, inputData)); + logger.atInfo().log(message); return message; } diff --git a/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java b/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java index 54837e6e851..d9d90ddcce9 100644 --- a/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java +++ b/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java @@ -71,14 +71,9 @@ protected String execute() { String.format( "Saved reserved list %s with %d entries.", name, reservedList.getReservedListEntries().size()); - try { - logger.atInfo().log("Saving reserved list for TLD %s.", name); - ReservedListDao.save(reservedList); - logger.atInfo().log(message); - } catch (Throwable e) { - message = "Unexpected error saving reserved list from nomulus tool command."; - logger.atSevere().withCause(e).log(message); - } + logger.atInfo().log("Saving reserved list for TLD %s.", name); + ReservedListDao.save(reservedList); + logger.atInfo().log(message); return message; } diff --git a/core/src/main/java/google/registry/tools/CreatePremiumListCommand.java b/core/src/main/java/google/registry/tools/CreatePremiumListCommand.java index a4b3bcde52f..4d5bf7e5e7c 100644 --- a/core/src/main/java/google/registry/tools/CreatePremiumListCommand.java +++ b/core/src/main/java/google/registry/tools/CreatePremiumListCommand.java @@ -16,6 +16,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static google.registry.model.tld.Tlds.assertTldExists; +import static google.registry.model.tld.label.PremiumListUtils.parseToPremiumList; import static google.registry.util.ListNamingUtils.convertFilePathToName; import static java.nio.charset.StandardCharsets.UTF_8; @@ -56,6 +57,8 @@ protected String prompt() throws Exception { + " (unless --override is specified), yet TLD %s does not exist"); } inputData = Files.readAllLines(inputFile, UTF_8); + checkArgument(!inputData.isEmpty(), "New premium list data cannot be empty"); + parseToPremiumList(name, currency, inputData, clock.now()); return String.format("Create new premium list for %s?", name); } } diff --git a/core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java b/core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java index e5813ecc136..ad8e6d8413c 100644 --- a/core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java +++ b/core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java @@ -15,50 +15,65 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; +import static google.registry.model.tld.label.PremiumListUtils.parseToPremiumList; import static google.registry.util.ListNamingUtils.convertFilePathToName; import static java.nio.charset.StandardCharsets.UTF_8; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.google.common.base.Strings; import google.registry.model.tld.label.PremiumList; import google.registry.model.tld.label.PremiumListDao; -import google.registry.model.tld.label.PremiumListUtils; import java.nio.file.Files; +import java.util.Optional; /** Command to safely update {@link PremiumList} in Database for a given TLD. */ @Parameters(separators = " =", commandDescription = "Update a PremiumList in Database.") class UpdatePremiumListCommand extends CreateOrUpdatePremiumListCommand { + @Parameter( + names = {"-u", "--upsert"}, + description = "Create the premium list if it does not already exist.") + boolean upsert; + // Indicates if there is a new change made by this command private boolean newChange = false; @Override protected String prompt() throws Exception { name = Strings.isNullOrEmpty(name) ? convertFilePathToName(inputFile) : name; - PremiumList existingList = - PremiumListDao.getLatestRevision(name) - .orElseThrow( - () -> - new IllegalArgumentException( - String.format( - "Could not update premium list %s because it doesn't exist", name))); + Optional existingList = PremiumListDao.getLatestRevision(name); + checkArgument( + upsert || existingList.isPresent(), + "Could not update premium list %s because it doesn't exist", + name); inputData = Files.readAllLines(inputFile, UTF_8); checkArgument(!inputData.isEmpty(), "New premium list data cannot be empty"); - currency = existingList.getCurrency(); + // If the list already exists in the database, pass its persisted currency so that all entries + // are validated against it. Otherwise (when creating a new list via --upsert), pass null so + // that parseToPremiumList infers the currency from the input file entries. PremiumList updatedPremiumList = - PremiumListUtils.parseToPremiumList(name, currency, inputData, clock.now()); + parseToPremiumList( + name, existingList.map(PremiumList::getCurrency).orElse(null), inputData, clock.now()); + currency = updatedPremiumList.getCurrency(); + if (existingList.isEmpty()) { + newChange = true; + return String.format( + "Create new premium list for %s?\n New List: %s", name, updatedPremiumList); + } if (!existingList + .get() .getLabelsToPrices() .entrySet() .equals(updatedPremiumList.getLabelsToPrices().entrySet())) { newChange = true; return String.format( "Update premium list for %s?\n Old List: %s\n New List: %s", - name, existingList, updatedPremiumList); + name, existingList.get(), updatedPremiumList); } else { return String.format( "This update contains no changes to the premium list for %s.\n List Contents: %s", - name, existingList); + name, existingList.get()); } } diff --git a/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java b/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java index 88bcc82a18e..5a0c4c2fa90 100644 --- a/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java +++ b/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java @@ -14,48 +14,64 @@ package google.registry.tools; +import static com.google.common.base.Preconditions.checkArgument; import static google.registry.util.DiffUtils.prettyPrintEntityDeepDiff; import static google.registry.util.ListNamingUtils.convertFilePathToName; import static java.nio.charset.StandardCharsets.UTF_8; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.google.common.base.Strings; import google.registry.model.tld.label.ReservedList; import java.nio.file.Files; import java.util.List; +import java.util.Optional; /** Command to safely update {@link ReservedList}. */ @Parameters(separators = " =", commandDescription = "Update a ReservedList.") final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand { + @Parameter( + names = {"-u", "--upsert"}, + description = "Create the reserved list if it does not already exist.") + boolean upsert; + // indicates if there is a new change made by this command private boolean newChange = true; @Override protected String prompt() throws Exception { name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name; - ReservedList existingReservedList = - ReservedList.get(name) - .orElseThrow( - () -> - new IllegalArgumentException( - String.format( - "Could not update reserved list %s because it doesn't exist.", name))); + Optional existingReservedList = ReservedList.get(name); + checkArgument( + upsert || existingReservedList.isPresent(), + "Could not update reserved list %s because it doesn't exist.", + name); List allLines = Files.readAllLines(input, UTF_8); - ReservedList.Builder updated = - existingReservedList.asBuilder().setReservedListMapFromLines(allLines); + if (existingReservedList.isEmpty()) { + newChange = true; + reservedList = + new ReservedList.Builder() + .setName(name) + .setReservedListMapFromLines(allLines) + .setCreationTimestamp(clock.now()) + .build(); + return String.format( + "Create new reserved list for %s?\n%s\nreservedListMap=%s\n", + name, reservedList, outputReservedListEntries(reservedList)); + } + ReservedList existing = existingReservedList.get(); + ReservedList.Builder updated = existing.asBuilder().setReservedListMapFromLines(allLines); reservedList = updated.build(); boolean reservedListEntriesChanged = - !existingReservedList - .getReservedListEntries() - .equals(reservedList.getReservedListEntries()); + !existing.getReservedListEntries().equals(reservedList.getReservedListEntries()); if (!reservedListEntriesChanged) { newChange = false; return "No entity changes to apply."; } return String.format("Update reserved list for %s?\n", name) + prettyPrintEntityDeepDiff( - existingReservedList.getReservedListEntries(), reservedList.getReservedListEntries()); + existing.getReservedListEntries(), reservedList.getReservedListEntries()); } @Override diff --git a/core/src/test/java/google/registry/model/tld/label/PremiumListUtilsTest.java b/core/src/test/java/google/registry/model/tld/label/PremiumListUtilsTest.java index 2e6594488c0..21d70d9f7bc 100644 --- a/core/src/test/java/google/registry/model/tld/label/PremiumListUtilsTest.java +++ b/core/src/test/java/google/registry/model/tld/label/PremiumListUtilsTest.java @@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.model.tld.label.PremiumListUtils.parseToPremiumList; +import static org.joda.money.CurrencyUnit.EUR; import static org.joda.money.CurrencyUnit.USD; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -56,6 +57,66 @@ void parseInputToPremiumList_throwsOnInconsistentCurrencies() { assertThat(thrown).hasMessageThat().isEqualTo("The currency unit must be USD"); } + @Test + void parseInputToPremiumList_nullCurrency_infersFromInput() { + PremiumList premiumList = + parseToPremiumList( + "testlist", + null, + ImmutableList.of("# leading comment", " ", "foo,EUR 99.50 # inline", "bar,EUR 30"), + SAMPLE_TIME); + assertThat(premiumList.getName()).isEqualTo("testlist"); + assertThat(premiumList.getCurrency()).isEqualTo(EUR); + assertThat(premiumList.getLabelsToPrices()) + .containsExactly("foo", twoDigits(99.50), "bar", twoDigits(30)); + } + + @Test + void parseInputToPremiumList_nullCurrency_throwsOnInconsistentCurrencies() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + parseToPremiumList( + "testlist", + null, + ImmutableList.of("foo,EUR 99.50", "bar,USD 30"), + SAMPLE_TIME)); + assertThat(thrown).hasMessageThat().isEqualTo("The currency unit must be EUR"); + } + + @Test + void parseInputToPremiumList_nullCurrency_throwsWhenCurrencyCannotBeDetermined() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + parseToPremiumList( + "testlist", + null, + ImmutableList.of("# comment only", "foo,99.50"), + SAMPLE_TIME)); + assertThat(thrown) + .hasMessageThat() + .isEqualTo("Could not determine currency for premium list from input file"); + } + + @Test + void parseInputToPremiumList_nullCurrency_throwsOnMalformedLine() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + parseToPremiumList( + "testlist", + null, + ImmutableList.of("malformed_line_without_comma"), + SAMPLE_TIME)); + assertThat(thrown) + .hasMessageThat() + .isEqualTo("Could not parse line in premium list: malformed_line_without_comma"); + } + private static BigDecimal twoDigits(double num) { return BigDecimal.valueOf((long) (num * 100.0), 2); } diff --git a/core/src/test/java/google/registry/tools/CreatePremiumListCommandTest.java b/core/src/test/java/google/registry/tools/CreatePremiumListCommandTest.java index 8e713ee95b5..05bcd6f9ada 100644 --- a/core/src/test/java/google/registry/tools/CreatePremiumListCommandTest.java +++ b/core/src/test/java/google/registry/tools/CreatePremiumListCommandTest.java @@ -15,12 +15,14 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.io.Files; import google.registry.model.tld.Tld; import google.registry.model.tld.label.PremiumListDao; import google.registry.testing.DatabaseHelper; +import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import org.junit.jupiter.api.BeforeEach; @@ -76,6 +78,54 @@ void commandPrompt_failureNoInputFile() { assertThrows(NullPointerException.class, command::prompt); } + @Test + void commandPrompt_failureEmptyInputFile() throws Exception { + Path emptyFile = tmpDir.resolve(TLD_TEST + "_empty.txt"); + Files.write(new byte[0], emptyFile.toFile()); + command.name = TLD_TEST; + command.inputFile = emptyFile; + command.currencyUnit = "USD"; + IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, command::prompt); + assertThat(thrown).hasMessageThat().isEqualTo("New premium list data cannot be empty"); + } + + @Test + void commandPrompt_failureMalformedInputFile() throws Exception { + Path malformedFile = tmpDir.resolve(TLD_TEST + "_malformed_prompt.txt"); + Files.asCharSink(malformedFile.toFile(), UTF_8).write("malformed_line_without_comma\n"); + command.name = TLD_TEST; + command.inputFile = malformedFile; + command.currencyUnit = "USD"; + IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, command::prompt); + assertThat(thrown).hasMessageThat().contains("Could not parse line in premium list"); + } + + @Test + void commandRun_failureMalformedInputFile() throws Exception { + File malformedFile = tmpDir.resolve(TLD_TEST + "_malformed.txt").toFile(); + Files.asCharSink(malformedFile, UTF_8).write("malformed_line_without_comma\n"); + assertThrows( + IllegalArgumentException.class, + () -> + runCommandForced( + "--name=" + TLD_TEST, "--input=" + malformedFile.getPath(), "--currency=USD")); + assertThat(PremiumListDao.getLatestRevision(TLD_TEST)).isEmpty(); + } + + @Test + void commandRun_failureCurrencyMismatchInInputFile() throws Exception { + File mismatchedFile = tmpDir.resolve(TLD_TEST + "_jpy.txt").toFile(); + Files.asCharSink(mismatchedFile, UTF_8).write("doge,JPY 9090\n"); + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + runCommandForced( + "--name=" + TLD_TEST, "--input=" + mismatchedFile.getPath(), "--currency=USD")); + assertThat(thrown).hasMessageThat().isEqualTo("The currency unit must be USD"); + assertThat(PremiumListDao.getLatestRevision(TLD_TEST)).isEmpty(); + } + @Test void commandPrompt_failurePremiumListAlreadyExists() { String randomStr = "random"; @@ -126,4 +176,32 @@ void testDryRun_doesNotCreateList() throws Exception { "--name=" + TLD_TEST, "--input=" + premiumTermsPath, "--currency=USD", "--dry_run"); assertThat(PremiumListDao.getLatestRevision(TLD_TEST)).isEmpty(); } + + @Test + void testFailure_runCommandOnProduction_noFlag() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + runCommandInEnvironment( + RegistryToolEnvironment.PRODUCTION, + "--name=" + TLD_TEST, + "--input=" + premiumTermsPath, + "--currency=USD")); + assertThat(thrown) + .hasMessageThat() + .isEqualTo("The --build_environment flag must be used when running in production"); + } + + @Test + void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception { + runCommandInEnvironment( + RegistryToolEnvironment.PRODUCTION, + "--name=" + TLD_TEST, + "--input=" + premiumTermsPath, + "--currency=USD", + "--build_environment", + "-f"); + assertThat(PremiumListDao.getLatestRevision(TLD_TEST)).isPresent(); + } } diff --git a/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java b/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java index 70d6e1202ac..7570e339357 100644 --- a/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java +++ b/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java @@ -187,4 +187,30 @@ void testDryRun_doesNotCreateList() throws Exception { "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--dry_run"); assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isEmpty(); } + + @Test + void testFailure_runCommandOnProduction_noFlag() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + runCommandInEnvironment( + RegistryToolEnvironment.PRODUCTION, + "--name=xn--q9jyb4c_common-reserved", + "--input=" + reservedTermsPath)); + assertThat(thrown) + .hasMessageThat() + .isEqualTo("The --build_environment flag must be used when running in production"); + } + + @Test + void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception { + runCommandInEnvironment( + RegistryToolEnvironment.PRODUCTION, + "--name=xn--q9jyb4c_common-reserved", + "--input=" + reservedTermsPath, + "--build_environment", + "-f"); + assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent(); + } } diff --git a/core/src/test/java/google/registry/tools/SyncDbObjectsTest.java b/core/src/test/java/google/registry/tools/SyncDbObjectsTest.java new file mode 100644 index 00000000000..00527afa9ae --- /dev/null +++ b/core/src/test/java/google/registry/tools/SyncDbObjectsTest.java @@ -0,0 +1,289 @@ +// Copyright 2026 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.util.BuildPathUtils.getProjectRoot; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Unit tests for {@code release/db-object-updater/sync_db_objects.sh}. */ +class SyncDbObjectsTest { + + private static final Path SCRIPT_PATH = + getProjectRoot().resolve("release/db-object-updater/sync_db_objects.sh"); + + @TempDir Path tmpDir; + + private Path fakeBinDir; + private Path invocationsLog; + private Path configDir; + private Path envDir; + private Path credentialFile; + + @BeforeEach + void beforeEach() throws IOException { + fakeBinDir = Files.createDirectories(tmpDir.resolve("bin")); + invocationsLog = tmpDir.resolve("invocations.log"); + configDir = Files.createDirectories(tmpDir.resolve("config")); + envDir = configDir.resolve("alpha"); + credentialFile = tmpDir.resolve("nomulus_tool_credential.json"); + Files.writeString(credentialFile, "{}", UTF_8); + installFakeJava("exit 0"); + } + + private void installFakeJava(String exitLogic) throws IOException { + Path fakeJava = fakeBinDir.resolve("java"); + String script = + String.join( + "\n", "#!/bin/bash", "echo \"$*\" >> \"" + invocationsLog + "\"", exitLogic, ""); + Files.writeString(fakeJava, script, UTF_8); + Files.setPosixFilePermissions(fakeJava, PosixFilePermissions.fromString("rwxr-xr-x")); + } + + private ScriptResult runScript(String... args) throws IOException, InterruptedException { + ImmutableList command = + ImmutableList.builder().add("/bin/bash", SCRIPT_PATH.toString()).add(args).build(); + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + String currentPath = System.getenv("PATH"); + pb.environment() + .put("PATH", fakeBinDir + (currentPath != null ? File.pathSeparator + currentPath : "")); + Process process = pb.start(); + String output = new String(process.getInputStream().readAllBytes(), UTF_8); + int exitCode = process.waitFor(); + ImmutableList invocations = + Files.exists(invocationsLog) + ? ImmutableList.copyOf(Files.readAllLines(invocationsLog, UTF_8)) + : ImmutableList.of(); + return new ScriptResult(exitCode, output, invocations); + } + + @Test + void testFailure_tooFewArguments() throws Exception { + ScriptResult result = runScript("alpha", credentialFile.toString(), "configure_tld"); + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.output()) + .contains( + "Expecting four parameters in order: " + + "env tools_credential nomulus_command config_file_directory"); + assertThat(result.invocations()).isEmpty(); + } + + @Test + void testFailure_tooManyArguments() throws Exception { + ScriptResult result = + runScript( + "alpha", credentialFile.toString(), "configure_tld", configDir.toString(), "extra_arg"); + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.output()) + .contains( + "Expecting four parameters in order: " + + "env tools_credential nomulus_command config_file_directory"); + assertThat(result.invocations()).isEmpty(); + } + + @Test + void testSuccess_configureTld_doesNotPassUpsert() throws Exception { + Files.createDirectories(envDir); + Path tldFile = envDir.resolve("example.yaml"); + Files.writeString(tldFile, "tldStr: example\n", UTF_8); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "configure_tld", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " configure_tld -i " + + tldFile + + " --force --build_environment"); + } + + @Test + void testSuccess_updatePremiumList_passesUpsert() throws Exception { + Files.createDirectories(envDir); + Path premiumFile = envDir.resolve("example.txt"); + Files.writeString(premiumFile, "foo,USD 100\n", UTF_8); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_premium_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_premium_list -i " + + premiumFile + + " --force --build_environment --upsert"); + } + + @Test + void testSuccess_updateReservedList_passesUpsert() throws Exception { + Files.createDirectories(envDir); + Path reservedFile = envDir.resolve("common_reserved.txt"); + Files.writeString(reservedFile, "baddies,FULLY_BLOCKED\n", UTF_8); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_reserved_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_reserved_list -i " + + reservedFile + + " --force --build_environment --upsert"); + } + + @Test + void testFailure_configureTldFails_exitsNonZeroImmediately() throws Exception { + Files.createDirectories(envDir); + Path tldFileA = envDir.resolve("a.yaml"); + Path tldFileB = envDir.resolve("b.yaml"); + Files.writeString(tldFileA, "tldStr: a\n", UTF_8); + Files.writeString(tldFileB, "tldStr: b\n", UTF_8); + installFakeJava("exit 1"); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "configure_tld", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " configure_tld -i " + + tldFileA + + " --force --build_environment"); + } + + @Test + void testFailure_updatePremiumListFails_exitsNonZeroImmediately() throws Exception { + Files.createDirectories(envDir); + Path fileA = envDir.resolve("a.txt"); + Path fileB = envDir.resolve("b.txt"); + Files.writeString(fileA, "foo,USD 10\n", UTF_8); + Files.writeString(fileB, "bar,USD 20\n", UTF_8); + installFakeJava("exit 1"); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_premium_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_premium_list -i " + + fileA + + " --force --build_environment --upsert"); + } + + @Test + void testFailure_updateReservedListFails_exitsNonZeroImmediately() throws Exception { + Files.createDirectories(envDir); + Path fileA = envDir.resolve("a.txt"); + Path fileB = envDir.resolve("b.txt"); + Files.writeString(fileA, "foo,FULLY_BLOCKED\n", UTF_8); + Files.writeString(fileB, "bar,FULLY_BLOCKED\n", UTF_8); + installFakeJava("exit 1"); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_reserved_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_reserved_list -i " + + fileA + + " --force --build_environment --upsert"); + } + + @Test + void testSuccess_emptyEnvironmentDirectory() throws Exception { + Files.createDirectories(envDir); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_premium_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()).isEmpty(); + } + + @Test + void testSuccess_missingEnvironmentDirectory() throws Exception { + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_premium_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()).isEmpty(); + } + + @Test + void testSuccess_directoryPathWithSpaces() throws Exception { + Path spacedConfigDir = Files.createDirectories(tmpDir.resolve("config with spaces")); + Path spacedEnvDir = Files.createDirectories(spacedConfigDir.resolve("alpha")); + Path premiumFile = spacedEnvDir.resolve("cad list.txt"); + Files.writeString(premiumFile, "maple,CAD 50\n", UTF_8); + + ScriptResult result = + runScript( + "alpha", credentialFile.toString(), "update_premium_list", spacedConfigDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_premium_list -i " + + premiumFile + + " --force --build_environment --upsert"); + } + + @Test + void testSuccess_multipleFiles_processedInOrder() throws Exception { + Files.createDirectories(envDir); + Path firstFile = envDir.resolve("a_first.txt"); + Path secondFile = envDir.resolve("b_second.txt"); + Files.writeString(firstFile, "foo,USD 10\n", UTF_8); + Files.writeString(secondFile, "bar,GBP 20\n", UTF_8); + + ScriptResult result = + runScript("alpha", credentialFile.toString(), "update_premium_list", configDir.toString()); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.invocations()) + .containsExactly( + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_premium_list -i " + + firstFile + + " --force --build_environment --upsert", + "-jar /nomulus.jar -e alpha --credential " + + credentialFile + + " update_premium_list -i " + + secondFile + + " --force --build_environment --upsert") + .inOrder(); + } + + private record ScriptResult(int exitCode, String output, ImmutableList invocations) {} +} diff --git a/core/src/test/java/google/registry/tools/UpdatePremiumListCommandTest.java b/core/src/test/java/google/registry/tools/UpdatePremiumListCommandTest.java index 73ca3e2d1c7..406e39199c2 100644 --- a/core/src/test/java/google/registry/tools/UpdatePremiumListCommandTest.java +++ b/core/src/test/java/google/registry/tools/UpdatePremiumListCommandTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.joda.money.CurrencyUnit.JPY; import static org.joda.money.CurrencyUnit.USD; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -31,6 +32,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; +import org.joda.money.Money; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -227,4 +229,151 @@ void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception { .comparingElementsUsing(immutableObjectCorrespondence("revisionId")) .containsExactly(PremiumEntry.create(0L, new BigDecimal("9999.00"), "eth")); } + + @Test + void commandRun_failureMalformedInputFile() throws Exception { + File malformedFile = tmpDir.resolve(TLD_TEST + "_malformed.txt").toFile(); + Files.asCharSink(malformedFile, UTF_8).write("malformed_line_without_comma\n"); + assertThrows( + IllegalArgumentException.class, + () -> runCommandForced("--name=" + TLD_TEST, "--input=" + malformedFile.getPath())); + } + + @Test + void commandRun_failureCurrencyMismatchInInputFile() throws Exception { + File mismatchedFile = tmpDir.resolve(TLD_TEST + "_jpy.txt").toFile(); + Files.asCharSink(mismatchedFile, UTF_8).write("doge,JPY 9090\n"); + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> runCommandForced("--name=" + TLD_TEST, "--input=" + mismatchedFile.getPath())); + assertThat(thrown).hasMessageThat().isEqualTo("The currency unit must be USD"); + } + + @Test + void commandPrompt_successCreateListWithUpsert() throws Exception { + File tmpFile = tmpDir.resolve("newlist.txt").toFile(); + Files.asCharSink(tmpFile, UTF_8).write("omg,USD 1234"); + command.inputFile = Paths.get(tmpFile.getPath()); + command.name = "newlist"; + command.upsert = true; + assertThat(command.prompt()).contains("Create new premium list for newlist?"); + } + + @Test + void commandRun_successCreateListWithUpsert_usd() throws Exception { + File tmpFile = tmpDir.resolve("newlist.txt").toFile(); + Files.asCharSink(tmpFile, UTF_8).write("eth,USD 9999"); + runCommandForced("--name=newlist", "--input=" + tmpFile.getPath(), "--upsert"); + + Optional list = PremiumListDao.getLatestRevision("newlist"); + assertThat(list).isPresent(); + assertThat(list.get().getCurrency()).isEqualTo(USD); + assertThat(PremiumListDao.loadAllPremiumEntries("newlist")) + .comparingElementsUsing(immutableObjectCorrespondence("revisionId")) + .containsExactly(PremiumEntry.create(0L, new BigDecimal("9999.00"), "eth")); + } + + @Test + void commandRun_successCreateListWithUpsertShortFlag_nonUsdAndComments() throws Exception { + File tmpFile = tmpDir.resolve("jpylist.txt").toFile(); + String content = + """ + # Leading comment + # Indented comment + + doge,JPY 9090 # inline comment + eth,JPY 100 + """; + Files.asCharSink(tmpFile, UTF_8).write(content); + runCommandForced("-i=" + tmpFile.getPath(), "-u"); + + Optional list = PremiumListDao.getLatestRevision("jpylist"); + assertThat(list).isPresent(); + assertThat(list.get().getCurrency()).isEqualTo(JPY); + assertThat(PremiumListDao.getPremiumPrice("jpylist", "doge")) + .hasValue(Money.ofMajor(JPY, 9090)); + assertThat(PremiumListDao.getPremiumPrice("jpylist", "eth")).hasValue(Money.ofMajor(JPY, 100)); + } + + @Test + void commandRun_successUpdateExistingListWithUpsert() throws Exception { + File tmpFile = tmpDir.resolve(String.format("%s.txt", TLD_TEST)).toFile(); + Files.asCharSink(tmpFile, UTF_8).write("eth,USD 9999"); + runCommandForced("--name=" + TLD_TEST, "--input=" + tmpFile.getPath(), "--upsert"); + + Optional list = PremiumListDao.getLatestRevision(TLD_TEST); + assertThat(list).isPresent(); + assertThat(list.get().getCurrency()).isEqualTo(USD); + assertThat(PremiumListDao.loadAllPremiumEntries(TLD_TEST)) + .comparingElementsUsing(immutableObjectCorrespondence("revisionId")) + .containsExactly(PremiumEntry.create(0L, new BigDecimal("9999.00"), "eth")); + } + + @Test + void commandDryRun_upsertNewList_noChangesMade() throws Exception { + File tmpFile = tmpDir.resolve("dryrunlist.txt").toFile(); + Files.asCharSink(tmpFile, UTF_8).write("eth,EUR 99.50"); + runCommandForced("--name=dryrunlist", "--input=" + tmpFile.getPath(), "--upsert", "--dry_run"); + + assertThat(PremiumListDao.getLatestRevision("dryrunlist")).isEmpty(); + } + + @Test + void commandRun_failureUpsertEmptyFile() throws Exception { + File emptyFile = tmpDir.resolve("empty.txt").toFile(); + Files.write(new byte[0], emptyFile); + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> runCommandForced("--name=empty", "--input=" + emptyFile.getPath(), "--upsert")); + assertThat(thrown).hasMessageThat().isEqualTo("New premium list data cannot be empty"); + assertThat(PremiumListDao.getLatestRevision("empty")).isEmpty(); + } + + @Test + void commandRun_failureUpsertCommentOnlyFile() throws Exception { + File commentFile = tmpDir.resolve("comments.txt").toFile(); + Files.asCharSink(commentFile, UTF_8).write("# only comment\n \n"); + assertThrows( + IllegalArgumentException.class, + () -> runCommandForced("--name=comments", "--input=" + commentFile.getPath(), "--upsert")); + assertThat(PremiumListDao.getLatestRevision("comments")).isEmpty(); + } + + @Test + void commandRun_failureUpsertMissingCurrencyToken() throws Exception { + File noCurrencyFile = tmpDir.resolve("nocurrency.txt").toFile(); + Files.asCharSink(noCurrencyFile, UTF_8).write("doge,9090\n"); + assertThrows( + IllegalArgumentException.class, + () -> + runCommandForced( + "--name=nocurrency", "--input=" + noCurrencyFile.getPath(), "--upsert")); + assertThat(PremiumListDao.getLatestRevision("nocurrency")).isEmpty(); + } + + @Test + void commandRun_failureUpsertMismatchedCurrenciesAcrossLines() throws Exception { + File mixedFile = tmpDir.resolve("mixed.txt").toFile(); + Files.asCharSink(mixedFile, UTF_8).write("doge,EUR 9090\neth,USD 100\n"); + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> runCommandForced("--name=mixed", "--input=" + mixedFile.getPath(), "--upsert")); + assertThat(thrown).hasMessageThat().isEqualTo("The currency unit must be EUR"); + assertThat(PremiumListDao.getLatestRevision("mixed")).isEmpty(); + } + + @Test + void commandRun_failureUpsertInvalidPriceLine() throws Exception { + File invalidFile = tmpDir.resolve("invalidprice.txt").toFile(); + Files.asCharSink(invalidFile, UTF_8).write("doge,USD not_a_price\n"); + assertThrows( + IllegalArgumentException.class, + () -> + runCommandForced( + "--name=invalidprice", "--input=" + invalidFile.getPath(), "--upsert")); + assertThat(PremiumListDao.getLatestRevision("invalidprice")).isEmpty(); + } } diff --git a/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java b/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java index 5593d569429..1a1b7658f65 100644 --- a/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java +++ b/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import google.registry.model.tld.label.ReservedList; +import google.registry.model.tld.label.ReservedListDao; import java.io.File; import java.nio.file.Paths; import org.junit.jupiter.api.BeforeEach; @@ -161,4 +162,58 @@ void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception { assertInStdout("baddies: null -> baddies,FULLY_BLOCKED"); assertInStdout("ford: null -> ford,FULLY_BLOCKED # random comment"); } + + @Test + void testSuccess_upsertCreatesNonExistentReservedList() throws Exception { + runCommandForced("--name=xn--q9jyb4c_new-reserved", "--input=" + reservedTermsPath, "--upsert"); + assertThat(ReservedList.get("xn--q9jyb4c_new-reserved")).isPresent(); + assertThat(ReservedListDao.getLatestRevision("xn--q9jyb4c_new-reserved")).isPresent(); + ReservedList reservedList = ReservedList.get("xn--q9jyb4c_new-reserved").get(); + assertThat(reservedList.getReservedListEntries()).hasSize(2); + assertThat(reservedList.getReservationInList("baddies")).hasValue(FULLY_BLOCKED); + assertThat(reservedList.getReservationInList("ford")).hasValue(FULLY_BLOCKED); + } + + @Test + void testSuccess_upsertShortFlag_defaultsToFileName() throws Exception { + File newReservedFile = tmpDir.resolve("soy_new-reserved.txt").toFile(); + String reservedTermsCsv = + loadFile(CreateOrUpdateReservedListCommandTestCase.class, "example_reserved_terms.csv"); + Files.asCharSink(newReservedFile, UTF_8).write(reservedTermsCsv); + + runCommandForced("-i=" + newReservedFile.getPath(), "-u"); + assertThat(ReservedList.get("soy_new-reserved")).isPresent(); + ReservedList reservedList = ReservedList.get("soy_new-reserved").get(); + assertThat(reservedList.getReservedListEntries()).hasSize(2); + assertThat(reservedList.getReservationInList("baddies")).hasValue(FULLY_BLOCKED); + assertThat(reservedList.getReservationInList("ford")).hasValue(FULLY_BLOCKED); + } + + @Test + void testSuccess_upsertUpdatesExistingReservedList() throws Exception { + runSuccessfulUpdateTest( + "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--upsert"); + } + + @Test + void testSuccess_upsertDryRun_doesNotCreateList() throws Exception { + runCommandForced( + "--name=xn--q9jyb4c_dryrun-reserved", + "--input=" + reservedTermsPath, + "--upsert", + "--dry_run"); + assertThat(ReservedList.get("xn--q9jyb4c_dryrun-reserved")).isEmpty(); + } + + @Test + void testFailure_upsertInvalidSyntax_throwsException() throws Exception { + File badFile = tmpDir.resolve("xn--q9jyb4c_bad-reserved.txt").toFile(); + Files.asCharSink(badFile, UTF_8).write("baddies,INVALID_TYPE\n"); + assertThrows( + IllegalArgumentException.class, + () -> + runCommandForced( + "--name=xn--q9jyb4c_bad-reserved", "--input=" + badFile.getPath(), "--upsert")); + assertThat(ReservedList.get("xn--q9jyb4c_bad-reserved")).isEmpty(); + } } diff --git a/release/cloudbuild-sync-db-objects.yaml b/release/cloudbuild-sync-db-objects.yaml index 29b336ad488..a89d870fa8c 100644 --- a/release/cloudbuild-sync-db-objects.yaml +++ b/release/cloudbuild-sync-db-objects.yaml @@ -39,13 +39,6 @@ steps: gcloud secrets versions access latest \ --secret nomulus-tool-cloudbuild-credential \ > nomulus_tool_credential.json -# Configure the TLDs using the stored configuration files in the internal repo -- name: 'gcr.io/$PROJECT_ID/db_object_updater:live' - args: - - ${_ENV} - - ./nomulus_tool_credential.json - - configure_tld - - nomulus-internal/core/src/main/java/google/registry/config/files/tld/ # Configure the premium lists using the stored configuration files in the internal repo - name: 'gcr.io/$PROJECT_ID/db_object_updater:live' args: @@ -60,6 +53,13 @@ steps: - ./nomulus_tool_credential.json - update_reserved_list - nomulus-internal/core/src/main/java/google/registry/config/files/reserved/ +# Configure the TLDs using the stored configuration files in the internal repo +- name: 'gcr.io/$PROJECT_ID/db_object_updater:live' + args: + - ${_ENV} + - ./nomulus_tool_credential.json + - configure_tld + - nomulus-internal/core/src/main/java/google/registry/config/files/tld/ timeout: 7200s options: diff --git a/release/db-object-updater/sync_db_objects.sh b/release/db-object-updater/sync_db_objects.sh index 50d245fd6de..fe75fcfb065 100644 --- a/release/db-object-updater/sync_db_objects.sh +++ b/release/db-object-updater/sync_db_objects.sh @@ -20,11 +20,12 @@ # - env: The Nomulus environment, production, sandbox, etc. # - tools_credential: The credential (.json) needed to run the nomulus command. # - nomulus_command: The nomulus command to run. -# - config_file_directory: The internal directory storing the TLD config files. +# - config_file_directory: The internal directory storing the config files. set -e if [ "$#" -ne 4 ]; then - echo "Expecting four parameters in order: env tools_credential nomulus_command config_file_directory" + echo "Expecting four parameters in order: env tools_credential" \ + "nomulus_command config_file_directory" exit 1 fi @@ -33,11 +34,20 @@ tools_credential="${2}" nomulus_command="${3}" config_file_directory="${4}" -echo ${config_file_directory} +extra_args=() +if [[ "${nomulus_command}" == "update_premium_list" || \ + "${nomulus_command}" == "update_reserved_list" ]]; then + extra_args+=(--upsert) +fi + +echo "${config_file_directory}" -for FILE in ${config_file_directory}/${nomulus_env}/*; do - echo $FILE - java -jar /nomulus.jar -e "${nomulus_env}" \ - --credential "${tools_credential}" \ - "${nomulus_command}" -i $FILE --force --build_environment +for FILE in "${config_file_directory}/${nomulus_env}"/*; do + if [[ -e "${FILE}" ]]; then + echo "${FILE}" + java -jar /nomulus.jar -e "${nomulus_env}" \ + --credential "${tools_credential}" \ + "${nomulus_command}" -i "${FILE}" --force --build_environment \ + "${extra_args[@]}" + fi done