Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ThirdPartySyncCommand - Adding new parameters #614

Merged
merged 1 commit into from
Jul 27, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.ParameterException;
import com.google.common.base.Strings;

import java.util.Arrays;

Expand All @@ -15,7 +16,7 @@ public final T convert(String value) {
T result = null;
if (value != null) {
try {
result = Enum.valueOf(getGenericClass(), value.toUpperCase());
result = Enum.valueOf(getGenericClass(), Strings.nullToEmpty(value).trim().toUpperCase());
} catch (IllegalArgumentException iae) {
String msg = "Invalid type [" + value + "], should be one of: " + Arrays.toString(getGenericClass().getEnumConstants());
throw new ParameterException(msg);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ public class ImportLocalizedAssetCommand extends Command {
@Parameter(names = {Param.TARGET_DIRECTORY_LONG, Param.TARGET_DIRECTORY_SHORT}, arity = 1, required = false, description = Param.TARGET_DIRECTORY_DESCRIPTION)
String targetDirectoryParam;

@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = "Locale mapping, format: \"fr:fr-FR,ja:ja-JP\". "
+ "The keys contain BCP47 tags of the generated files and the values indicate which repository locales are used to fetch the translations.")
@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION)
String localeMappingParam;

@Parameter(names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, arity = 1, required = false, description = Param.FILE_TYPE_DESCRIPTION,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@ public class PullCommand extends Command {
@Parameter(names = {Param.TARGET_DIRECTORY_LONG, Param.TARGET_DIRECTORY_SHORT}, arity = 1, required = false, description = Param.TARGET_DIRECTORY_DESCRIPTION)
String targetDirectoryParam;

@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = "Locale mapping, format: \"fr:fr-FR,ja:ja-JP\". "
+ "The keys contain BCP47 tags of the generated files and the values indicate which repository locales are used to fetch the translations.")
@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION)
String localeMappingParam;

@Parameter(names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, arity = 1, required = false, description = Param.FILE_TYPE_DESCRIPTION,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ public class ScreenshotCommand extends Command {
@Parameter(names = {Param.REPOSITORY_LONG, Param.REPOSITORY_SHORT}, arity = 1, required = true, description = Param.REPOSITORY_DESCRIPTION)
String repositoryParam;

@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = "Locale mapping, format: \"fr:fr-FR,ja:ja-JP\". "
+ "The keys contain BCP47 tags of the generated files and the values indicate which repository locales are used to fetch the translations.")
@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION)
String localeMappingParam;

@Parameter(names = {Param.SOURCE_DIRECTORY_LONG, Param.SOURCE_DIRECTORY_SHORT}, arity = 1, required = false, description = "Directory to scan for screenshot")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
package com.box.l10n.mojito.cli.command;

import com.box.l10n.mojito.rest.client.ThirdPartySync;
import com.box.l10n.mojito.rest.ThirdPartySyncAction;
Copy link
Collaborator

@aurambaj aurambaj Jul 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh sorry, maybe I wasn't super clear last time we chatted about it. I don't want the rest client and the webservice layer to be linked closely (with webapp depending on the client). The main reason is that in most cases there is not a 1:1 mapping between what's done in the server and what makes sense in the client (hibernate entities, slightly different object). I know that in many case the POJO are or feel like duplicates between the client and server but I'd rather keep that way. Let's not have the rest client as dependency of webapp, it also becomes messy and we may end up having some code going through the client instead of directly through the service which doesn't make sense. It may become confusing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just updated the code


/**
*
* @author sdemyanenko
*/
public class ThirdPartySyncActionsConverter extends EnumConverter<ThirdPartySync.Action> {
public class ThirdPartySyncActionsConverter extends EnumConverter<ThirdPartySyncAction> {

@Override
protected Class<ThirdPartySync.Action> getGenericClass() {
return ThirdPartySync.Action.class;
protected Class<ThirdPartySyncAction> getGenericClass() {
return ThirdPartySyncAction.class;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import com.beust.jcommander.Parameters;
import com.box.l10n.mojito.cli.command.param.Param;
import com.box.l10n.mojito.cli.console.ConsoleWriter;
import com.box.l10n.mojito.rest.ThirdPartySyncAction;
import com.box.l10n.mojito.rest.client.ThirdPartyClient;
import com.box.l10n.mojito.rest.client.ThirdPartySync;
import com.box.l10n.mojito.rest.entity.PollableTask;
import com.box.l10n.mojito.rest.entity.Repository;
import org.fusesource.jansi.Ansi;
Expand Down Expand Up @@ -44,14 +44,20 @@ public class ThirdPartySyncCommand extends Command {
String thirdPartyProjectId;

@Parameter(names = {"--actions", "-a"}, variableArity = true, required = false, description = "Actions to synchronize", converter = ThirdPartySyncActionsConverter.class)
List<ThirdPartySync.Action> actions = Arrays.asList(ThirdPartySync.Action.MAP_TEXTUNIT, ThirdPartySync.Action.PUSH_SCREENSHOT);
List<ThirdPartySyncAction> actions = Arrays.asList(ThirdPartySyncAction.MAP_TEXTUNIT, ThirdPartySyncAction.PUSH_SCREENSHOT);

@Parameter(names = {"--plural-separator", "-ps"}, arity = 1, required = false, description = "Plural separator for name")
String pluralSeparator;

@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = "Locale mapping")
@Parameter(names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT}, arity = 1, required = false, description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION)
String localeMapping;

@Parameter(names = {"--skip-text-units-with-pattern", "-st"}, arity = 1, required = false, description = "Do not process text units matching with the SQL LIKE expression")
String skipTextUnitsWithPattern;

@Parameter(names = {"--skip-assets-path-pattern", "-sa"}, arity = 1, required = false, description = "Do not process text units whose assets path match the SQL LIKE expression")
String skipAssetsWithPathPattern;

@Parameter(names = {"--options", "-o"}, variableArity = true, required = false, description = "Options to synchronize")
List<String> options;

Expand All @@ -69,11 +75,14 @@ public void execute() throws CommandException {
.a(" actions: ").fg(CYAN).a(Objects.toString(actions)).reset()
.a(" plural-separator: ").fg(CYAN).a(Objects.toString(pluralSeparator)).reset()
.a(" locale-mapping: ").fg(CYAN).a(Objects.toString(localeMapping)).reset()
.a(" skip-text-units-with-pattern: ").fg(CYAN).a(Objects.toString(skipTextUnitsWithPattern)).reset()
.a(" skip-assets-path-pattern: ").fg(CYAN).a(Objects.toString(skipAssetsWithPathPattern)).reset()
.a(" options: ").fg(CYAN).a(Objects.toString(options)).println(2);

Repository repository = commandHelper.findRepositoryByName(repositoryParam);

PollableTask pollableTask = thirdPartyClient.sync(repository.getId(), thirdPartyProjectId, pluralSeparator, localeMapping, actions, options);
PollableTask pollableTask = thirdPartyClient.sync(repository.getId(), thirdPartyProjectId, pluralSeparator, localeMapping,
actions, skipTextUnitsWithPattern, skipAssetsWithPathPattern, options);

commandHelper.waitForPollableTask(pollableTask.getId());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ public class Param {

public static final String REPOSITORY_LOCALES_MAPPING_LONG = "--locale-mapping";
public static final String REPOSITORY_LOCALES_MAPPING_SHORT = "-lm";
public static final String REPOSITORY_LOCALES_MAPPING_DESCRIPTION = "Locale mapping, format: \"fr:fr-FR,ja:ja-JP\". "
+ "The keys contain BCP47 tags of the generated files and the values indicate which repository locales are used to fetch the translations.";

public static final String REPOSITORY_SOURCE_LOCALE_LONG = "--source-locale";
public static final String REPOSITORY_SOURCE_LOCALE_SHORT = "-sl";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,32 +1,105 @@
package com.box.l10n.mojito.cli.command;

import com.box.l10n.mojito.cli.CLITestBase;
import com.box.l10n.mojito.cli.utils.PollableTaskJobMatcher;
import com.box.l10n.mojito.cli.utils.TestingJobListener;
import com.box.l10n.mojito.entity.Repository;
import com.box.l10n.mojito.rest.client.ThirdPartySync;
import com.box.l10n.mojito.json.ObjectMapper;
import com.box.l10n.mojito.rest.thirdparty.ThirdPartySyncAction;
import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJob;
import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJobInput;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.quartz.JobKey;
import org.quartz.Matcher;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import java.util.Arrays;
import java.util.List;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static com.box.l10n.mojito.rest.thirdparty.ThirdPartySyncAction.MAP_TEXTUNIT;
import static com.box.l10n.mojito.rest.thirdparty.ThirdPartySyncAction.PUSH_SCREENSHOT;
import static org.assertj.core.api.Assertions.assertThat;

public class ThirdPartySyncCommandTest extends CLITestBase {

/**
* logger
*/
static Logger logger = LoggerFactory.getLogger(ThirdPartySyncCommandTest.class);
@Autowired
@Qualifier("fail_on_unknown_properties_false")
ObjectMapper objectMapper;

@Autowired
Scheduler scheduler;

Matcher<JobKey> jobMatcher;
TestingJobListener testingJobListener;

@Before
public void setUp() throws SchedulerException {
testingJobListener = new TestingJobListener(objectMapper);
jobMatcher = new PollableTaskJobMatcher<>(ThirdPartySyncJob.class);
scheduler.getListenerManager().addJobListener(testingJobListener, jobMatcher);
aurambaj marked this conversation as resolved.
Show resolved Hide resolved
}

@After
public void tearDown() throws SchedulerException {
scheduler.getListenerManager().removeJobListener(testingJobListener.getName());
scheduler.getListenerManager().removeJobListenerMatcher(testingJobListener.getName(), jobMatcher);
aurambaj marked this conversation as resolved.
Show resolved Hide resolved
}

@Test
public void execute() throws Exception {

String repoName = testIdWatcher.getEntityName("thirdpartysync_execute");

Repository repository = repositoryService.createRepository(repoName, repoName + " description", null, false);
getL10nJCommander().run("thirdparty-sync", "-r", repository.getName(), "-p", "does-not-matter-yet", "-ps", " _");
String projectId = testIdWatcher.getEntityName("projectId");

// TODO: For a plural separator like " _" this test will fail. The current version we have for
// JCommander trims the argument values, even when quoted.
// https://github.com/cbeust/jcommander/issues/417
// https://github.com/cbeust/jcommander/commit/4aec38b4a0ea63a8dc6f41636fa81c2ebafddc18
String pluralSeparator = "_";
String skipTextUnitPattern = "%skip_text_pattern";
String skipAssetPattern = "%skip_asset_pattern%";
List<String> options = Arrays.asList(
"special-option=value@of%Option",
"smartling-placeholder-custom=\\{\\{\\}\\}|\\{\\{?.+?\\}\\}?|\\%\\%\\(.+?\\)s|\\%\\(.+?\\)s|\\%\\(.+?\\)d|\\%\\%s|\\%s"
);

getL10nJCommander().run("thirdparty-sync",
"-r", repository.getName(),
"-p", projectId,
"-a", MAP_TEXTUNIT.name(), PUSH_SCREENSHOT.name(),
"-ps", pluralSeparator,
"-st", skipTextUnitPattern,
"-sa", skipAssetPattern,
"-o", options.get(0), options.get(1));

String output = outputCapture.toString();
assertThat(output).contains("repository: " + repository.getName());
assertThat(output).contains("project id: " + projectId);
assertThat(output).contains("actions: " + Arrays.asList(ThirdPartySyncAction.MAP_TEXTUNIT, ThirdPartySyncAction.PUSH_SCREENSHOT).toString());
assertThat(output).contains("skip-text-units-with-pattern: " + skipTextUnitPattern);
assertThat(output).contains("skip-assets-path-pattern: " + skipAssetPattern);
assertThat(output).contains("options: " + options.toString());

waitForCondition("Ensure ThirdPartySyncJob gets executed",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess that listener is ok for now though I think changing the application context is not too great if we ever start running test in parallel, etc.

So my preference would be avoiding the listener and just wait for the state (or different part of the state) to get to what we expect which would indicate a successful migration.

On top of my head I don't remember other test intercepting part of the request to perform input validation. In general those are more like end to end test. The parameters are not check directly but instead the results are checked inferring that the proper arguments were passed

Copy link
Contributor Author

@comunacho comunacho Jul 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've prioritized on testing that the parameters given to the CLI were passed around downstream all the way to the Quartz job for this test. I do agree that here is better to test final states or side effects, but since non of the parameters is being used in ThirdPartyTMSInMemory method we have nowadays and we're not triggering side effects, my preference for this PR was to have the argument passing fully tested (since that's what we're changing now).

Since we'll still implement new methods in ThirdPartyTMS in the upcoming PRs, I will add end to end testing through the ThirdPartyTMSInMemory class once we have these new method added to the interface signature as well as to the interface implementors (ThirdPartyTMSSmartling and ThirdPartyTMSInMemory)

() -> testingJobListener.getExecuted().size() > 0);

ThirdPartySyncJobInput jobInput = testingJobListener.getFirstInputMapAs(ThirdPartySyncJobInput.class);

String outputString = outputCapture.toString();
assertTrue(outputString.contains(Arrays.asList(ThirdPartySync.Action.MAP_TEXTUNIT, ThirdPartySync.Action.PUSH_SCREENSHOT).toString()));
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not keeping that?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

assertThat(jobInput).isNotNull();
assertThat(jobInput.getRepositoryId()).isEqualTo(repository.getId());
assertThat(jobInput.getThirdPartyProjectId()).isEqualTo(projectId);
assertThat(jobInput.getActions()).containsExactlyInAnyOrder(MAP_TEXTUNIT, PUSH_SCREENSHOT);
assertThat(jobInput.getPluralSeparator()).isEqualTo(pluralSeparator);
assertThat(jobInput.getSkipTextUnitsWithPattern()).isEqualTo(skipTextUnitPattern);
assertThat(jobInput.getSkipAssetsWithPathPattern()).isEqualTo(skipAssetPattern);
assertThat(jobInput.getOptions()).containsExactlyInAnyOrderElementsOf(options);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package com.box.l10n.mojito.cli.utils;

import com.box.l10n.mojito.quartz.QuartzPollableJob;
import org.quartz.JobKey;
import org.quartz.Matcher;

import static com.box.l10n.mojito.quartz.QuartzConfig.DYNAMIC_GROUP_NAME;

public class PollableTaskJobMatcher<T extends QuartzPollableJob> implements Matcher<JobKey> {

private final Class<T> target;

public PollableTaskJobMatcher(Class<T> target) {
this.target = target;
}

@Override
public boolean isMatch(JobKey key) {
return key.getName().startsWith(target.getName()) && DYNAMIC_GROUP_NAME.equals(key.getGroup());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package com.box.l10n.mojito.cli.utils;

import com.box.l10n.mojito.json.ObjectMapper;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.JobListener;

import java.util.LinkedList;
import java.util.Queue;

import static com.box.l10n.mojito.quartz.QuartzPollableJob.INPUT;

public class TestingJobListener implements JobListener {

private final Queue<JobExecutionContext> toBeExecuted = new LinkedList<>();
private final Queue<JobExecutionContext> executed = new LinkedList<>();
private final Queue<JobExecutionException> exceptions = new LinkedList<>();
private final ObjectMapper objectMapper;

public TestingJobListener(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}

public TestingJobListener() {
this.objectMapper = new ObjectMapper();
}

@Override
public String getName() {
return "TestingJobListener";
}

@Override
public void jobToBeExecuted(JobExecutionContext context) {
toBeExecuted.offer(context);
}

@Override
public void jobExecutionVetoed(JobExecutionContext context) {
}

@Override
public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) {
executed.offer(context);
exceptions.offer(jobException);
}

public Queue<JobExecutionContext> getExecuted() {
return executed;
}

public Queue<JobExecutionContext> getToBeExecuted() {
return toBeExecuted;
}

public <T> T getFirstInputMapAs(Class<T> klass) {
String input = getExecuted().stream()
.filter(context -> context.getMergedJobDataMap().containsKey(INPUT))
.map(context -> context.getMergedJobDataMap().getString(INPUT))
.findFirst()
.orElse("");

return objectMapper.readValueUnchecked(input, klass);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package com.box.l10n.mojito.rest;

public enum ThirdPartySyncAction {
PUSH,
PUSH_TRANSLATION,
PULL,
MAP_TEXTUNIT,
PUSH_SCREENSHOT
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package com.box.l10n.mojito.rest.client;

import com.box.l10n.mojito.rest.ThirdPartySyncAction;
import com.box.l10n.mojito.rest.entity.PollableTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.util.UriComponentsBuilder;

Expand All @@ -16,19 +15,19 @@
@Component
public class ThirdPartyClient extends BaseClient {

/**
* logger
*/
static Logger logger = LoggerFactory.getLogger(ThirdPartyClient.class);

@Override
public String getEntityName() {
return "thirdparty";
}

public PollableTask sync(Long repositoryId, String projectId, String pluralSeparator, String localeMapping,
List<ThirdPartySync.Action> actions, List<String> options)
{
public PollableTask sync(Long repositoryId,
String projectId,
String pluralSeparator,
String localeMapping,
List<ThirdPartySyncAction> actions,
String skipTextUnitsWithPattern,
String skipAssetsWithPathPattern,
List<String> options) {

ThirdPartySync thirdPartySync = new ThirdPartySync();

Expand All @@ -37,6 +36,8 @@ public PollableTask sync(Long repositoryId, String projectId, String pluralSepar
thirdPartySync.setActions(actions);
thirdPartySync.setPluralSeparator(pluralSeparator);
thirdPartySync.setLocaleMapping(localeMapping);
thirdPartySync.setSkipTextUnitsWithPattern(skipTextUnitsWithPattern);
thirdPartySync.setSkipAssetsWithPathPattern(skipAssetsWithPathPattern);
thirdPartySync.setOptions(options);

UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromPath(getBasePathForEntity()).pathSegment("sync");
Expand Down
Loading