Skip to content

Commit

Permalink
wip is that just re-indent?
Browse files Browse the repository at this point in the history
  • Loading branch information
ja-openai committed Aug 5, 2024
1 parent 5cecf37 commit d8ad75a
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ForkJoinPool;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -294,55 +292,64 @@ public PollableFuture<Void> pull(

String currentTags = getCurrentTagsForRepository(repository, projectId);

// may already hit rate limit, according it is 4 qps ... there is a retry in the locale client though.
// may already hit rate limit, according it is 4 qps ... there is a retry in the locale client
// though.
// but 4 qps is very low to download 64 locales.
repositoryLocalesWithoutRootLocale.parallelStream()
.forEach(locale -> pullLocaleTimed(repository, projectId, locale, pluralSeparator, currentTags));
.forEach(
locale -> pullLocaleTimed(repository, projectId, locale, pluralSeparator, currentTags));

return null;
}

private void pullLocaleTimed(
Repository repository, String projectId, RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags) {
Repository repository,
String projectId,
RepositoryLocale repositoryLocale,
String pluralSeparator,
String currentTags) {
try (var timer =
Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale")
.tag("repository", repository.getName())) {

pullLocale(repository, projectId, repositoryLocale,
pluralSeparator, currentTags);
pullLocale(repository, projectId, repositoryLocale, pluralSeparator, currentTags);
}
}

private void pullLocale(
Repository repository, String projectId, RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags) {
Repository repository,
String projectId,
RepositoryLocale repositoryLocale,
String pluralSeparator,
String currentTags) {

String localeTag = repositoryLocale.getLocale().getBcp47Tag();
logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags);
String localeTag = repositoryLocale.getLocale().getBcp47Tag();
logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags);

Stopwatch localeDownloadStopWatch = Stopwatch.createStarted();
String fileContent =
phraseClient.localeDownload(
projectId,
localeTag,
"xml",
currentTags,
() -> getCurrentTagsForRepository(repository, projectId));
Stopwatch localeDownloadStopWatch = Stopwatch.createStarted();
String fileContent =
phraseClient.localeDownload(
projectId,
localeTag,
"xml",
currentTags,
() -> getCurrentTagsForRepository(repository, projectId));

logger.info("Phrase locale download took: {}", localeDownloadStopWatch.elapsed());
logger.info("Phrase locale download took: {}", localeDownloadStopWatch.elapsed());

logger.debug("file content from pull: {}", fileContent);
logger.debug("file content from pull: {}", fileContent);

AndroidStringDocumentMapper mapper =
new AndroidStringDocumentMapper(
pluralSeparator, null, localeTag, repository.getName(), true, null);
AndroidStringDocumentMapper mapper =
new AndroidStringDocumentMapper(
pluralSeparator, null, localeTag, repository.getName(), true, null);

List<TextUnitDTO> textUnitDTOS =
mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent));
List<TextUnitDTO> textUnitDTOS =
mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent));

// make sure that the source comment does not get replicated to all translations. Eventually
// we may want to
// communicate some comments, but anyway it should not be the source comment
textUnitDTOS.forEach(t -> t.setComment(null));
// make sure that the source comment does not get replicated to all translations. Eventually
// we may want to
// communicate some comments, but anyway it should not be the source comment
textUnitDTOS.forEach(t -> t.setComment(null));

Stopwatch importStopWatch = Stopwatch.createStarted();
textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -282,8 +282,9 @@ public String localeDownload(
"Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s"
.formatted(projectId, locale));

if (onTagErrorRefreshCallback != null && getErrorMessageFromOptionalApiException(doBeforeRetry.failure())
.contains("Invalid Download Options. Parameter tags ")) {
if (onTagErrorRefreshCallback != null
&& getErrorMessageFromOptionalApiException(doBeforeRetry.failure())
.contains("Invalid Download Options. Parameter tags ")) {
String newTags = onTagErrorRefreshCallback.get();
logger.warn(
"Replacing old tags: {} with new tags: {} for download locale",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ void importTextUnitsOfLocaleAndAsset(
"Start import text units for asset: {}, locale: {}, count: {}",
asset.getPath(),
locale.getBcp47Tag(),
textUnitsToImport.size());
textUnitsToImport.size());

textUnitsToImport.stream()
.filter(
Expand Down Expand Up @@ -249,7 +249,6 @@ void importTextUnitsOfLocaleAndAsset(

TMTextUnitCurrentVariant tmTextUnitCurrentVariant = null;
if (currentTextUnit.getTmTextUnitCurrentVariantId() != null) {
// this is making many calls!
tmTextUnitCurrentVariant =
tmTextUnitCurrentVariantRepository.findByLocale_IdAndTmTextUnit_Id(
currentTextUnit.getLocaleId(), currentTextUnit.getTmTextUnitId());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ public class ThirdPartyTMSPhraseTest extends ServiceTestBase {

@Rule public TestIdWatcher testIdWatcher = new TestIdWatcher();



@Test
public void testBasics() throws RepositoryLocaleCreationException {
Assume.assumeNotNull(thirdPartyTMSPhrase);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.box.l10n.mojito.service.thirdparty.phrase;

import com.box.l10n.mojito.JSR310Migration;
import com.box.l10n.mojito.aspect.StopWatch;
import com.google.common.base.Stopwatch;
import com.phrase.client.model.Tag;
import java.time.ZonedDateTime;
Expand Down Expand Up @@ -56,28 +55,27 @@ public void testParallelDownload() {
// measure time of following call
Stopwatch total = Stopwatch.createStarted();


List<String> locales =
List.of(
"bg", "bn", "bs", "ca", "cs", "da", "de", "el", "es-419", "es", "et", "fi", "fr-CA",
"fr", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "ja", "ka", "kn", "ko", "lv",
"mk", "mr", "ms", "my", "nb", "nl", "pl", "pt", "pt-PT", "ro", "ru", "sk", "sl", "so",
"sq", "sr", "sv", "sw", "ta", "te", "th", "tr", "uk", "vi", "zh", "zh-HK", "zh-Hant");

locales.parallelStream().forEach(l -> {

Stopwatch started = Stopwatch.createStarted();
String s =
phraseClient.localeDownload(
locales.parallelStream()
.forEach(
l -> {
Stopwatch started = Stopwatch.createStarted();
String s =
phraseClient.localeDownload(
"9b6f4e167397549fe7ec1fe82497159b",
l,
"xml",
"push_chatgpt-web_2024_06_27_05_14_58_356_208",
null);
// logger.info(s);
logger.info("Download: {} in {}", l, started.elapsed());
});

// logger.info(s);
logger.info("Download: {} in {}", l, started.elapsed());
});

logger.info("total time: {}", total.elapsed());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,15 +100,15 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException {
textUnitDTO3.setTarget("zuora_error_message_verify_state_province translation for fr");

List<TextUnitDTO> textUnitDTOsForImport =
Arrays.asList(textUnitDTO, textUnitDTO2, textUnitDTO3);
Arrays.asList(textUnitDTO, textUnitDTO2, textUnitDTO3);

PollableFuture<Void> asyncImportTextUnits =
textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false);
textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false);

pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId());

TextUnitSearcherParameters textUnitSearcherParameters =
new TextUnitSearcherParametersForTesting();
new TextUnitSearcherParametersForTesting();
textUnitSearcherParameters.setRepositoryNames(Arrays.asList(tmTestData.repository.getName()));
textUnitSearcherParameters.setAssetPath(tmTestData.asset.getPath());
textUnitSearcherParameters.setLocaleTags(Arrays.asList("fr-FR"));
Expand All @@ -118,10 +118,10 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException {
int i = 0;

assertEquals(
"zuora_error_message_verify_state_province", textUnitDTOsFromSearch.get(i).getName());
"zuora_error_message_verify_state_province", textUnitDTOsFromSearch.get(i).getName());
assertEquals(
"zuora_error_message_verify_state_province translation for fr",
textUnitDTOsFromSearch.get(i).getTarget());
"zuora_error_message_verify_state_province translation for fr",
textUnitDTOsFromSearch.get(i).getTarget());
i++;
assertEquals("TEST2", textUnitDTOsFromSearch.get(i).getName());
assertEquals("TEST2 translation for fr", textUnitDTOsFromSearch.get(i).getTarget());
Expand Down

0 comments on commit d8ad75a

Please sign in to comment.