From 8e89665f3951c4830246830e744a5b2561c81d90 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 10 Jun 2024 14:55:27 -0700 Subject: [PATCH] wip --- webapp/pom.xml | 6 + .../thirdparty/ThirdPartyTMSPhrase.java | 241 +++++++++++ .../ThridPartyTMSPhraseException.java | 7 + .../thirdparty/phrase/PhraseClient.java | 390 ++++++++++++++++++ .../thirdparty/phrase/PhraseClientConfig.java | 23 ++ .../phrase/PhraseClientException.java | 22 + .../phrase/PhraseClientPropertiesConfig.java | 19 + .../thirdparty/ThirdPartyServiceTestData.java | 4 + .../thirdparty/ThirdPartyTMSPhraseTest.java | 91 ++++ .../thirdparty/phrase/PhraseClientTest.java | 112 +++++ 10 files changed, 915 insertions(+) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java diff --git a/webapp/pom.xml b/webapp/pom.xml index 3a43625de3..20b9fde407 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -55,6 +55,12 @@ spring-boot-starter-actuator + + com.phrase + phrase-java + 2.0.2 + + org.springframework.boot spring-boot-starter-data-jpa diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java new file mode 100644 index 0000000000..6dc1180281 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -0,0 +1,241 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.android.strings.AndroidStringDocument; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; +import com.box.l10n.mojito.entity.PollableTask; +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.google.common.collect.ImmutableList; +import com.phrase.client.model.Tag; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@ConditionalOnProperty(value = "l10n.ThirdPartyTMS.impl", havingValue = "ThirdPartyTMSPhrase") +@Component +public class ThirdPartyTMSPhrase implements ThirdPartyTMS { + + static final int MAX_TEXT_UNIT_SUPPORTED = 10000; + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); + + @Autowired TextUnitSearcher textUnitSearcher = new TextUnitSearcher(); + + @Autowired TextUnitBatchImporterService textUnitBatchImporterService; + + @Autowired PhraseClient phraseClient; + + @Autowired RepositoryService repositoryService; + + @Override + public void removeImage(String projectId, String imageId) { + throw new UnsupportedOperationException("Remove image is not supported"); + } + + @Override + public ThirdPartyTMSImage uploadImage(String projectId, String name, byte[] content) { + throw new UnsupportedOperationException("Upload image is not supported"); + } + + @Override + public List getThirdPartyTextUnits( + Repository repository, String projectId, List optionList) { + + throw new UnsupportedOperationException("Get third party text units is not supported"); + } + + @Override + public void createImageToTextUnitMappings( + String projectId, List thirdPartyImageToTextUnits) { + throw new UnsupportedOperationException("Create image to text units is not supported"); + } + + @Override + public void push( + Repository repository, + String projectId, + String pluralSeparator, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List options) { + + List search = + getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + + String text = getFileContent(pluralSeparator, search, true); + + String tagForUpload = getTagForUpload(); + phraseClient.uploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload)); + + phraseClient.removeKeysNotTaggedWith(projectId, tagForUpload); + + List tagsToDelete = + phraseClient.listTags(projectId).stream() + .filter( + tag -> + tag.getName() != null + && !tag.getName().equals(tagForUpload) + && tag.getName().startsWith("push_")) + .toList(); + phraseClient.deleteTags(projectId, tagsToDelete); + } + + private List getSourceTextUnitDTOs( + Repository repository, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + + parameters.setRepositoryIds(repository.getId()); + parameters.setForRootLocale(true); + parameters.setDoNotTranslateFilter(false); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setPluralFormsFiltered(false); + parameters.setOrderByTextUnitID(true); + + return textUnitSearcher.search(parameters); + } + + public static String getTagForUpload() { + ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); + return "push_%s_%s" + .formatted( + formatter.format(zonedDateTime), + Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000)); + } + + @Override + public PollableFuture pull( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List optionList, + String schedulerName, + PollableTask currentTask) { + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { + String localeTag = repositoryLocale.getLocale().getBcp47Tag(); + logger.info("Downloading locale: {} from Phrase", localeTag); + String fileContent = phraseClient.localeDownload(projectId, localeTag, "xml"); + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, localeTag, repository.getName(), true); + + List textUnitDTOS = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + + textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + } + + return null; + } + + @Override + public void pushTranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList) { + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { + List textUnitDTOS = + getTextUnitDTOSForLocale( + repository, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + repositoryLocale); + + String fileContent = getFileContent(pluralSeparator, textUnitDTOS, false); + + phraseClient.uploadCreateFile( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null); + } + } + + private List getTextUnitDTOSForLocale( + Repository repository, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + RepositoryLocale repositoryLocale) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + parameters.setRepositoryIds(repository.getId()); + parameters.setLocaleId(repositoryLocale.getLocale().getId()); + parameters.setDoNotTranslateFilter(false); + parameters.setStatusFilter(StatusFilter.TRANSLATED); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setIncludeTextUnitsWithPattern(includeTextUnitsWithPattern); + parameters.setPluralFormsFiltered(true); + return textUnitSearcher.search(parameters); + } + + private static String getFileContent( + String pluralSeparator, List textUnitDTOS, boolean useSource) { + + AndroidStringDocumentMapper androidStringDocumentMapper = + new AndroidStringDocumentMapper(pluralSeparator, null, null, null, true); + + AndroidStringDocument androidStringDocument = + androidStringDocumentMapper.readFromTextUnits(textUnitDTOS, useSource); + + return new AndroidStringDocumentWriter(androidStringDocument).toText(); + } + + @Override + public void pullSource( + Repository repository, + String projectId, + List optionList, + Map localeMapping) { + throw new UnsupportedOperationException("Pull source is not supported"); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java new file mode 100644 index 0000000000..835c16c069 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.thirdparty; + +public class ThridPartyTMSPhraseException extends RuntimeException { + public ThridPartyTMSPhraseException(String msg, Throwable e) { + super(msg, e); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java new file mode 100644 index 0000000000..7557ca3a99 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -0,0 +1,390 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import static com.box.l10n.mojito.io.Files.createDirectories; +import static com.box.l10n.mojito.io.Files.createTempDirectory; +import static com.box.l10n.mojito.io.Files.write; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.google.common.collect.ImmutableSet; +import com.phrase.client.ApiClient; +import com.phrase.client.ApiException; +import com.phrase.client.api.KeysApi; +import com.phrase.client.api.LocalesApi; +import com.phrase.client.api.TagsApi; +import com.phrase.client.api.UploadsApi; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import com.phrase.client.model.Upload; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class PhraseClient { + + static Logger logger = LoggerFactory.getLogger(PhraseClient.class); + + static final int BATCH_SIZE = 100; + + final ApiClient apiClient; + + final RetryBackoffSpec retryBackoffSpec; + + public PhraseClient(ApiClient apiClient) { + this.apiClient = apiClient; + this.retryBackoffSpec = + Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); + } + + public Upload uploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags) { + + String uploadId = + uploadCreateFile(projectId, localeId, fileFormat, fileName, fileContent, tags); + return waitForUploadToFinish(projectId, uploadId); + } + + Upload waitForUploadToFinish(String projectId, String uploadId) { + UploadsApi uploadsApi = new UploadsApi(apiClient); + + try { + logger.debug("Waiting for upload to finish: {}", uploadId); + + Upload upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "Upload info, first fetch: {}", new ObjectMapper().writeValueAsStringUnchecked(upload)); + + while (!ImmutableSet.of("success", "error").contains(upload.getState())) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "upload info after polling for success or error: {}", + new ObjectMapper().writeValueAsStringUnchecked(upload)); + } + + if ("error".equals(upload.getState())) { + throw new PhraseClientException( + "Upload failed: %s".formatted(new ObjectMapper().writeValueAsStringUnchecked(upload))); + } + + return upload; + } catch (ApiException e) { + logger.error("Error calling Phrase for waitForUploadToFinish: {}", e.getResponseBody()); + throw new PhraseClientException(e); + } + } + + public String uploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags) { + + Path tmpWorkingDirectory = null; + + try { + tmpWorkingDirectory = createTempDirectory("phrase-integration"); + + if (tmpWorkingDirectory.toFile().exists()) { + logger.info("Created temporary working directory: {}", tmpWorkingDirectory); + } + + Path fileToUpload = tmpWorkingDirectory.resolve(fileName); + + logger.info("Create file: {}", fileToUpload); + createDirectories(fileToUpload.getParent()); + write(fileToUpload, fileContent); + + Upload upload = + uploadsApiUploadCreateWithRetry(projectId, localeId, fileFormat, tags, fileToUpload); + + return upload.getId(); + } finally { + if (tmpWorkingDirectory != null) { + com.box.l10n.mojito.io.Files.deleteRecursivelyIfExists(tmpWorkingDirectory); + } + } + } + + Upload uploadsApiUploadCreateWithRetry( + String projectId, String localeId, String fileFormat, List tags, Path fileToUpload) { + + return Mono.fromCallable( + () -> + new UploadsApi(apiClient) + .uploadCreate( + projectId, + fileToUpload.toFile(), + fileFormat, + localeId, + null, + null, + tags == null ? null : String.join(",", tags), + true, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId))) + .block(); + } + + public void removeKeysNotTaggedWith(String projectId, String tag) { + logger.info("Removing keys not tagged with: {}", tag); + + Mono.fromCallable( + () -> { + KeysApi keysApi = new KeysApi(apiClient); + keysApi.keysDeleteCollection(projectId, null, null, "-tags:%s".formatted(tag), null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId))) + .block(); + } + + public String localeDownload(String projectId, String locale, String fileFormat) { + return Mono.fromCallable( + () -> { + LocalesApi localesApi = new LocalesApi(apiClient); + logger.info( + "Downloading locale: {} from project id: {} in file format: {}", + locale, + projectId, + fileFormat); + File file = + localesApi.localeDownload( + projectId, + locale, + null, + null, + null, + null, + fileFormat, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + + String localeDownloadContent = Files.readString(file.toPath()); + logger.debug("File: {}, Content: {}", file.toPath(), localeDownloadContent); + + return localeDownloadContent; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale))) + .block(); + } + + public List getKeys(String projectId) { + KeysApi keysApi = new KeysApi(apiClient); + AtomicInteger page = new AtomicInteger(0); + int batchSize = BATCH_SIZE; + List translationKeys = new ArrayList<>(); + while (true) { + List translationKeysInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching keys for project: {}, page: {}", projectId, page); + return keysApi.keysList( + projectId, null, page.get(), batchSize, null, null, null, null, null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch keys for project: %s, page: %s" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch keys for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + translationKeys.addAll(translationKeysInPage); + + if (translationKeysInPage.size() < batchSize) { + break; + } else { + page.incrementAndGet(); + } + } + return translationKeys; + } + + public List listTags(String projectId) { + TagsApi tagsApi = new TagsApi(apiClient); + final AtomicInteger page = new AtomicInteger(0); + List tags = new ArrayList<>(); + while (true) { + List tagsInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching tags for project: {}", projectId); + return tagsApi.tagsList(projectId, null, page.get(), BATCH_SIZE, null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch tags for project: %s, page: %d" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch tags for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + tags.addAll(tagsInPage); + + if (tagsInPage.size() < BATCH_SIZE) { + break; + } else { + page.incrementAndGet(); + } + } + + return tags; + } + + public void deleteTags(String projectId, List tags) { + TagsApi tagsApi = new TagsApi(apiClient); + Map exceptions = new LinkedHashMap<>(); + for (Tag tag : tags) { + Mono.fromCallable( + () -> { + logger.debug( + "Deleting tag: %s in project id: %s".formatted(tag.getName(), projectId)); + tagsApi.tagDelete(projectId, tag.getName(), null, null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to delete tag: %s in project id: %s" + .formatted(tag.getName(), projectId)); + })) + .doOnError( + throwable -> { + exceptions.put(tag.getName(), throwable); + rethrowExceptionWithLog( + throwable, + "Final error to delete tag: %s in project id: %s" + .formatted(tag.getName(), projectId)); + }) + .block(); + } + + if (!exceptions.isEmpty()) { + List tagsWithErrors = exceptions.keySet().stream().limit(10).toList(); + String andMore = (tagsWithErrors.size() < exceptions.size()) ? " and more." : ""; + throw new PhraseClientException( + String.format("Can't delete tags: %s%s", tagsWithErrors, andMore)); + } + } + + private void logAttempt(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.info("%s, error: %s".formatted(message, errorMessage), throwable); + } + + private void rethrowExceptionWithLog(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.error("%s, error: %s".formatted(message, errorMessage)); + if (throwable.getCause() instanceof ApiException) { + throw new PhraseClientException(errorMessage, (ApiException) throwable.getCause()); + } else { + throw new RuntimeException(errorMessage, throwable); + } + } + + private String getErrorMessageFromOptionalApiException(Throwable t) { + String errorMessage; + if (t instanceof ApiException) { + errorMessage = ((ApiException) t).getResponseBody(); + } else { + errorMessage = t.getMessage(); + } + return errorMessage; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java new file mode 100644 index 0000000000..65e69823ab --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java @@ -0,0 +1,23 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiClient; +import com.phrase.client.auth.ApiKeyAuth; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PhraseClientConfig { + + @ConditionalOnProperty("l10n.phrase.client.token") + @Bean + public PhraseClient getPhraseClient(PhraseClientPropertiesConfig phraseClientPropertiesConfig) { + + ApiClient apiClient = new ApiClient(); + ApiKeyAuth authentication = (ApiKeyAuth) apiClient.getAuthentication("Token"); + authentication.setApiKey(phraseClientPropertiesConfig.getToken()); + authentication.setApiKeyPrefix("token"); + + return new PhraseClient(apiClient); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java new file mode 100644 index 0000000000..65f0ffc6c4 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java @@ -0,0 +1,22 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiException; + +public class PhraseClientException extends RuntimeException { + + ApiException apiException; + + public PhraseClientException(String message) { + super(message); + } + + public PhraseClientException(ApiException e) { + super(e); + apiException = e; + } + + public PhraseClientException(String message, ApiException apiException) { + super(message); + this.apiException = apiException; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java new file mode 100644 index 0000000000..9baf70b304 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.phrase.client") +public class PhraseClientPropertiesConfig { + + String token; + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java index 1b081589f3..642bf18ca9 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java @@ -177,4 +177,8 @@ public ThirdPartyServiceTestData init() throws Exception { logger.debug("Finished init of ThirdPartyServiceTestData"); return this; } + + public String getPluralSeparator() { + return "_"; + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java new file mode 100644 index 0000000000..f7ddc2ba61 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -0,0 +1,91 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.repository.RepositoryLocaleCreationException; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.test.TestIdWatcher; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +public class ThirdPartyTMSPhraseTest extends ServiceTestBase { + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhraseTest.class); + + @Autowired(required = false) + ThirdPartyTMSPhrase thirdPartyTMSPhrase; + + @Autowired RepositoryService repositoryService; + + @Autowired TextUnitSearcher textUnitSearcher; + + @Value("${test.phrase-client.projectId}") + String testProjectId; + + @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); + + @Test + public void testBasics() throws RepositoryLocaleCreationException { + Assume.assumeNotNull(thirdPartyTMSPhrase); + Assume.assumeNotNull(testProjectId); + + ThirdPartyServiceTestData thirdPartyServiceTestData = + new ThirdPartyServiceTestData(testIdWatcher); + + Repository repository = thirdPartyServiceTestData.repository; + repositoryService.addRepositoryLocale(repository, "fr"); + + repository + .getRepositoryLocales() + .forEach(rl -> logger.info("repository locale: {}", rl.getLocale().getBcp47Tag())); + + thirdPartyTMSPhrase.push( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + null, + null, + null); + + thirdPartyTMSPhrase.pull( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + ImmutableMap.of("fr-FR", "fr"), + null, + null, + null, + null, + null); + + TextUnitSearcherParameters params = new TextUnitSearcherParameters(); + params.setRepositoryIds(repository.getId()); + params.setRootLocaleExcluded(false); + params.setStatusFilter(StatusFilter.TRANSLATED); + List search = textUnitSearcher.search(params); + + if (logger.isInfoEnabled()) { + ObjectMapper objectMapper = new ObjectMapper(); + search.stream().forEach(t -> logger.info(objectMapper.writeValueAsStringUnchecked(t))); + } + + // List thirdPartyTextUnits = + // thirdPartyTMSPhrase.getThirdPartyTextUnits(repository, testProjectId, null); + // logger.info("Get") + // thirdPartyTextUnits.stream().forEach(t -> logger.info("third party text unit: {}", + // t)); + + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java new file mode 100644 index 0000000000..4b50897c9b --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -0,0 +1,112 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase; +import com.google.common.collect.ImmutableList; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import java.util.List; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = { + PhraseClientTest.class, + PhraseClientConfig.class, + PhraseClientPropertiesConfig.class + }) +@EnableConfigurationProperties +public class PhraseClientTest { + + static Logger logger = LoggerFactory.getLogger(PhraseClientTest.class); + + @Autowired PhraseClient phraseClient; + + @Value("${test.phrase-client.projectId}") + String testProjectId; + + @Test + public void testRemoveTag() { + String tagForUpload = "push_2024_06_10_07_17_00_089_122"; + List tagsToDelete = + phraseClient.listTags(testProjectId).stream() + .peek(tag -> logger.info("tag: {}", tag)) + .filter(tag -> tag.getName() != null && !tag.getName().equals(tagForUpload)) + .toList(); + + phraseClient.deleteTags(testProjectId, tagsToDelete); + } + + @Test + public void test() { + Assume.assumeNotNull(testProjectId); + + String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(); + + logger.info("tagForUpload: {}", tagForUpload); + + StringBuilder fileContentAndroidBuilder = generateFileContent(); + + String fileContentAndroid = fileContentAndroidBuilder.toString(); + phraseClient.uploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + fileContentAndroid, + ImmutableList.of(tagForUpload)); + + phraseClient.removeKeysNotTaggedWith(testProjectId, tagForUpload); + + List translationKeys = phraseClient.getKeys(testProjectId); + for (TranslationKey translationKey : translationKeys) { + logger.info("{}", translationKey); + } + + // + // String fileContentAndroid2 = + // """ + // + // + // Locale Tester - fr + // Settings - fr + // Hello + // + // One thing - fr + // Multiple things - fr + // + // + // """; + // phraseClient.uploadCreateFile( + // testProjectId, "fr", "xml", "strings.xml", fileContentAndroid2, null); + // String s2 = phraseClient.localeDownload(testProjectId, "fr", "xml"); + // logger.info(s2); + } + + static StringBuilder generateFileContent() { + StringBuilder fileContentAndroidBuilder = new StringBuilder(); + + fileContentAndroidBuilder.append( + """ + + + Locale Tester + """); + + for (int i = 0; i < 2000; i++) { + fileContentAndroidBuilder.append( + String.format("Settings\n", i)); + } + + fileContentAndroidBuilder.append(""); + return fileContentAndroidBuilder; + } +}