Merge branch 'master' into docs/observability

CHANGES.md

@@ -7,7 +7,11 @@ Apollo 2.4.0
 ------------------
 * [Update the server config link in system info page](https://github.com/apolloconfig/apollo/pull/5204)
 * [Feature support portal restTemplate Client connection pool config](https://github.com/apolloconfig/apollo/pull/5200)
-
+* [Feature added the ability for administrators to globally search for Value](https://github.com/apolloconfig/apollo/pull/5182)
+* [Fix: Resolve issues with duplicate comments and blank lines in configuration management](https://github.com/apolloconfig/apollo/pull/5232)
+* [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)
+* [Feature: Add limit and whitelist for namespace count per appid+cluster](https://github.com/apolloconfig/apollo/pull/5228)
+* [Feature support the observe status access-key for pre-check and logging only](https://github.com/apolloconfig/apollo/pull/5236)
 
 ------------------
 All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)

README.md

@@ -37,36 +37,31 @@ Demo Environment:
   * The same codebase could have different configurations when deployed in different clusters
   * With the namespace concept, it is easy to support multiple applications to share the same configurations, while also allowing them to customize the configurations
   * Multiple languages is provided in user interface(currently Chinese and English)
-
 * **Configuration changes takes effect in real time (hot release)**
   * After the user modified the configuration and released it in Apollo, the sdk will receive the latest configurations in real time (1 second) and notify the application
-
 * **Release version management**
   * Every configuration releases are versioned, which is friendly to support configuration rollback
-
 * **Grayscale release**
   * Support grayscale configuration release, for example, after clicking release, it will only take effect for some application instances. After a period of observation, we could push the configurations to all application instances if there is no problem
-
+* **Global Search Configuration Items**
+  * A fuzzy search of the key and value of a configuration item finds in which application, environment, cluster, namespace the configuration item with the corresponding value is used
+  * It is easy for administrators and SRE roles to quickly and easily find and change the configuration values of resources by highlighting, paging and jumping through configurations
 * **Authorization management, release approval and operation audit**
   * Great authorization mechanism is designed for applications and configurations management, and the management of configurations is divided into two operations: editing and publishing, therefore greatly reducing human errors
   * All operations have audit logs for easy tracking of problems
-
 * **Client side configuration information monitoring**
   * It's very easy to see which instances are using the configurations and what versions they are using
-
 * **Rich SDKs available**
   * Provides native sdks of Java and .Net to facilitate application integration
   * Support Spring Placeholder, Annotation and Spring Boot ConfigurationProperties for easy application use (requires Spring 3.1.1+)
   * Http APIs are provided, so non-Java and .Net applications can integrate conveniently
   * Rich third party sdks are also available, e.g. Golang, Python, NodeJS, PHP, C, etc
-
 * **Open platform API**
   * Apollo itself provides a unified configuration management interface, which supports features such as multi-environment, multi-data center configuration management, permissions, and process governance
   * However, for the sake of versatility, Apollo will not put too many restrictions on the modification of the configuration, as long as it conforms to the basic format, it can be saved.
   * In our research, we found that for some users, their configurations may have more complicated formats, such as xml, json, and the format needs to be verified
   * There are also some users such as DAL, which not only have a specific format, but also need to verify the entered value before saving, such as checking whether the database, username and password match
   * For this type of application, Apollo allows the application to modify and release configurations through open APIs, which has great authorization and permission control mechanism built in
-
 * **Simple deployment**
   * As an infrastructure service, the configuration center has very high availability requirements, which forces Apollo to rely on external dependencies as little as possible
   * Currently, the only external dependency is MySQL, so the deployment is very simple. Apollo can run as long as Java and MySQL are installed

apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/AccessKeyController.java

@@ -16,6 +16,8 @@
  */
 package com.ctrip.framework.apollo.adminservice.controller;
 
+import static com.ctrip.framework.apollo.common.constants.AccessKeyMode.FILTER;
+
 import com.ctrip.framework.apollo.biz.entity.AccessKey;
 import com.ctrip.framework.apollo.biz.service.AccessKeyService;
 import com.ctrip.framework.apollo.common.dto.AccessKeyDTO;
@@ -27,6 +29,7 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 /**
@@ -61,9 +64,11 @@ public void delete(@PathVariable String appId, @PathVariable long id, String ope
   }
 
   @PutMapping(value = "/apps/{appId}/accesskeys/{id}/enable")
-  public void enable(@PathVariable String appId, @PathVariable long id, String operator) {
+  public void enable(@PathVariable String appId, @PathVariable long id,
+      @RequestParam(required = false, defaultValue = "" + FILTER) int mode, String operator) {
     AccessKey entity = new AccessKey();
     entity.setId(id);
+    entity.setMode(mode);
     entity.setEnabled(true);
     entity.setDataChangeLastModifiedBy(operator);
 
@@ -74,6 +79,7 @@ public void enable(@PathVariable String appId, @PathVariable long id, String ope
   public void disable(@PathVariable String appId, @PathVariable long id, String operator) {
     AccessKey entity = new AccessKey();
     entity.setId(id);
+    entity.setMode(FILTER);
     entity.setEnabled(false);
     entity.setDataChangeLastModifiedBy(operator);
 

apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java

@@ -27,6 +27,7 @@
 import com.ctrip.framework.apollo.biz.service.ReleaseService;
 import com.ctrip.framework.apollo.biz.utils.ConfigChangeContentBuilder;
 import com.ctrip.framework.apollo.common.dto.ItemDTO;
+import com.ctrip.framework.apollo.common.dto.ItemInfoDTO;
 import com.ctrip.framework.apollo.common.dto.PageDTO;
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.exception.NotFoundException;
@@ -201,6 +202,14 @@ public List<ItemDTO> findDeletedItems(@PathVariable("appId") String appId,
     return Collections.emptyList();
   }
 
+  @GetMapping("/items-search/key-and-value")
+  public PageDTO<ItemInfoDTO> getItemInfoBySearch(@RequestParam(value = "key", required = false) String key,
+                                                  @RequestParam(value = "value", required = false) String value,
+                                                  Pageable limit) {
+    Page<ItemInfoDTO> pageItemInfoDTO = itemService.getItemInfoBySearch(key, value, limit);
+    return new PageDTO<>(pageItemInfoDTO.getContent(), limit, pageItemInfoDTO.getTotalElements());
+  }
+
   @GetMapping("/items/{itemId}")
   public ItemDTO get(@PathVariable("itemId") long itemId) {
     Item item = itemService.findOne(itemId);

apollo-adminservice/src/test/java/com/ctrip/framework/apollo/adminservice/controller/ItemControllerTest.java

@@ -21,18 +21,19 @@
 import com.ctrip.framework.apollo.biz.entity.Commit;
 import com.ctrip.framework.apollo.biz.repository.CommitRepository;
 import com.ctrip.framework.apollo.biz.repository.ItemRepository;
-import com.ctrip.framework.apollo.common.dto.AppDTO;
-import com.ctrip.framework.apollo.common.dto.ClusterDTO;
-import com.ctrip.framework.apollo.common.dto.ItemDTO;
-import com.ctrip.framework.apollo.common.dto.NamespaceDTO;
+import com.ctrip.framework.apollo.biz.service.ItemService;
+import com.ctrip.framework.apollo.common.dto.*;
+
 import java.util.List;
 import java.util.Objects;
 import org.junit.Assert;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
 import org.springframework.test.context.jdbc.Sql;
 import org.springframework.test.context.jdbc.Sql.ExecutionPhase;
 
@@ -48,6 +49,9 @@ public class ItemControllerTest extends AbstractControllerTest {
   @Autowired
   private ItemRepository itemRepository;
 
+  @Autowired
+  private ItemService itemService;
+
   @Test
   @Sql(scripts = "/controller/test-itemset.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD)
   @Sql(scripts = "/controller/cleanup.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD)
@@ -58,7 +62,7 @@ public void testCreate() {
     ClusterDTO cluster = restTemplate.getForObject(clusterBaseUrl(), ClusterDTO.class, app.getAppId(), "default");
     assert cluster != null;
     NamespaceDTO namespace = restTemplate.getForObject(namespaceBaseUrl(),
-        NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
+            NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
 
     String itemKey = "test-key";
     String itemValue = "test-value";
@@ -68,12 +72,12 @@ public void testCreate() {
     item.setDataChangeLastModifiedBy("apollo");
 
     ResponseEntity<ItemDTO> response = restTemplate.postForEntity(itemBaseUrl(),
-        item, ItemDTO.class, app.getAppId(), cluster.getName(), namespace.getNamespaceName());
+            item, ItemDTO.class, app.getAppId(), cluster.getName(), namespace.getNamespaceName());
     Assert.assertEquals(HttpStatus.OK, response.getStatusCode());
     Assert.assertEquals(itemKey, Objects.requireNonNull(response.getBody()).getKey());
 
     List<Commit> commitList = commitRepository.findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(app.getAppId(), cluster.getName(), namespace.getNamespaceName(),
-        Pageable.ofSize(10));
+            Pageable.ofSize(10));
     Assert.assertEquals(1, commitList.size());
 
     Commit commit = commitList.get(0);
@@ -93,15 +97,15 @@ public void testUpdate() {
     ClusterDTO cluster = restTemplate.getForObject(clusterBaseUrl(), ClusterDTO.class, app.getAppId(), "default");
     assert cluster != null;
     NamespaceDTO namespace = restTemplate.getForObject(namespaceBaseUrl(),
-        NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
+            NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
 
     String itemKey = "test-key";
     String itemValue = "test-value-updated";
 
     long itemId = itemRepository.findByKey(itemKey, Pageable.ofSize(1))
-        .getContent()
-        .get(0)
-        .getId();
+            .getContent()
+            .get(0)
+            .getId();
     ItemDTO item = new ItemDTO(itemKey, itemValue, "", 1);
     item.setDataChangeLastModifiedBy("apollo");
 
@@ -115,7 +119,7 @@ public void testUpdate() {
     });
 
     List<Commit> commitList = commitRepository.findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(app.getAppId(), cluster.getName(), namespace.getNamespaceName(),
-        Pageable.ofSize(10));
+            Pageable.ofSize(10));
     assertThat(commitList).hasSize(2);
   }
 
@@ -131,23 +135,44 @@ public void testDelete() {
     ClusterDTO cluster = restTemplate.getForObject(clusterBaseUrl(), ClusterDTO.class, app.getAppId(), "default");
     assert cluster != null;
     NamespaceDTO namespace = restTemplate.getForObject(namespaceBaseUrl(),
-        NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
+            NamespaceDTO.class, app.getAppId(), cluster.getName(), "application");
 
     String itemKey = "test-key";
 
     long itemId = itemRepository.findByKey(itemKey, Pageable.ofSize(1))
-        .getContent()
-        .get(0)
-        .getId();
+            .getContent()
+            .get(0)
+            .getId();
 
     String deleteUrl = url(  "/items/{itemId}?operator=apollo");
     restTemplate.delete(deleteUrl, itemId);
     assertThat(itemRepository.findById(itemId).isPresent())
-        .isFalse();
+            .isFalse();
 
     assert namespace != null;
     List<Commit> commitList = commitRepository.findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(app.getAppId(), cluster.getName(), namespace.getNamespaceName(),
-        Pageable.ofSize(10));
+            Pageable.ofSize(10));
     assertThat(commitList).hasSize(2);
   }
+
+  @Test
+  @Sql(scripts = "/controller/test-itemset.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD)
+  @Sql(scripts = "/controller/cleanup.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD)
+  public void testSearch() {
+    this.testCreate();
+
+    String itemKey = "test-key";
+    String itemValue = "test-value";
+    Page<ItemInfoDTO> itemInfoDTOS = itemService.getItemInfoBySearch(itemKey, itemValue, PageRequest.of(0, 200));
+    HttpHeaders headers = new HttpHeaders();
+    HttpEntity<Void> entity = new HttpEntity<>(headers);
+    ResponseEntity<PageDTO<ItemInfoDTO>> response = restTemplate.exchange(
+            url("/items-search/key-and-value?key={key}&value={value}&page={page}&size={size}"),
+            HttpMethod.GET,
+            entity,
+            new ParameterizedTypeReference<PageDTO<ItemInfoDTO>>() {},
+            itemKey, itemValue, 0, 200
+    );
+    assertThat(itemInfoDTOS.getContent().toString()).isEqualTo(response.getBody().getContent().toString());
+  }
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java

@@ -21,12 +21,14 @@
 import com.ctrip.framework.apollo.common.config.RefreshablePropertySource;
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.springframework.stereotype.Component;
@@ -36,6 +38,9 @@ public class BizConfig extends RefreshableConfig {
 
   private static final int DEFAULT_ITEM_KEY_LENGTH = 128;
   private static final int DEFAULT_ITEM_VALUE_LENGTH = 20000;
+
+  private static final int DEFAULT_MAX_NAMESPACE_NUM = 200;
+
   private static final int DEFAULT_APPNAMESPACE_CACHE_REBUILD_INTERVAL = 60; //60s
   private static final int DEFAULT_GRAY_RELEASE_RULE_SCAN_INTERVAL = 60; //60s
   private static final int DEFAULT_APPNAMESPACE_CACHE_SCAN_INTERVAL = 1; //1s
@@ -99,6 +104,19 @@ public int itemValueLengthLimit() {
     return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_ITEM_VALUE_LENGTH);
   }
 
+  public boolean isNamespaceNumLimitEnabled() {
+    return getBooleanProperty("namespace.num.limit.enabled", false);
+  }
+
+  public int namespaceNumLimit() {
+    int limit = getIntProperty("namespace.num.limit", DEFAULT_MAX_NAMESPACE_NUM);
+    return checkInt(limit, 0, Integer.MAX_VALUE, DEFAULT_MAX_NAMESPACE_NUM);
+  }
+
+  public Set<String> namespaceNumLimitWhite() {
+    return Sets.newHashSet(getArrayProperty("namespace.num.limit.white", new String[0]));
+  }
+
   public Map<Long, Integer> namespaceValueLengthLimitOverride() {
     String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
     Map<Long, Integer> namespaceValueLengthOverride = Maps.newHashMap();

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/entity/AccessKey.java

@@ -36,6 +36,9 @@ public class AccessKey extends BaseEntity {
   @Column(name = "`Secret`", nullable = false)
   private String secret;
 
+  @Column(name = "`Mode`")
+  private int mode;
+
   @Column(name = "`IsEnabled`", columnDefinition = "Bit default '0'")
   private boolean enabled;
 
@@ -55,6 +58,14 @@ public void setSecret(String secret) {
     this.secret = secret;
   }
 
+  public int getMode() {
+    return mode;
+  }
+
+  public void setMode(int mode) {
+    this.mode = mode;
+  }
+
   public boolean isEnabled() {
     return enabled;
   }
@@ -66,6 +77,6 @@ public void setEnabled(boolean enabled) {
   @Override
   public String toString() {
     return toStringHelper().add("appId", appId).add("secret", secret)
-        .add("enabled", enabled).toString();
+        .add("mode", mode).add("enabled", enabled).toString();
   }
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java

@@ -18,11 +18,13 @@
 
 import com.ctrip.framework.apollo.biz.entity.Item;
 
+import com.ctrip.framework.apollo.common.dto.ItemInfoDTO;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.PagingAndSortingRepository;
+import org.springframework.data.repository.query.Param;
 
 import java.util.Date;
 import java.util.List;
@@ -43,6 +45,21 @@ public interface ItemRepository extends PagingAndSortingRepository<Item, Long> {
 
   Item findFirst1ByNamespaceIdOrderByLineNumDesc(Long namespaceId);
 
+  @Query("SELECT new com.ctrip.framework.apollo.common.dto.ItemInfoDTO(n.appId, n.clusterName, n.namespaceName, i.key, i.value) " +
+          "FROM Item i RIGHT JOIN Namespace n ON i.namespaceId = n.id " +
+          "WHERE i.key LIKE %:key% AND i.value LIKE %:value% AND i.isDeleted = 0")
+  Page<ItemInfoDTO> findItemsByKeyAndValueLike(@Param("key") String key, @Param("value") String value, Pageable pageable);
+
+  @Query("SELECT new com.ctrip.framework.apollo.common.dto.ItemInfoDTO(n.appId, n.clusterName, n.namespaceName, i.key, i.value) " +
+          "FROM Item i RIGHT JOIN Namespace n ON i.namespaceId = n.id " +
+          "WHERE i.key LIKE %:key% AND i.isDeleted = 0")
+  Page<ItemInfoDTO> findItemsByKeyLike(@Param("key") String key, Pageable pageable);
+
+  @Query("SELECT new com.ctrip.framework.apollo.common.dto.ItemInfoDTO(n.appId, n.clusterName, n.namespaceName, i.key, i.value) " +
+          "FROM Item i RIGHT JOIN Namespace n ON i.namespaceId = n.id " +
+          "WHERE i.value LIKE %:value% AND i.isDeleted = 0")
+  Page<ItemInfoDTO> findItemsByValueLike(@Param("value") String value, Pageable pageable);
+
   @Modifying
   @Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false")
   int deleteByNamespaceId(long namespaceId, String operator);

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/NamespaceRepository.java

@@ -44,4 +44,6 @@ public interface NamespaceRepository extends PagingAndSortingRepository<Namespac
 
   int countByNamespaceNameAndAppIdNot(String namespaceName, String appId);
 
+  int countByAppIdAndClusterName(String appId, String clusterName);
+
 }

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/AccessKeyService.java

@@ -74,6 +74,7 @@ public AccessKey update(String appId, AccessKey entity) {
       throw BadRequestException.accessKeyNotExists();
     }
 
+    accessKey.setMode(entity.getMode());
     accessKey.setEnabled(entity.isEnabled());
     accessKey.setDataChangeLastModifiedBy(operator);
     accessKeyRepository.save(accessKey);