support export/import configs

apolloconfig · Sep 22, 2021 · ea7e4f3 · ea7e4f3
1 parent 0b5d227
commit ea7e4f3
Show file tree

Hide file tree

Showing 25 changed files with 1,452 additions and 313 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -13,6 +13,8 @@ Apollo 1.10.0
 * [remove ctrip profile](https://github.com/ctripcorp/apollo/pull/3920)
 * [Remove spring dependencies from internal code](https://github.com/apolloconfig/apollo/pull/3937)
 * [Fix issue: ingress syntax](https://github.com/apolloconfig/apollo/pull/3933)
+* [Support export/import configs](https://github.com/apolloconfig/apollo/pull/3947)
+
 
 ------------------
 All issues and pull requests are [here](https://github.com/ctripcorp/apollo/milestone/8?closed=1)
diff --git a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java
@@ -19,6 +19,7 @@
 import com.google.gson.reflect.TypeToken;
 
 import com.ctrip.framework.apollo.common.dto.GrayReleaseRuleItemDTO;
+import com.ctrip.framework.apollo.common.dto.ItemDTO;
 
 import java.lang.reflect.Type;
 import java.util.List;
@@ -30,4 +31,5 @@ public interface GsonType {
 
   Type RULE_ITEMS = new TypeToken<List<GrayReleaseRuleItemDTO>>() {}.getType();
 
+  Type ITEM_DTOS = new TypeToken<List<ItemDTO>>(){}.getType();
 }
diff --git a/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java b/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java
@@ -16,21 +16,17 @@
  */
 package com.ctrip.framework.apollo.portal.controller;
 
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
 import com.ctrip.framework.apollo.common.exception.ServiceException;
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
 import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.ConfigsExportService;
 import com.ctrip.framework.apollo.portal.service.NamespaceService;
 import com.ctrip.framework.apollo.portal.util.NamespaceBOUtils;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Date;
-import java.util.List;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
+
 import org.apache.commons.lang.time.DateFormatUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,15 +35,26 @@
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
 /**
  * jian.tan
  */
 @RestController
 public class ConfigsExportController {
 
-  private static final Logger logger = LoggerFactory.getLogger(ConfigsExportController.class);
+  private static final Logger logger        = LoggerFactory.getLogger(ConfigsExportController.class);
+  private static final String ENV_SEPARATOR = ",";
 
   private final ConfigsExportService configsExportService;
 
@@ -62,9 +69,7 @@ public ConfigsExportController(
   }
 
   /**
-   * export one config as file.
-   * keep compatibility.
-   * file name examples:
+   * export one config as file. keep compatibility. file name examples:
    * <pre>
    *   application.properties
    *   application.yml
@@ -74,8 +79,8 @@ public ConfigsExportController(
   @PreAuthorize(value = "[email protected](#appId, #env, #namespaceName)")
   @GetMapping("/apps/{appId}/envs/{env}/clusters/{clusterName}/namespaces/{namespaceName}/items/export")
   public void exportItems(@PathVariable String appId, @PathVariable String env,
-      @PathVariable String clusterName, @PathVariable String namespaceName,
-      HttpServletResponse res) {
+                          @PathVariable String clusterName, @PathVariable String namespaceName,
+                          HttpServletResponse res) {
     List<String> fileNameSplit = Splitter.on(".").splitToList(namespaceName);
 
     String fileName = fileNameSplit.size() <= 1 ? Joiner.on(".")
@@ -96,21 +101,26 @@ public void exportItems(@PathVariable String appId, @PathVariable String env,
   }
 
   /**
-   * Export all configs in a compressed file.
-   * Just export namespace which current exists read permission.
-   * The permission check in service.
+   * Export all configs in a compressed file. Just export namespace which current exists read permission. The permission
+   * check in service.
    */
-  @GetMapping("/export")
-  public void exportAll(HttpServletRequest request, HttpServletResponse response) throws IOException {
+  @GetMapping("/configs/export")
+  public void exportAll(@RequestParam(value = "envs") String envs,
+                        HttpServletRequest request, HttpServletResponse response) throws IOException {
     // filename must contain the information of time
     final String filename = "apollo_config_export_" + DateFormatUtils.format(new Date(), "yyyy_MMdd_HH_mm_ss") + ".zip";
     // log who download the configs
-    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(), request.getRemoteHost(), filename);
+    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(),
+                request.getRemoteHost(), filename);
     // set downloaded filename
     response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + filename);
 
+    List<Env>
+        exportEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
     try (OutputStream outputStream = response.getOutputStream()) {
-      configsExportService.exportAllTo(outputStream);
+      configsExportService.exportData(outputStream, exportEnvs);
     }
   }
 

diff --git a/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java b/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java
@@ -16,27 +16,38 @@
  */
 package com.ctrip.framework.apollo.portal.controller;
 
+import com.google.common.base.Splitter;
+
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
+import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.ConfigsImportService;
 import com.ctrip.framework.apollo.portal.util.ConfigFileUtils;
-import java.io.IOException;
+
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.zip.ZipInputStream;
+
 /**
  * Import the configs from file.
  * First version: move code from {@link ConfigsExportController}
  * @author wxq
  */
 @RestController
 public class ConfigsImportController {
+  private static final String ENV_SEPARATOR = ",";
 
   private final ConfigsImportService configsImportService;
 
+
   public ConfigsImportController(
       final ConfigsImportService configsImportService
   ) {
@@ -58,8 +69,38 @@ public void importConfigFile(@PathVariable String appId, @PathVariable String en
     // check file
     ConfigFileUtils.check(file);
     final String format = ConfigFileUtils.getFormat(file.getOriginalFilename());
-    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName, namespaceName, 
-            ConfigFileFormat.fromString(format));
-    configsImportService.importOneConfigFromFile(env, standardFilename, file.getInputStream());
+    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName,
+                                                               namespaceName,
+                                                               ConfigFileFormat.fromString(format));
+
+    configsImportService.forceImportNamespaceFromFile(Env.valueOf(env), standardFilename, file.getInputStream());
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=cover")
+  public void importConfigByZipWithCoverConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List<Env>
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, false);
+    }
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=ignore")
+  public void importConfigByZipWithIgnoreConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List<Env>
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, true);
+    }
   }
 }
diff --git a/...portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java b/...portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java
@@ -69,7 +69,7 @@ public ResponseEntity<Number> getInstanceCountByNamespace(@PathVariable String e
                                                               @RequestParam String clusterName,
                                                               @RequestParam String namespaceName) {
 
-        int count = instanceService.getInstanceCountByNamepsace(appId, Env.valueOf(env), clusterName, namespaceName);
+        int count = instanceService.getInstanceCountByNamespace(appId, Env.valueOf(env), clusterName, namespaceName);
         return ResponseEntity.ok(new Number(count));
     }
 

diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java
@@ -40,22 +40,25 @@ public class ConfigBO {
 
   private final ConfigFileFormat format;
 
+  private final boolean isPublic;
+
   public ConfigBO(Env env, String ownerName, String appId, String clusterName,
-      String namespace, String configFileContent, ConfigFileFormat format) {
+      String namespace, boolean isPublic, String configFileContent, ConfigFileFormat format) {
     this.env = env;
     this.ownerName = ownerName;
     this.appId = appId;
     this.clusterName = clusterName;
     this.namespace = namespace;
+    this.isPublic = isPublic;
     this.configFileContent = configFileContent;
     this.format = format;
   }
 
   public ConfigBO(Env env, String ownerName, String appId, String clusterName, NamespaceBO namespaceBO) {
     this(env, ownerName, appId, clusterName,
-        namespaceBO.getBaseInfo().getNamespaceName(),
-        NamespaceBOUtils.convert2configFileContent(namespaceBO),
-        ConfigFileFormat.fromString(namespaceBO.getFormat())
+         namespaceBO.getBaseInfo().getNamespaceName(), namespaceBO.isPublic(),
+         NamespaceBOUtils.convert2configFileContent(namespaceBO),
+         ConfigFileFormat.fromString(namespaceBO.getFormat())
     );
   }
 
@@ -67,6 +70,7 @@ public String toString() {
         ", appId='" + appId + '\'' +
         ", clusterName='" + clusterName + '\'' +
         ", namespace='" + namespace + '\'' +
+        ", isPublic='" + isPublic + '\'' +
         ", configFileContent='" + configFileContent + '\'' +
         ", format=" + format +
         '}';
@@ -99,4 +103,8 @@ public String getConfigFileContent() {
   public ConfigFileFormat getFormat() {
     return format;
   }
+
+  public boolean isPublic() {
+    return isPublic;
+  }
 }
diff --git a/...rtal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java b/...rtal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java
@@ -30,6 +30,7 @@ public class NamespaceTextModel implements Verifiable {
   private long namespaceId;
   private String format;
   private String configText;
+  private String operator;
 
 
   @Override
@@ -92,4 +93,12 @@ public ConfigFileFormat getFormat() {
   public void setFormat(String format) {
     this.format = format;
   }
+
+  public String getOperator() {
+    return operator;
+  }
+
+  public void setOperator(String operator) {
+    this.operator = operator;
+  }
 }
diff --git a/...o-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java b/...o-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java
@@ -25,6 +25,7 @@
 import com.ctrip.framework.apollo.portal.repository.AppNamespaceRepository;
 import com.ctrip.framework.apollo.portal.spi.UserInfoHolder;
 import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
@@ -89,6 +90,11 @@ public List<AppNamespace> findByAppId(String appId) {
     return appNamespaceRepository.findByAppId(appId);
   }
 
+  public List<AppNamespace> findAll() {
+    Iterable<AppNamespace> appNamespaces = appNamespaceRepository.findAll();
+    return Lists.newArrayList(appNamespaces);
+  }
+
   @Transactional
   public void createDefaultAppNamespace(String appId) {
     if (!isAppNamespaceNameUnique(appId, ConfigConsts.NAMESPACE_APPLICATION)) {
@@ -171,6 +177,30 @@ public AppNamespace createAppNamespaceInLocal(AppNamespace appNamespace, boolean
     return createdAppNamespace;
   }
 
+  @Transactional
+  public AppNamespace importAppNamespaceInLocal(AppNamespace appNamespace) {
+    // globally uniqueness check for public app namespace
+    if (appNamespace.isPublic()) {
+      checkAppNamespaceGlobalUniqueness(appNamespace);
+    } else {
+      // check private app namespace
+      if (appNamespaceRepository.findByAppIdAndName(appNamespace.getAppId(), appNamespace.getName()) != null) {
+        throw new BadRequestException("Private AppNamespace " + appNamespace.getName() + " already exists!");
+      }
+      // should not have the same with public app namespace
+      checkPublicAppNamespaceGlobalUniqueness(appNamespace);
+    }
+
+    AppNamespace createdAppNamespace = appNamespaceRepository.save(appNamespace);
+
+    String operator = appNamespace.getDataChangeCreatedBy();
+
+    roleInitializationService.initNamespaceRoles(appNamespace.getAppId(), appNamespace.getName(), operator);
+    roleInitializationService.initNamespaceEnvRoles(appNamespace.getAppId(), appNamespace.getName(), operator);
+
+    return createdAppNamespace;
+  }
+
   private void checkAppNamespaceGlobalUniqueness(AppNamespace appNamespace) {
     checkPublicAppNamespaceGlobalUniqueness(appNamespace);
 

diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java
@@ -21,6 +21,7 @@
 import com.ctrip.framework.apollo.common.entity.App;
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
+import com.ctrip.framework.apollo.core.utils.StringUtils;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.api.AdminServiceAPI;
 import com.ctrip.framework.apollo.portal.constant.TracerEventType;
@@ -114,9 +115,11 @@ public AppDTO load(Env env, String appId) {
   }
 
   public void createAppInRemote(Env env, App app) {
-    String username = userInfoHolder.getUser().getUserId();
-    app.setDataChangeCreatedBy(username);
-    app.setDataChangeLastModifiedBy(username);
+    if (StringUtils.isBlank(app.getDataChangeCreatedBy())) {
+      String username = userInfoHolder.getUser().getUserId();
+      app.setDataChangeCreatedBy(username);
+      app.setDataChangeLastModifiedBy(username);
+    }
 
     AppDTO appDTO = BeanUtils.transform(AppDTO.class, app);
     appAPI.createApp(env, appDTO);
@@ -151,6 +154,31 @@ public App createAppInLocal(App app) {
     return createdApp;
   }
 
+  @Transactional
+  public App importAppInLocal(App app) {
+    String appId = app.getAppId();
+    App managedApp = appRepository.findByAppId(appId);
+
+    if (managedApp != null) {
+      return app;
+    }
+
+    UserInfo owner = userService.findByUserId(app.getOwnerName());
+    if (owner == null) {
+      throw new BadRequestException("Application's owner not exist.");
+    }
+
+    app.setOwnerEmail(owner.getEmail());
+
+    App createdApp = appRepository.save(app);
+
+    roleInitializationService.initAppRoles(createdApp);
+
+    Tracer.logEvent(TracerEventType.CREATE_APP, appId);
+
+    return createdApp;
+  }
+
   @Transactional
   public App updateAppInLocal(App app) {
     String appId = app.getAppId();