diff --git a/CHANGES.md b/CHANGES.md index 937a7ab3ff5..6f1726199d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ Apollo 1.10.0 * [remove ctrip profile](https://github.com/ctripcorp/apollo/pull/3920) * [Remove spring dependencies from internal code](https://github.com/apolloconfig/apollo/pull/3937) * [Fix issue: ingress syntax](https://github.com/apolloconfig/apollo/pull/3933) +* [Support export/import configs](https://github.com/apolloconfig/apollo/pull/3947) + ------------------ All issues and pull requests are [here](https://github.com/ctripcorp/apollo/milestone/8?closed=1) diff --git a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java index 3bbbc3b0b08..8526f0a0349 100644 --- a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java +++ b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java @@ -19,6 +19,7 @@ import com.google.gson.reflect.TypeToken; import com.ctrip.framework.apollo.common.dto.GrayReleaseRuleItemDTO; +import com.ctrip.framework.apollo.common.dto.ItemDTO; import java.lang.reflect.Type; import java.util.List; @@ -30,4 +31,5 @@ public interface GsonType { Type RULE_ITEMS = new TypeToken>() {}.getType(); + Type ITEM_DTOS = new TypeToken>(){}.getType(); } diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java index 3cc73136440..9306c32c574 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java @@ -16,6 +16,9 @@ */ package com.ctrip.framework.apollo.portal.controller; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; + import com.ctrip.framework.apollo.common.exception.ServiceException; import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO; @@ -23,14 +26,7 @@ import com.ctrip.framework.apollo.portal.service.ConfigsExportService; import com.ctrip.framework.apollo.portal.service.NamespaceService; import com.ctrip.framework.apollo.portal.util.NamespaceBOUtils; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Date; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; + import org.apache.commons.lang.time.DateFormatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,15 +35,26 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + /** * jian.tan */ @RestController public class ConfigsExportController { - private static final Logger logger = LoggerFactory.getLogger(ConfigsExportController.class); + private static final Logger logger = LoggerFactory.getLogger(ConfigsExportController.class); + private static final String ENV_SEPARATOR = ","; private final ConfigsExportService configsExportService; @@ -62,9 +69,7 @@ public ConfigsExportController( } /** - * export one config as file. - * keep compatibility. - * file name examples: + * export one config as file. keep compatibility. file name examples: *
    *   application.properties
    *   application.yml
@@ -74,8 +79,8 @@ public ConfigsExportController(
   @PreAuthorize(value = "!@permissionValidator.shouldHideConfigToCurrentUser(#appId, #env, #namespaceName)")
   @GetMapping("/apps/{appId}/envs/{env}/clusters/{clusterName}/namespaces/{namespaceName}/items/export")
   public void exportItems(@PathVariable String appId, @PathVariable String env,
-      @PathVariable String clusterName, @PathVariable String namespaceName,
-      HttpServletResponse res) {
+                          @PathVariable String clusterName, @PathVariable String namespaceName,
+                          HttpServletResponse res) {
     List fileNameSplit = Splitter.on(".").splitToList(namespaceName);
 
     String fileName = fileNameSplit.size() <= 1 ? Joiner.on(".")
@@ -96,21 +101,26 @@ public void exportItems(@PathVariable String appId, @PathVariable String env,
   }
 
   /**
-   * Export all configs in a compressed file.
-   * Just export namespace which current exists read permission.
-   * The permission check in service.
+   * Export all configs in a compressed file. Just export namespace which current exists read permission. The permission
+   * check in service.
    */
-  @GetMapping("/export")
-  public void exportAll(HttpServletRequest request, HttpServletResponse response) throws IOException {
+  @GetMapping("/configs/export")
+  public void exportAll(@RequestParam(value = "envs") String envs,
+                        HttpServletRequest request, HttpServletResponse response) throws IOException {
     // filename must contain the information of time
     final String filename = "apollo_config_export_" + DateFormatUtils.format(new Date(), "yyyy_MMdd_HH_mm_ss") + ".zip";
     // log who download the configs
-    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(), request.getRemoteHost(), filename);
+    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(),
+                request.getRemoteHost(), filename);
     // set downloaded filename
     response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + filename);
 
+    List
+        exportEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
     try (OutputStream outputStream = response.getOutputStream()) {
-      configsExportService.exportAllTo(outputStream);
+      configsExportService.exportData(outputStream, exportEnvs);
     }
   }
 
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java
index 672edf70962..c187ec934f9 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java
@@ -16,10 +16,13 @@
  */
 package com.ctrip.framework.apollo.portal.controller;
 
+import com.google.common.base.Splitter;
+
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
+import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.ConfigsImportService;
 import com.ctrip.framework.apollo.portal.util.ConfigFileUtils;
-import java.io.IOException;
+
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -27,6 +30,12 @@
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.zip.ZipInputStream;
+
 /**
  * Import the configs from file.
  * First version: move code from {@link ConfigsExportController}
@@ -34,9 +43,11 @@
  */
 @RestController
 public class ConfigsImportController {
+  private static final String ENV_SEPARATOR = ",";
 
   private final ConfigsImportService configsImportService;
 
+
   public ConfigsImportController(
       final ConfigsImportService configsImportService
   ) {
@@ -58,8 +69,38 @@ public void importConfigFile(@PathVariable String appId, @PathVariable String en
     // check file
     ConfigFileUtils.check(file);
     final String format = ConfigFileUtils.getFormat(file.getOriginalFilename());
-    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName, namespaceName, 
-            ConfigFileFormat.fromString(format));
-    configsImportService.importOneConfigFromFile(env, standardFilename, file.getInputStream());
+    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName,
+                                                               namespaceName,
+                                                               ConfigFileFormat.fromString(format));
+
+    configsImportService.forceImportNamespaceFromFile(Env.valueOf(env), standardFilename, file.getInputStream());
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=cover")
+  public void importConfigByZipWithCoverConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, false);
+    }
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=ignore")
+  public void importConfigByZipWithIgnoreConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, true);
+    }
   }
 }
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java
index 3cd86966374..45425b51d0b 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java
@@ -69,7 +69,7 @@ public ResponseEntity getInstanceCountByNamespace(@PathVariable String e
                                                               @RequestParam String clusterName,
                                                               @RequestParam String namespaceName) {
 
-        int count = instanceService.getInstanceCountByNamepsace(appId, Env.valueOf(env), clusterName, namespaceName);
+        int count = instanceService.getInstanceCountByNamespace(appId, Env.valueOf(env), clusterName, namespaceName);
         return ResponseEntity.ok(new Number(count));
     }
 
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java
index 0ae0596d7a2..349bd4ee672 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java
@@ -40,22 +40,25 @@ public class ConfigBO {
 
   private final ConfigFileFormat format;
 
+  private final boolean isPublic;
+
   public ConfigBO(Env env, String ownerName, String appId, String clusterName,
-      String namespace, String configFileContent, ConfigFileFormat format) {
+      String namespace, boolean isPublic, String configFileContent, ConfigFileFormat format) {
     this.env = env;
     this.ownerName = ownerName;
     this.appId = appId;
     this.clusterName = clusterName;
     this.namespace = namespace;
+    this.isPublic = isPublic;
     this.configFileContent = configFileContent;
     this.format = format;
   }
 
   public ConfigBO(Env env, String ownerName, String appId, String clusterName, NamespaceBO namespaceBO) {
     this(env, ownerName, appId, clusterName,
-        namespaceBO.getBaseInfo().getNamespaceName(),
-        NamespaceBOUtils.convert2configFileContent(namespaceBO),
-        ConfigFileFormat.fromString(namespaceBO.getFormat())
+         namespaceBO.getBaseInfo().getNamespaceName(), namespaceBO.isPublic(),
+         NamespaceBOUtils.convert2configFileContent(namespaceBO),
+         ConfigFileFormat.fromString(namespaceBO.getFormat())
     );
   }
 
@@ -67,6 +70,7 @@ public String toString() {
         ", appId='" + appId + '\'' +
         ", clusterName='" + clusterName + '\'' +
         ", namespace='" + namespace + '\'' +
+        ", isPublic='" + isPublic + '\'' +
         ", configFileContent='" + configFileContent + '\'' +
         ", format=" + format +
         '}';
@@ -99,4 +103,8 @@ public String getConfigFileContent() {
   public ConfigFileFormat getFormat() {
     return format;
   }
+
+  public boolean isPublic() {
+    return isPublic;
+  }
 }
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java
index b33c89d82de..239aff0b39a 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java
@@ -30,6 +30,7 @@ public class NamespaceTextModel implements Verifiable {
   private long namespaceId;
   private String format;
   private String configText;
+  private String operator;
 
 
   @Override
@@ -92,4 +93,12 @@ public ConfigFileFormat getFormat() {
   public void setFormat(String format) {
     this.format = format;
   }
+
+  public String getOperator() {
+    return operator;
+  }
+
+  public void setOperator(String operator) {
+    this.operator = operator;
+  }
 }
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java
index 281a75fe793..a299ed7f79d 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppNamespaceService.java
@@ -25,6 +25,7 @@
 import com.ctrip.framework.apollo.portal.repository.AppNamespaceRepository;
 import com.ctrip.framework.apollo.portal.spi.UserInfoHolder;
 import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
@@ -89,6 +90,11 @@ public List findByAppId(String appId) {
     return appNamespaceRepository.findByAppId(appId);
   }
 
+  public List findAll() {
+    Iterable appNamespaces = appNamespaceRepository.findAll();
+    return Lists.newArrayList(appNamespaces);
+  }
+
   @Transactional
   public void createDefaultAppNamespace(String appId) {
     if (!isAppNamespaceNameUnique(appId, ConfigConsts.NAMESPACE_APPLICATION)) {
@@ -171,6 +177,30 @@ public AppNamespace createAppNamespaceInLocal(AppNamespace appNamespace, boolean
     return createdAppNamespace;
   }
 
+  @Transactional
+  public AppNamespace importAppNamespaceInLocal(AppNamespace appNamespace) {
+    // globally uniqueness check for public app namespace
+    if (appNamespace.isPublic()) {
+      checkAppNamespaceGlobalUniqueness(appNamespace);
+    } else {
+      // check private app namespace
+      if (appNamespaceRepository.findByAppIdAndName(appNamespace.getAppId(), appNamespace.getName()) != null) {
+        throw new BadRequestException("Private AppNamespace " + appNamespace.getName() + " already exists!");
+      }
+      // should not have the same with public app namespace
+      checkPublicAppNamespaceGlobalUniqueness(appNamespace);
+    }
+
+    AppNamespace createdAppNamespace = appNamespaceRepository.save(appNamespace);
+
+    String operator = appNamespace.getDataChangeCreatedBy();
+
+    roleInitializationService.initNamespaceRoles(appNamespace.getAppId(), appNamespace.getName(), operator);
+    roleInitializationService.initNamespaceEnvRoles(appNamespace.getAppId(), appNamespace.getName(), operator);
+
+    return createdAppNamespace;
+  }
+
   private void checkAppNamespaceGlobalUniqueness(AppNamespace appNamespace) {
     checkPublicAppNamespaceGlobalUniqueness(appNamespace);
 
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java
index 34632a28184..094cea24002 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/AppService.java
@@ -21,6 +21,7 @@
 import com.ctrip.framework.apollo.common.entity.App;
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
+import com.ctrip.framework.apollo.core.utils.StringUtils;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.api.AdminServiceAPI;
 import com.ctrip.framework.apollo.portal.constant.TracerEventType;
@@ -114,9 +115,11 @@ public AppDTO load(Env env, String appId) {
   }
 
   public void createAppInRemote(Env env, App app) {
-    String username = userInfoHolder.getUser().getUserId();
-    app.setDataChangeCreatedBy(username);
-    app.setDataChangeLastModifiedBy(username);
+    if (StringUtils.isBlank(app.getDataChangeCreatedBy())) {
+      String username = userInfoHolder.getUser().getUserId();
+      app.setDataChangeCreatedBy(username);
+      app.setDataChangeLastModifiedBy(username);
+    }
 
     AppDTO appDTO = BeanUtils.transform(AppDTO.class, app);
     appAPI.createApp(env, appDTO);
@@ -151,6 +154,31 @@ public App createAppInLocal(App app) {
     return createdApp;
   }
 
+  @Transactional
+  public App importAppInLocal(App app) {
+    String appId = app.getAppId();
+    App managedApp = appRepository.findByAppId(appId);
+
+    if (managedApp != null) {
+      return app;
+    }
+
+    UserInfo owner = userService.findByUserId(app.getOwnerName());
+    if (owner == null) {
+      throw new BadRequestException("Application's owner not exist.");
+    }
+
+    app.setOwnerEmail(owner.getEmail());
+
+    App createdApp = appRepository.save(app);
+
+    roleInitializationService.initAppRoles(createdApp);
+
+    Tracer.logEvent(TracerEventType.CREATE_APP, appId);
+
+    return createdApp;
+  }
+
   @Transactional
   public App updateAppInLocal(App app) {
     String appId = app.getAppId();
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsExportService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsExportService.java
index 126cacc24f4..431ec7a9a5d 100644
--- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsExportService.java
+++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsExportService.java
@@ -16,8 +16,13 @@
  */
 package com.ctrip.framework.apollo.portal.service;
 
+import com.google.gson.Gson;
+
 import com.ctrip.framework.apollo.common.dto.ClusterDTO;
 import com.ctrip.framework.apollo.common.entity.App;
+import com.ctrip.framework.apollo.common.entity.AppNamespace;
+import com.ctrip.framework.apollo.common.exception.BadRequestException;
+import com.ctrip.framework.apollo.common.exception.ServiceException;
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
 import com.ctrip.framework.apollo.portal.component.PermissionValidator;
 import com.ctrip.framework.apollo.portal.component.PortalSettings;
@@ -25,35 +30,40 @@
 import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.util.ConfigFileUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
-import java.util.function.BiFunction;
-import java.util.function.BinaryOperator;
 import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Service;
 
 @Service
 public class ConfigsExportService {
 
   private static final Logger logger = LoggerFactory.getLogger(ConfigsExportService.class);
 
+  private final Gson gson = new Gson();
+
   private final AppService appService;
 
   private final ClusterService clusterService;
 
   private final NamespaceService namespaceService;
 
+  private final AppNamespaceService appNamespaceService;
+
   private final PortalSettings portalSettings;
 
   private final PermissionValidator permissionValidator;
@@ -62,133 +72,242 @@ public ConfigsExportService(
       AppService appService,
       ClusterService clusterService,
       final @Lazy NamespaceService namespaceService,
+      final AppNamespaceService appNamespaceService,
       PortalSettings portalSettings,
       PermissionValidator permissionValidator) {
     this.appService = appService;
     this.clusterService = clusterService;
     this.namespaceService = namespaceService;
+    this.appNamespaceService = appNamespaceService;
     this.portalSettings = portalSettings;
     this.permissionValidator = permissionValidator;
   }
 
   /**
-   * write multiple namespace to a zip. use {@link Stream#reduce(Object, BiFunction,
-   * BinaryOperator)} to forbid concurrent write.
+   * Export all application which current user own them.
+   * 

+ * File Struts: + *

* - * @param configBOStream namespace's stream - * @param outputStream receive zip file output stream - * @throws IOException if happen write problem - */ - private static void writeAsZipOutputStream( - Stream configBOStream, OutputStream outputStream) throws IOException { - try (final ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { - final Consumer configBOConsumer = - configBO -> { - try { - // TODO, Stream.reduce will cause some problems. Is There other way to speed up the - // downloading? - synchronized (zipOutputStream) { - write2ZipOutputStream(zipOutputStream, configBO); - } - } catch (IOException e) { - logger.error("Write error. {}", configBO); - throw new IllegalStateException(e); - } - }; - configBOStream.forEach(configBOConsumer); - } - } - - /** - * write {@link ConfigBO} as file to {@link ZipOutputStream}. Watch out the concurrent problem! - * zip output stream is same like cannot write concurrently! the name of file is determined by - * {@link ConfigFileUtils#toFilename(String, String, String, ConfigFileFormat)}. the path of file - * is determined by {@link ConfigFileUtils#toFilePath(String, String, Env, String)}. + * List + * List -> List -> List -> List + * -----------------> app.metadata + * -------------------------------------------> List * - * @param zipOutputStream zip file output stream - * @param configBO a namespace represent - * @return zip file output stream same as parameter zipOutputStream + * @param outputStream network file download stream to user */ - private static ZipOutputStream write2ZipOutputStream( - final ZipOutputStream zipOutputStream, final ConfigBO configBO) throws IOException { - final Env env = configBO.getEnv(); - final String ownerName = configBO.getOwnerName(); - final String appId = configBO.getAppId(); - final String clusterName = configBO.getClusterName(); - final String namespace = configBO.getNamespace(); - final String configFileContent = configBO.getConfigFileContent(); - final ConfigFileFormat configFileFormat = configBO.getFormat(); - final String configFilename = - ConfigFileUtils.toFilename(appId, clusterName, namespace, configFileFormat); - final String filePath = ConfigFileUtils.toFilePath(ownerName, appId, env, configFilename); - final ZipEntry zipEntry = new ZipEntry(filePath); - try { - zipOutputStream.putNextEntry(zipEntry); - zipOutputStream.write(configFileContent.getBytes()); - zipOutputStream.closeEntry(); - } catch (IOException e) { - logger.error("config export failed. {}", configBO); - throw new IOException("config export failed", e); + public void exportData(OutputStream outputStream, List exportEnvs) { + if (CollectionUtils.isEmpty(exportEnvs)) { + exportEnvs = portalSettings.getActiveEnvs(); } - return zipOutputStream; - } - /** @return the namespaces current user exists */ - private Stream makeStreamBy( - final Env env, final String ownerName, final String appId, final String clusterName) { - final List namespaceBOS = - namespaceService.findNamespaceBOs(appId, env, clusterName); - final Function function = - namespaceBO -> new ConfigBO(env, ownerName, appId, clusterName, namespaceBO); - return namespaceBOS.parallelStream().map(function); + exportApps(exportEnvs, outputStream); } - private Stream makeStreamBy(final Env env, final String ownerName, final String appId) { - final List clusterDTOS = clusterService.findClusters(env, appId); - final Function> function = - clusterDTO -> this.makeStreamBy(env, ownerName, appId, clusterDTO.getName()); - return clusterDTOS.parallelStream().flatMap(function); - } + private void exportApps(final Collection exportEnvs, OutputStream outputStream) { + List hasPermissionApps = findHasPermissionApps(); + + if (CollectionUtils.isEmpty(hasPermissionApps)) { + return; + } - private Stream makeStreamBy(final Env env, final List apps) { - final Function> function = - app -> this.makeStreamBy(env, app.getOwnerName(), app.getAppId()); + try (final ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { + //write app info to zip + writeAppInfoToZip(hasPermissionApps, zipOutputStream); + + //export app namespace + exportAppNamespaces(zipOutputStream); - return apps.parallelStream().flatMap(function); + //export app's clusters + exportEnvs.parallelStream().forEach(env -> { + try { + this.exportClusters(env, hasPermissionApps, zipOutputStream); + } catch (Exception e) { + logger.error("export cluster error. env = {}", env, e); + } + }); + } catch (IOException e) { + logger.error("export config error", e); + throw new ServiceException("export config error", e); + } } - private Stream makeStreamBy(final Collection envs) { + private List findHasPermissionApps() { // get all apps final List apps = appService.findAll(); + if (CollectionUtils.isEmpty(apps)) { + return Collections.emptyList(); + } + // permission check final Predicate isAppAdmin = app -> { try { return permissionValidator.isAppAdmin(app.getAppId()); } catch (Exception e) { - logger.error("app = {}", app); - logger.error(app.getAppId()); + logger.error("permission check failed. app = {}", app); + return false; } - return false; }; // app admin permission filter - final List appsExistPermission = - apps.stream().filter(isAppAdmin).collect(Collectors.toList()); - return envs.parallelStream().flatMap(env -> this.makeStreamBy(env, appsExistPermission)); + return apps.stream().filter(isAppAdmin).collect(Collectors.toList()); } - /** - * Export all projects which current user own them. Permission check by {@link - * PermissionValidator#isAppAdmin(java.lang.String)} - * - * @param outputStream network file download stream to user - * @throws IOException if happen write problem - */ - public void exportAllTo(OutputStream outputStream) throws IOException { - final List activeEnvs = portalSettings.getActiveEnvs(); - final Stream configBOStream = this.makeStreamBy(activeEnvs); - writeAsZipOutputStream(configBOStream, outputStream); + private void writeAppInfoToZip(List apps, ZipOutputStream zipOutputStream) { + logger.info("to import app size = {}", apps.size()); + + final Consumer appConsumer = + app -> { + try { + synchronized (zipOutputStream) { + String fileName = ConfigFileUtils.genAppInfoPath(app); + String content = gson.toJson(app); + + writeToZip(fileName, content, zipOutputStream); + } + } catch (IOException e) { + logger.error("Write error. {}", app); + throw new ServiceException("Write app error. {}", e); + } + }; + + apps.forEach(appConsumer); } + + private void exportAppNamespaces(ZipOutputStream zipOutputStream) { + List appNamespaces = appNamespaceService.findAll(); + + logger.info("to import appnamespace size = " + appNamespaces.size()); + + Consumer appNamespaceConsumer = appNamespace -> { + try { + synchronized (zipOutputStream) { + String fileName = ConfigFileUtils.genAppNamespaceInfoPath(appNamespace); + String content = gson.toJson(appNamespace); + + writeToZip(fileName, content, zipOutputStream); + } + } catch (Exception e) { + logger.error("Write appnamespace error. {}", appNamespace); + throw new IllegalStateException(e); + } + }; + + appNamespaces.forEach(appNamespaceConsumer); + + } + + private void exportClusters(final Env env, final List exportApps, ZipOutputStream zipOutputStream) { + exportApps.parallelStream().forEach(exportApp -> { + try { + this.exportCluster(env, exportApp, zipOutputStream); + } catch (Exception e) { + logger.error("export cluster error. appId = {}", exportApp.getAppId(), e); + } + }); + } + + private void exportCluster(final Env env, final App exportApp, ZipOutputStream zipOutputStream) { + final List exportClusters = clusterService.findClusters(env, exportApp.getAppId()); + + if (CollectionUtils.isEmpty(exportClusters)) { + return; + } + + //write cluster info to zip + writeClusterInfoToZip(env, exportApp, exportClusters, zipOutputStream); + + //export namespaces + exportClusters.parallelStream().forEach(cluster -> { + try { + this.exportNamespaces(env, exportApp, cluster, zipOutputStream); + } catch (BadRequestException badRequestException) { + //ignore + } catch (Exception e) { + logger.error("export namespace error. appId = {}, cluster = {}", exportApp.getAppId(), cluster, e); + } + }); + } + + private void exportNamespaces(final Env env, final App exportApp, final ClusterDTO exportCluster, + ZipOutputStream zipOutputStream) { + String clusterName = exportCluster.getName(); + + List namespaceBOS = namespaceService.findNamespaceBOs(exportApp.getAppId(), env, clusterName); + + if (CollectionUtils.isEmpty(namespaceBOS)) { + return; + } + + Stream configBOStream = namespaceBOS.stream() + .map( + namespaceBO -> new ConfigBO(env, exportApp.getOwnerName(), exportApp.getAppId(), clusterName, namespaceBO)); + + writeNamespacesToZip(configBOStream, zipOutputStream); + } + + private void writeNamespacesToZip(Stream configBOStream, ZipOutputStream zipOutputStream) { + final Consumer configBOConsumer = + configBO -> { + try { + synchronized (zipOutputStream) { + String appId = configBO.getAppId(); + String clusterName = configBO.getClusterName(); + String namespace = configBO.getNamespace(); + String configFileContent = configBO.getConfigFileContent(); + ConfigFileFormat configFileFormat = configBO.getFormat(); + + String + configFileName = + ConfigFileUtils.toFilename(appId, clusterName, namespace, configFileFormat); + String filePath = + ConfigFileUtils.genNamespacePath(configBO.getOwnerName(), appId, configBO.getEnv(), configFileName); + + writeToZip(filePath, configFileContent, zipOutputStream); + } + } catch (IOException e) { + logger.error("Write error. {}", configBO); + throw new ServiceException("Write namespace error. {}", e); + } + }; + + configBOStream.forEach(configBOConsumer); + } + + private void writeClusterInfoToZip(Env env, App app, List exportClusters, + ZipOutputStream zipOutputStream) { + final Consumer clusterConsumer = + cluster -> { + try { + synchronized (zipOutputStream) { + String fileName = ConfigFileUtils.genClusterInfoPath(app, env, cluster); + String content = gson.toJson(cluster); + + writeToZip(fileName, content, zipOutputStream); + } + } catch (IOException e) { + logger.error("Write error. {}", cluster); + throw new ServiceException("Write error. {}", e); + } + }; + + exportClusters.forEach(clusterConsumer); + } + + private void writeToZip(String filePath, String content, ZipOutputStream zipOutputStream) + throws IOException { + final ZipEntry zipEntry = new ZipEntry(filePath); + try { + zipOutputStream.putNextEntry(zipEntry); + zipOutputStream.write(content.getBytes()); + zipOutputStream.closeEntry(); + } catch (IOException e) { + String errorMsg = "write content to zip error. file = " + filePath + ", content = " + content; + logger.error(errorMsg); + throw new IOException(errorMsg, e); + } + } + } diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsImportService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsImportService.java index f5aa4b155ff..f5607eea2a0 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsImportService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ConfigsImportService.java @@ -16,18 +16,40 @@ */ package com.ctrip.framework.apollo.portal.service; +import com.google.common.collect.Lists; +import com.google.gson.Gson; + +import com.ctrip.framework.apollo.common.constants.GsonType; +import com.ctrip.framework.apollo.common.dto.ClusterDTO; +import com.ctrip.framework.apollo.common.dto.ItemDTO; import com.ctrip.framework.apollo.common.dto.NamespaceDTO; +import com.ctrip.framework.apollo.common.entity.App; +import com.ctrip.framework.apollo.common.entity.AppNamespace; import com.ctrip.framework.apollo.common.exception.ServiceException; -import com.ctrip.framework.apollo.portal.component.PermissionValidator; -import com.ctrip.framework.apollo.portal.entity.model.NamespaceTextModel; +import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.portal.environment.Env; +import com.ctrip.framework.apollo.portal.listener.AppNamespaceCreationEvent; +import com.ctrip.framework.apollo.portal.spi.UserInfoHolder; import com.ctrip.framework.apollo.portal.util.ConfigFileUtils; import com.ctrip.framework.apollo.portal.util.ConfigToFileUtils; -import java.io.IOException; -import java.io.InputStream; -import java.security.AccessControlException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.rmi.ServerException; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; /** * @author wxq @@ -35,103 +57,419 @@ @Service public class ConfigsImportService { - private final ItemService itemService; + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigsImportService.class); + + private Gson gson = new Gson(); - private final NamespaceService namespaceService; + private final ItemService itemService; + private final AppService appService; + private final ClusterService clusterService; + private final NamespaceService namespaceService; + private final AppNamespaceService appNamespaceService; + private final ApplicationEventPublisher publisher; + private final UserInfoHolder userInfoHolder; + private final RoleInitializationService roleInitializationService; + private String currentUser; - private final PermissionValidator permissionValidator; public ConfigsImportService( final ItemService itemService, + final AppService appService, + final ClusterService clusterService, final @Lazy NamespaceService namespaceService, - PermissionValidator permissionValidator) { + final AppNamespaceService appNamespaceService, + final ApplicationEventPublisher publisher, + final UserInfoHolder userInfoHolder, + final RoleInitializationService roleInitializationService) { this.itemService = itemService; + this.appService = appService; + this.clusterService = clusterService; this.namespaceService = namespaceService; - this.permissionValidator = permissionValidator; + this.appNamespaceService = appNamespaceService; + this.publisher = publisher; + this.userInfoHolder = userInfoHolder; + this.roleInitializationService = roleInitializationService; } /** - * move from {@link com.ctrip.framework.apollo.portal.controller.ConfigsImportController} + * force import, new items will overwrite existed items. */ - private void importConfig( - final String appId, - final String env, - final String clusterName, - final String namespaceName, - final long namespaceId, - final String format, - final String configText - ) { - final NamespaceTextModel model = new NamespaceTextModel(); - - model.setAppId(appId); - model.setEnv(env); - model.setClusterName(clusterName); - model.setNamespaceName(namespaceName); - model.setNamespaceId(namespaceId); - model.setFormat(format); - model.setConfigText(configText); - - itemService.updateConfigItemByText(model); + public void forceImportNamespaceFromFile(final Env env, final String standardFilename, + final InputStream zipInputStream) { + String configText; + try (InputStream in = zipInputStream) { + configText = ConfigToFileUtils.fileToString(in); + } catch (IOException e) { + throw new ServiceException("Read config file errors:{}", e); + } + + this.importNamespaceFromText(env, standardFilename, configText, false); } /** - * import one config from file + * import all data include app、appnamespace、cluster、namespace、item */ - private void importOneConfigFromFile( - final String appId, - final String env, - final String clusterName, - final String namespaceName, - final String configText, - final String format - ) { - final NamespaceDTO namespaceDTO = namespaceService - .loadNamespaceBaseInfo(appId, Env.valueOf(env), clusterName, namespaceName); - this.importConfig(appId, env, clusterName, namespaceName, namespaceDTO.getId(), format, configText); + public void importDataFromZipFile(List importEnvs, ZipInputStream dataZip, boolean ignoreConflictNamespace) + throws IOException { + List toImportApps = Lists.newArrayList(); + List toImportAppNSs = Lists.newArrayList(); + List toImportClusters = Lists.newArrayList(); + List toImportNSs = Lists.newArrayList(); + + ZipEntry entry; + while ((entry = dataZip.getNextEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + String filePath = entry.getName(); + String content = readContent(dataZip); + + String[] info = filePath.split("/"); + + String fileName; + if (info.length == 1) { + //app namespace metadata file. path format : ${namespaceName}.appnamespace.metadata + fileName = info[0]; + if (fileName.endsWith(ConfigFileUtils.APP_NAMESPACE_METADATA_FILE_SUFFIX)) { + toImportAppNSs.add(content); + } + } else if (info.length == 3) { + fileName = info[2]; + if (fileName.equals(ConfigFileUtils.APP_METADATA_FILENAME)) { + //app metadata file. path format : apollo/${appId}/app.metadata + toImportApps.add(content); + } + } else { + String env = info[2]; + fileName = info[3]; + if (fileName.endsWith(ConfigFileUtils.CLUSTER_METADATA_FILE_SUFFIX)) { + //cluster metadata file. path format : apollo/${appId}/${env}/${clusterName}.cluster.metadata + toImportClusters.add(content); + } else { + //namespace file.path format : apollo/${appId}/${env}/${appId}+${cluster}+${namespaceName} + //only import for selected envs. + for (Env importEnv : importEnvs) { + if (Objects.equals(importEnv.getName(), env)) { + toImportNSs.add(new NamespaceImportData(Env.valueOf(env), fileName, content, ignoreConflictNamespace)); + } + } + } + } + } + + try { + LOGGER.info("Import data. app = {}, appns = {}, cluster = {}, namespace = {}", toImportApps.size(), + toImportAppNSs.size(), + toImportClusters.size(), toImportNSs.size()); + + doImport(importEnvs, toImportApps, toImportAppNSs, toImportClusters, toImportNSs); + + } catch (Exception e) { + LOGGER.error("import config error.", e); + throw new ServerException("import config error.", e); + } + } + + private void doImport(List importEnvs, List toImportApps, List toImportAppNSs, + List toImportClusters, List toImportNSs) + throws InterruptedException { + currentUser = userInfoHolder.getUser().getUserId(); + + LOGGER.info("Start to import app. size = {}", toImportApps.size()); + + long startTime = System.currentTimeMillis(); + CountDownLatch appLatch = new CountDownLatch(toImportApps.size()); + toImportApps.parallelStream().forEach(app -> { + try { + importApp(app, importEnvs); + } catch (Exception e) { + LOGGER.error("import app error. app = {}", app, e); + } finally { + appLatch.countDown(); + } + }); + appLatch.await(); + + LOGGER.info("Finish to import app. duration = {}", System.currentTimeMillis() - startTime); + LOGGER.info("Start to import appnamespace. size = {}", toImportAppNSs.size()); + + startTime = System.currentTimeMillis(); + CountDownLatch appNSLatch = new CountDownLatch(toImportAppNSs.size()); + toImportAppNSs.parallelStream().forEach(appNS -> { + try { + importAppNamespace(appNS); + } catch (Exception e) { + LOGGER.error("import appnamespace error. appnamespace = {}", appNS, e); + } finally { + appNSLatch.countDown(); + } + }); + appNSLatch.await(); + + LOGGER.info("Finish to import appnamespace. duration = {}", System.currentTimeMillis() - startTime); + LOGGER.info("Start to import cluster. size = {}", toImportClusters.size()); + + startTime = System.currentTimeMillis(); + CountDownLatch clusterLatch = new CountDownLatch(toImportClusters.size()); + toImportClusters.parallelStream().forEach(cluster -> { + try { + importCluster(cluster, importEnvs); + } catch (Exception e) { + LOGGER.error("import cluster error. cluster = {}", cluster, e); + } finally { + clusterLatch.countDown(); + } + }); + clusterLatch.await(); + + LOGGER.info("Finish to import cluster. duration = {}", System.currentTimeMillis() - startTime); + LOGGER.info("Start to import namespace. size = {}", toImportNSs.size()); + + startTime = System.currentTimeMillis(); + CountDownLatch nsLatch = new CountDownLatch(toImportNSs.size()); + toImportNSs.parallelStream().forEach(namespace -> { + try { + importNamespaceFromText(namespace.getEnv(), namespace.getFileName(), namespace.getContent(), + namespace.isIgnoreConflictNamespace()); + } catch (Exception e) { + LOGGER.error("import namespace error. namespace = {}", namespace, e); + } finally { + nsLatch.countDown(); + } + }); + nsLatch.await(); + + LOGGER.info("Finish to import namespace. duration = {}", System.currentTimeMillis() - startTime); + } + + private void importApp(String appInfo, List importEnvs) { + App toImportApp = gson.fromJson(appInfo, App.class); + String appId = toImportApp.getAppId(); + + //imported app set owner to current user. + toImportApp.setOwnerName(currentUser); + toImportApp.setDataChangeCreatedBy(currentUser); + toImportApp.setDataChangeLastModifiedBy(currentUser); + toImportApp.setDataChangeCreatedTime(new Date()); + toImportApp.setDataChangeLastModifiedTime(new Date()); + + App managedApp = appService.load(appId); + if (managedApp == null) { + appService.importAppInLocal(toImportApp); + } + + importEnvs.parallelStream().forEach(env -> { + try { + appService.load(env, appId); + } catch (Exception e) { + //not existed + appService.createAppInRemote(env, toImportApp); + } + }); + } + + private void importAppNamespace(String appNamespace) { + AppNamespace toImportPubAppNS = gson.fromJson(appNamespace, AppNamespace.class); + + String appId = toImportPubAppNS.getAppId(); + String namespaceName = toImportPubAppNS.getName(); + boolean isPublic = toImportPubAppNS.isPublic(); + + AppNamespace + managedAppNamespace = + isPublic ? appNamespaceService.findPublicAppNamespace(namespaceName) + : appNamespaceService.findByAppIdAndName(appId, namespaceName); + + if (managedAppNamespace == null) { + managedAppNamespace = new AppNamespace(); + managedAppNamespace.setAppId(toImportPubAppNS.getAppId()); + managedAppNamespace.setPublic(isPublic); + managedAppNamespace.setFormat(toImportPubAppNS.getFormat()); + managedAppNamespace.setComment(toImportPubAppNS.getComment()); + managedAppNamespace.setDataChangeCreatedBy(currentUser); + managedAppNamespace.setDataChangeLastModifiedBy(currentUser); + managedAppNamespace.setName(namespaceName); + + AppNamespace createdAppNamespace = appNamespaceService.importAppNamespaceInLocal(managedAppNamespace); + + //application namespace will be auto created when creating app + if (!ConfigConsts.NAMESPACE_APPLICATION.equals(namespaceName)) { + publisher.publishEvent(new AppNamespaceCreationEvent(createdAppNamespace)); + } + } + } + + private void importCluster(String clusterInfo, List importEnvs) { + ClusterDTO toImportCluster = gson.fromJson(clusterInfo, ClusterDTO.class); + + toImportCluster.setDataChangeCreatedBy(currentUser); + toImportCluster.setDataChangeLastModifiedBy(currentUser); + toImportCluster.setDataChangeCreatedTime(new Date()); + toImportCluster.setDataChangeLastModifiedTime(new Date()); + + String appId = toImportCluster.getAppId(); + String clusterName = toImportCluster.getName(); + + importEnvs.parallelStream().forEach(env -> { + try { + clusterService.loadCluster(appId, env, clusterName); + } catch (Exception e) { + //not existed + clusterService.createCluster(env, toImportCluster); + } + }); } /** - * import a config file. - * the name of config file must be special like - * appId+cluster+namespace.format - * Example: + * import a config file. the name of config file must be special like appId+cluster+namespace.format Example: *

    *   123456+default+application.properties (appId is 123456, cluster is default, namespace is application, format is properties)
    *   654321+north+password.yml (appId is 654321, cluster is north, namespace is password, format is yml)
    * 
* so we can get the information of appId, cluster, namespace, format from the file name. - * @param env environment + * + * @param env environment * @param standardFilename appId+cluster+namespace.format - * @param configText config content + * @param configText config content */ - private void importOneConfigFromText( - final String env, - final String standardFilename, - final String configText - ) { + private void importNamespaceFromText(final Env env, final String standardFilename, final String configText, + boolean ignoreConflictNamespace) { final String appId = ConfigFileUtils.getAppId(standardFilename); final String clusterName = ConfigFileUtils.getClusterName(standardFilename); final String namespace = ConfigFileUtils.getNamespace(standardFilename); final String format = ConfigFileUtils.getFormat(standardFilename); - this.importOneConfigFromFile(appId, env, clusterName, namespace, configText, format); + + this.importNamespace(appId, env, clusterName, namespace, configText, format, ignoreConflictNamespace); } - /** - * @see ConfigsImportService#importOneConfigFromText(java.lang.String, java.lang.String, java.lang.String) - * @throws AccessControlException if has no modify namespace permission - */ - public void importOneConfigFromFile( - final String env, - final String standardFilename, - final InputStream inputStream - ) { - final String configText; - try(InputStream in = inputStream) { - configText = ConfigToFileUtils.fileToString(in); + private void importNamespace(final String appId, final Env env, + final String clusterName, final String namespaceName, + final String configText, final String format, + boolean ignoreConflictNamespace) { + NamespaceDTO namespaceDTO; + try { + namespaceDTO = namespaceService.loadNamespaceBaseInfo(appId, env, clusterName, namespaceName); + } catch (Exception e) { + //not existed + namespaceDTO = null; + } + + if (namespaceDTO == null) { + namespaceDTO = new NamespaceDTO(); + namespaceDTO.setAppId(appId); + namespaceDTO.setClusterName(clusterName); + namespaceDTO.setNamespaceName(namespaceName); + namespaceDTO.setDataChangeCreatedBy(currentUser); + namespaceDTO.setDataChangeLastModifiedBy(currentUser); + namespaceDTO = namespaceService.createNamespace(env, namespaceDTO); + + roleInitializationService.initNamespaceRoles(appId, namespaceName, currentUser); + roleInitializationService.initNamespaceEnvRoles(appId, namespaceName, currentUser); + } + + List itemDTOS = itemService.findItems(appId, env, clusterName, namespaceName); + // skip import if target namespace has existed items + if (!CollectionUtils.isEmpty(itemDTOS) && ignoreConflictNamespace) { + return; + } + + importItems(appId, env, clusterName, namespaceName, configText, namespaceDTO); + } + + private void importItems(String appId, Env env, String clusterName, String namespaceName, String configText, + NamespaceDTO namespaceDTO) { + List toImportItems = gson.fromJson(configText, GsonType.ITEM_DTOS); + + toImportItems.parallelStream().forEach(newItem -> { + String key = newItem.getKey(); + newItem.setNamespaceId(namespaceDTO.getId()); + newItem.setDataChangeCreatedBy(currentUser); + newItem.setDataChangeLastModifiedBy(currentUser); + newItem.setDataChangeCreatedTime(new Date()); + newItem.setDataChangeLastModifiedTime(new Date()); + + try { + ItemDTO oldItem = itemService.loadItem(env, appId, clusterName, namespaceName, key); + newItem.setId(oldItem.getId()); + //existed + itemService.updateItem(appId, env, clusterName, namespaceName, newItem); + } catch (Exception e) { + //not existed + itemService.createItem(appId, env, clusterName, namespaceName, newItem); + } + }); + } + + + private String readContent(ZipInputStream zipInputStream) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buffer = new byte[1024]; + int offset; + while ((offset = zipInputStream.read(buffer)) != -1) { + out.write(buffer, 0, offset); + } + return out.toString("UTF-8"); } catch (IOException e) { - throw new ServiceException("Read config file errors:{}", e); + LOGGER.error("Read file content from zip error.", e); + return null; + } + } + + static class NamespaceImportData { + + private Env env; + private String fileName; + private String content; + private boolean ignoreConflictNamespace; + + public NamespaceImportData(Env env, String fileName, String content, boolean ignoreConflictNamespace) { + this.env = env; + this.fileName = fileName; + this.content = content; + this.ignoreConflictNamespace = ignoreConflictNamespace; + } + + public Env getEnv() { + return env; + } + + public void setEnv(Env env) { + this.env = env; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public boolean isIgnoreConflictNamespace() { + return ignoreConflictNamespace; + } + + public void setIgnoreConflictNamespace(boolean ignoreConflictNamespace) { + this.ignoreConflictNamespace = ignoreConflictNamespace; + } + + @Override + public String toString() { + return "NamespaceImportData{" + + "env=" + env + + ", fileName='" + fileName + '\'' + + ", content='" + content + '\'' + + ", ignoreConflictNamespace=" + ignoreConflictNamespace + + '}'; } - this.importOneConfigFromText(env, standardFilename, configText); } } diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/InstanceService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/InstanceService.java index d13625b62d9..681d2ac0d1c 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/InstanceService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/InstanceService.java @@ -44,7 +44,7 @@ public PageDTO getByNamespace(Env env, String appId, String cluster return instanceAPI.getByNamespace(appId, env, clusterName, namespaceName, instanceAppId, page, size); } - public int getInstanceCountByNamepsace(String appId, Env env, String clusterName, String namespaceName){ + public int getInstanceCountByNamespace(String appId, Env env, String clusterName, String namespaceName){ return instanceAPI.getInstanceCountByNamespace(appId, env, clusterName, namespaceName); } diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java index 72011b195d5..6d02a9e1a0a 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java @@ -107,7 +107,12 @@ public void updateConfigItemByText(NamespaceTextModel model) { return; } - changeSets.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId()); + String operator = model.getOperator(); + if (StringUtils.isBlank(operator)) { + operator = userInfoHolder.getUser().getUserId(); + } + changeSets.setDataChangeLastModifiedBy(operator); + updateItems(appId, env, clusterName, namespaceName, changeSets); Tracer.logEvent(TracerEventType.MODIFY_NAMESPACE_BY_TEXT, diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java index eb227651d3e..813fda892a8 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java @@ -99,7 +99,10 @@ public NamespaceDTO createNamespace(Env env, NamespaceDTO namespace) { if (StringUtils.isEmpty(namespace.getDataChangeCreatedBy())) { namespace.setDataChangeCreatedBy(userInfoHolder.getUser().getUserId()); } - namespace.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId()); + + if (StringUtils.isEmpty(namespace.getDataChangeLastModifiedBy())) { + namespace.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId()); + } NamespaceDTO createdNamespace = namespaceAPI.createNamespace(env, namespace); Tracer.logEvent(TracerEventType.CREATE_NAMESPACE, @@ -198,7 +201,7 @@ public NamespaceBO loadNamespaceBO(String appId, Env env, String clusterName, public boolean namespaceHasInstances(String appId, Env env, String clusterName, String namespaceName) { - return instanceService.getInstanceCountByNamepsace(appId, env, clusterName, namespaceName) > 0; + return instanceService.getInstanceCountByNamespace(appId, env, clusterName, namespaceName) > 0; } public boolean publicAppNamespaceHasAssociatedNamespace(String publicNamespaceName, Env env) { diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtils.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtils.java index 73ba87bee82..53ad52faef8 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtils.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtils.java @@ -16,14 +16,20 @@ */ package com.ctrip.framework.apollo.portal.util; +import com.ctrip.framework.apollo.common.dto.ClusterDTO; +import com.ctrip.framework.apollo.common.entity.App; +import com.ctrip.framework.apollo.common.entity.AppNamespace; import com.ctrip.framework.apollo.common.exception.BadRequestException; import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; import com.ctrip.framework.apollo.core.utils.StringUtils; import com.ctrip.framework.apollo.portal.controller.ConfigsImportController; import com.ctrip.framework.apollo.portal.environment.Env; + import com.google.common.base.Splitter; + import java.io.File; import java.util.List; + import org.springframework.web.multipart.MultipartFile; /** @@ -32,6 +38,10 @@ */ public class ConfigFileUtils { + public static final String APP_METADATA_FILENAME = "app.metadata"; + public static final String CLUSTER_METADATA_FILE_SUFFIX = ".cluster.metadata"; + public static final String APP_NAMESPACE_METADATA_FILE_SUFFIX = ".appnamespace.metadata"; + public static void check(MultipartFile file) { checkEmpty(file); final String originalFilename = file.getOriginalFilename(); @@ -168,7 +178,7 @@ public static String toFilename( * file path = ownerName/appId/env/configFilename * @return file path in compressed file */ - public static String toFilePath( + public static String genNamespacePath( final String ownerName, final String appId, final Env env, @@ -176,4 +186,27 @@ public static String toFilePath( ) { return String.join(File.separator, ownerName, appId, env.getName(), configFilename); } + + /** + * path = ownerName/appId/app.metadata + */ + public static String genAppInfoPath(App app) { + return String.join(File.separator, app.getOwnerName(), app.getAppId(), APP_METADATA_FILENAME); + } + + /** + * path = {appNamespace}.appnamespace.metadata + */ + public static String genAppNamespaceInfoPath(AppNamespace appNamespace) { + return String.join(File.separator, + appNamespace.getAppId() + "+" + appNamespace.getName() + APP_NAMESPACE_METADATA_FILE_SUFFIX); + } + + /** + * path = ownerName/appId/env/${clusterName}.metadata + */ + public static String genClusterInfoPath(App app, Env env, ClusterDTO cluster) { + return String.join(File.separator, app.getOwnerName(), app.getAppId(), env.getName(), + cluster.getName() + CLUSTER_METADATA_FILE_SUFFIX); + } } diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/NamespaceBOUtils.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/NamespaceBOUtils.java index bbb9dad8a19..b9b14ce0cd4 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/NamespaceBOUtils.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/util/NamespaceBOUtils.java @@ -16,24 +16,33 @@ */ package com.ctrip.framework.apollo.portal.util; +import com.google.common.collect.Lists; +import com.google.gson.Gson; + +import com.ctrip.framework.apollo.common.dto.ItemDTO; import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; -import com.ctrip.framework.apollo.core.utils.PropertiesUtil; import com.ctrip.framework.apollo.portal.controller.ConfigsExportController; import com.ctrip.framework.apollo.portal.entity.bo.ItemBO; import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO; -import java.io.IOException; + +import org.springframework.util.CollectionUtils; + +import java.util.Collections; import java.util.List; -import java.util.Properties; +import java.util.stream.Collectors; /** * @author wxq */ public class NamespaceBOUtils { + private static final Gson GSON = new Gson(); + /** - * namespace must not be {@link ConfigFileFormat#Properties}. - * the content of namespace in item's value which item's key is {@link ConfigConsts#CONFIG_FILE_CONTENT_KEY}. + * namespace must not be {@link ConfigFileFormat#Properties}. the content of namespace in item's value which item's + * key is {@link ConfigConsts#CONFIG_FILE_CONTENT_KEY}. + * * @param namespaceBO namespace * @return content of non-properties's namespace */ @@ -43,17 +52,18 @@ static String convertNonProperties2configFileContent(NamespaceBO namespaceBO) { String key = itemBO.getItem().getKey(); // special namespace format(not properties) if (ConfigConsts.CONFIG_FILE_CONTENT_KEY.equals(key)) { - return itemBO.getItem().getValue(); + ItemDTO dto = itemBO.getItem(); + dto.setId(0); + dto.setNamespaceId(0); + return GSON.toJson(Lists.newArrayList(dto)); } } - // If there is no items? - // return empty string "" return ""; } /** - * copy from old {@link ConfigsExportController}. - * convert {@link NamespaceBO} to a file content. + * copy from old {@link ConfigsExportController}. convert {@link NamespaceBO} to a file content. + * * @return content of config file * @throws IllegalStateException if convert properties to string fail */ @@ -66,23 +76,18 @@ public static String convert2configFileContent(NamespaceBO namespaceBO) { // it must be a properties format namespace List itemBOS = namespaceBO.getItems(); - // save the kev value pair - Properties properties = new Properties(); - for (ItemBO itemBO : itemBOS) { - String key = itemBO.getItem().getKey(); - String value = itemBO.getItem().getValue(); - // ignore comment, so the comment will lack - properties.put(key, value); + if (CollectionUtils.isEmpty(itemBOS)) { + return GSON.toJson(Collections.emptyList()); } - // use a special method convert properties to string - final String configFileContent; - try { - configFileContent = PropertiesUtil.toString(properties); - } catch (IOException e) { - throw new IllegalStateException("convert properties to string fail.", e); - } - return configFileContent; + List itemDTOS = itemBOS.stream().map(itemBO -> { + ItemDTO dto = itemBO.getItem(); + dto.setId(0); + dto.setNamespaceId(0); + return dto; + }).collect(Collectors.toList()); + + return GSON.toJson(itemDTOS); } } diff --git a/apollo-portal/src/main/resources/application.yml b/apollo-portal/src/main/resources/application.yml index 34b3fffedda..8905d89f729 100644 --- a/apollo-portal/src/main/resources/application.yml +++ b/apollo-portal/src/main/resources/application.yml @@ -27,6 +27,10 @@ spring: store-type: jdbc jdbc: initialize-schema: never + servlet: + multipart: + max-file-size: 100MB # import data configs + max-request-size: 100MB server: port: 8070 compression: diff --git a/apollo-portal/src/main/resources/static/config_export.html b/apollo-portal/src/main/resources/static/config_export.html index 27e22b16928..1e35c5e79d2 100644 --- a/apollo-portal/src/main/resources/static/config_export.html +++ b/apollo-portal/src/main/resources/static/config_export.html @@ -30,76 +30,139 @@ - -
-
-
-
-
- {{'ConfigExport.Title' | translate }} - - {{'ConfigExport.TitleTips' | translate}} - -
-
- - - + +
+
+
+
+ {{'ConfigExport.Title' | translate }} + {{'ConfigExport.TitleTips' | translate }} +
+
+
+ +
+ + + + + + + +
-
-
+
+ +
+ {{'ConfigExport.Export' | translate }} +

({{'ConfigExport.ExportTips' | translate }})

+
+
+ + +
+ +
+
+ +
+ + + + + + + +
+
+
+
+ +
+
+ + {{'ConfigExport.IgnoreExistedNamespace' | translate }} +
+
+ + {{'ConfigExport.OverwriteExistedNamespace' | translate }} +
+
+
+
+ +
+ +
+
+
+ +
+ {{'ConfigExport.Import' | translate }} +
+

({{'ConfigExport.ImportTips' | translate }})

+
+
+
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - \ No newline at end of file + diff --git a/apollo-portal/src/main/resources/static/i18n/en.json b/apollo-portal/src/main/resources/static/i18n/en.json index 08dafbbf805..bba41ee7483 100644 --- a/apollo-portal/src/main/resources/static/i18n/en.json +++ b/apollo-portal/src/main/resources/static/i18n/en.json @@ -11,7 +11,7 @@ "Common.Nav.SystemConfig": "System Configuration", "Common.Nav.DeleteApp-Cluster-Namespace": "Delete Apps, Clusters, AppNamespace", "Common.Nav.SystemInfo": "System Information", - "Common.Nav.ConfigExport": "Config Export", + "Common.Nav.ConfigExport": "Config Export / Import", "Common.Nav.Logout": "Logout", "Common.Department": "Department", "Common.Cluster": "Cluster", @@ -684,8 +684,23 @@ "Config.Diff.DiffCluster": "Clusters to be compared", "Config.Diff.HasDiffComment": "Whether to compare comments or not", "Config.Diff.PleaseChooseTwoCluster": "Please select at least two clusters", - "ConfigExport.Title": "Config Export", - "ConfigExport.TitleTips" : "Super administrators will download the configuration of all projects, normal users will only download the configuration of their own projects", + "ConfigExport.Title": "Config Export/Import", + "ConfigExport.TitleTips" : "(The data (application, cluster and namespace) of one cluster can be migrated to another cluster by exporting and importing the configuration)", + "ConfigExport.SelectExportEnv" : "Select the environment to export", + "ConfigExport.SelectImportEnv" : "Select the environment to import", + "ConfigExport.ExportTips" : "In case of large amount of data, the export speed is slow. Please wait patiently", + "ConfigExport.ImportConflictLabel" : "How to deal with existing namespaces when importing", + "ConfigExport.IgnoreExistedNamespace" : "Ignore existing namespaces", + "ConfigExport.OverwriteExistedNamespace" : "Overwrite existing namespace", + "ConfigExport.UploadFile" : "Upload the exported zip file", + "ConfigExport.UploadFileTip" : "Please upload the exported compressed file", + "ConfigExport.ImportSuccess" : "Import success", + "ConfigExport.ImportingTip" : "Importing, please wait patiently. After importing, please check whether the namespace configuration is correct. If it is correct, publish the namespace to take effect", + "ConfigExport.ImportFailed" : "Import failed", + "ConfigExport.ExportSuccess" : "Exporting data. The data volume will cause slow speed. Please wait patiently", + "ConfigExport.ImportTips" : "After the import is completed, please check whether the namespace configuration is correct. After the check is correct, it needs to be published to take effect", + "ConfigExport.Export" : "Export", + "ConfigExport.Import" : "Import", "ConfigExport.Download": "Download", "App.CreateProject": "Create Project", "App.AppIdTips": "(Application's unique identifiers)", @@ -764,4 +779,4 @@ "Rollback.RollbackFailed": "Failed to Rollback", "Revoke.RevokeFailed": "Failed to Revoke", "Revoke.RevokeSuccessfully": "Revoke Successfully" -} \ No newline at end of file +} diff --git a/apollo-portal/src/main/resources/static/i18n/zh-CN.json b/apollo-portal/src/main/resources/static/i18n/zh-CN.json index a284b6d036a..c486a94f62e 100644 --- a/apollo-portal/src/main/resources/static/i18n/zh-CN.json +++ b/apollo-portal/src/main/resources/static/i18n/zh-CN.json @@ -11,7 +11,7 @@ "Common.Nav.SystemConfig": "系统参数", "Common.Nav.DeleteApp-Cluster-Namespace": "删除应用、集群、AppNamespace", "Common.Nav.SystemInfo": "系统信息", - "Common.Nav.ConfigExport": "配置导出", + "Common.Nav.ConfigExport": "配置导出导入", "Common.Nav.Logout": "退出", "Common.Department": "部门", "Common.Cluster": "集群", @@ -684,8 +684,23 @@ "Config.Diff.DiffCluster": "要比较的集群", "Config.Diff.HasDiffComment": "是否比较注释", "Config.Diff.PleaseChooseTwoCluster": "请至少选择两个集群", - "ConfigExport.Title": "配置导出", - "ConfigExport.TitleTips" : "超级管理员会下载所有应用的配置,普通用户只会下载自己应用的配置", + "ConfigExport.Title": "配置导出导入", + "ConfigExport.TitleTips" : "(通过导出导入配置,把一个集群的数据(应用、集群、Namespace)迁移到另外一个集群)", + "ConfigExport.SelectExportEnv" : "选择导出的环境", + "ConfigExport.SelectImportEnv" : "选择导入的环境", + "ConfigExport.ImportConflictLabel" : "导入时该如何处理已存在的 Namespace", + "ConfigExport.ExportSuccess" : "正在导出数据,数据量大会导致速度慢,请耐心等待", + "ConfigExport.ExportTips" : "数据量大的情况下,导出速度较慢请耐心等待", + "ConfigExport.IgnoreExistedNamespace" : "跳过已存在的 Namespace", + "ConfigExport.OverwriteExistedNamespace" : "覆盖已存在的 Namespace", + "ConfigExport.UploadFile" : "上传导出的压缩文件", + "ConfigExport.UploadFileTip" : "请上传导出的压缩文件", + "ConfigExport.ImportSuccess" : "导入成功", + "ConfigExport.ImportingTip" : "正在导入,请耐心等待。导入完成后,请检查 namespace 的配置是否正确,如果无误再发布 namespace", + "ConfigExport.ImportFailed" : "导入失败", + "ConfigExport.Export" : "导出", + "ConfigExport.Import" : "导入", + "ConfigExport.ImportTips" : "导入完成之后,请检查 namespace 的配置是否正确,检查无误后需要发布才能生效", "ConfigExport.Download": "下载", "App.CreateProject": "创建应用", "App.AppIdTips": "(应用唯一标识)", diff --git a/apollo-portal/src/main/resources/static/scripts/controller/ConfigExportController.js b/apollo-portal/src/main/resources/static/scripts/controller/ConfigExportController.js new file mode 100644 index 00000000000..9e044b709b6 --- /dev/null +++ b/apollo-portal/src/main/resources/static/scripts/controller/ConfigExportController.js @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +config_export_module.controller('ConfigExportController', + ['$scope', '$location', '$window', '$http', '$translate', 'toastr', 'AppService', + 'EnvService', + 'ExportService', + 'AppUtil', + function ($scope, $location, $window, $http, $translate, toastr, AppService, + EnvService, + ExportService, + AppUtil) { + + $scope.conflictAction = 'ignore'; + + EnvService.find_all_envs().then(function (result) { + $scope.exportEnvs = []; + $scope.importEnvs = []; + result.forEach(function (env) { + $scope.exportEnvs.push({name: env, checked: false}); + $scope.importEnvs.push({name: env, checked: false}); + + }); + $(".apollo-container").removeClass("hidden"); + }, function (result) { + toastr.error(AppUtil.errorMsg(result), + $translate.instant('Cluster.LoadingEnvironmentError')); + }); + + $scope.switchChecked = function (env, $event) { + env.checked = !env.checked; + $event.stopPropagation(); + }; + + $scope.toggleEnvCheckedStatus = function (env) { + env.checked = !env.checked; + }; + + $scope.export = function () { + var selectedEnvs = [] + $scope.exportEnvs.forEach(function (env) { + if (env.checked) { + selectedEnvs.push(env.name); + } + }); + + if (selectedEnvs.length === 0) { + toastr.warning($translate.instant('Cluster.PleaseChooseEnvironment')); + return + } + + var selectedEnvStr = selectedEnvs.join(","); + $window.location.href = '/configs/export?envs=' + selectedEnvStr; + + toastr.success($translate.instant('ConfigExport.ExportSuccess')); + }; + + $scope.import = function () { + var selectedEnvs = [] + $scope.importEnvs.forEach(function (env) { + if (env.checked) { + selectedEnvs.push(env.name); + } + }); + + if (selectedEnvs.length === 0) { + toastr.warning($translate.instant('Cluster.PleaseChooseEnvironment')); + return + } + + var selectedEnvStr = selectedEnvs.join(","); + var file = document.getElementById("fileUpload").files[0]; + + if (file == null) { + toastr.warning($translate.instant('ConfigExport.UploadFileTip')) + return + } + + var form = new FormData(); + form.append('file', file); + $http({ + method: 'POST', + url: '/configs/import?envs=' + selectedEnvStr + "&conflictAction=" + + $scope.conflictAction, + data: form, + headers: {'Content-Type': undefined}, + transformRequest: angular.identity + }).success(function (data) { + toastr.success(data, $translate.instant('ConfigExport.ImportSuccess')) + }).error(function (data) { + toastr.error(data, $translate.instant('ConfigExport.ImportFailed')) + }) + toastr.info($translate.instant('ConfigExport.ImportingTip')) + } + + }]); diff --git a/apollo-portal/src/main/resources/static/scripts/services/ExportService.js b/apollo-portal/src/main/resources/static/scripts/services/ExportService.js new file mode 100644 index 00000000000..929cdc76467 --- /dev/null +++ b/apollo-portal/src/main/resources/static/scripts/services/ExportService.js @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +appService.service('ExportService', ['$resource', '$q', function ($resource, $q) { + var resource = $resource('', {}, { + importConfig: { + method: 'POST', + url: '/import', + headers: {'Content-Type': undefined}, + } + }); + return { + importConfig: function (envs, file) { + var form = new FormData(); + form.append('file', file); + var d = $q.defer(); + resource.importConfig({ + data: form, + envs: envs, + }, + function (result) { + d.resolve(result); + }, function (result) { + d.reject(result); + }); + return d.promise; + } + } +}]); diff --git a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/ConfigsExportServiceTest.java b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/ConfigsExportServiceTest.java new file mode 100644 index 00000000000..47464516ccf --- /dev/null +++ b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/ConfigsExportServiceTest.java @@ -0,0 +1,258 @@ +/* + * Copyright 2021 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.portal.service; + +import com.ctrip.framework.apollo.common.dto.ClusterDTO; +import com.ctrip.framework.apollo.common.dto.ItemDTO; +import com.ctrip.framework.apollo.common.dto.NamespaceDTO; +import com.ctrip.framework.apollo.common.entity.App; +import com.ctrip.framework.apollo.common.entity.AppNamespace; +import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; +import com.ctrip.framework.apollo.portal.AbstractUnitTest; +import com.ctrip.framework.apollo.portal.component.PermissionValidator; +import com.ctrip.framework.apollo.portal.entity.bo.ItemBO; +import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO; +import com.ctrip.framework.apollo.portal.entity.bo.UserInfo; +import com.ctrip.framework.apollo.portal.environment.Env; +import com.ctrip.framework.apollo.portal.spi.UserInfoHolder; + +import org.assertj.core.util.Lists; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.springframework.context.ApplicationEventPublisher; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.util.List; +import java.util.zip.ZipInputStream; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author lepdou 2021-08-30 + */ +public class ConfigsExportServiceTest extends AbstractUnitTest { + + @Mock + private AppService appService; + @Mock + private ClusterService clusterService; + @Mock + private NamespaceService namespaceService; + @Mock + private PermissionValidator permissionValidator; + @Mock + private UserInfoHolder userInfoHolder; + @Mock + private AppNamespaceService appNamespaceService; + @InjectMocks + private ConfigsExportService configsExportService; + @Mock + private ItemService itemService; + @Mock + private ApplicationEventPublisher applicationEventPublisher; + @Mock + private RoleInitializationService roleInitializationService; + @InjectMocks + private ConfigsImportService configsImportService; + + @Test + public void testNamespaceExportImport() throws FileNotFoundException { + String filePath = "/tmp/apollo.zip"; + File file = new File(filePath); + if (file.exists()) { + file.delete(); + } + + //export config + UserInfo userInfo = genUser(); + when(userInfoHolder.getUser()).thenReturn(userInfo); + + Env env = Env.DEV; + String appId1 = "app1"; + String appId2 = "app2"; + + App app1 = genApp(appId1, appId1, "org1", "org2"); + App app2 = genApp(appId2, appId2, "org1", "org2"); + + List exportApps = Lists.newArrayList(app1, app2); + + String appNamespaceName1 = "ns1"; + String appNamespaceName2 = "ns2"; + AppNamespace app1Namespace1 = genAppNamespace(appId1, appNamespaceName1, false); + AppNamespace app1Namespace2 = genAppNamespace(appId1, appNamespaceName2, true); + AppNamespace app2Namespace1 = genAppNamespace(appId2, appNamespaceName1, false); + + List appNamespaces = Lists.newArrayList(app1Namespace1, app1Namespace2, app2Namespace1); + + String clusterName1 = "c1"; + String clusterName2 = "c2"; + ClusterDTO app1Cluster1 = genCluster(clusterName1, appId1); + ClusterDTO app1Cluster2 = genCluster(clusterName2, appId1); + + ClusterDTO app2Cluster1 = genCluster(clusterName1, appId2); + ClusterDTO app2Cluster2 = genCluster(clusterName2, appId2); + + List + app1Clusters = + Lists.newArrayList(app1Cluster1, app1Cluster2); + List + app2Clusters = + Lists.newArrayList(app2Cluster1, app2Cluster2); + + ItemBO item1 = genItem("k1", "v1"); + ItemBO item2 = genItem("k2", "v2"); + List items = Lists.newArrayList(item1, item2); + + String namespaceName1 = "namespace1"; + String namespaceName2 = "namespace2"; + NamespaceBO app1Cluster1Namespace1 = genNamespace(app1, app1Cluster1, items, namespaceName1); + NamespaceBO app1Cluster1Namespace2 = genNamespace(app1, app1Cluster1, items, namespaceName2); + List app1Cluster1Namespace = Lists.newArrayList(app1Cluster1Namespace1, app1Cluster1Namespace2); + NamespaceBO app1Cluster2Namespace1 = genNamespace(app1, app1Cluster2, items, namespaceName1); + List app1Cluster2Namespace = Lists.newArrayList(app1Cluster2Namespace1); + + NamespaceBO app2Cluster1Namespace1 = genNamespace(app2, app1Cluster1, items, namespaceName1); + List app2Cluster1Namespace = Lists.newArrayList(app2Cluster1Namespace1); + NamespaceBO app2Cluster2Namespace1 = genNamespace(app2, app1Cluster2, items, namespaceName1); + NamespaceBO app2Cluster2Namespace2 = genNamespace(app2, app1Cluster2, items, namespaceName2); + List app2Cluster2Namespace = Lists.newArrayList(app2Cluster2Namespace1, app2Cluster2Namespace2); + + when(appService.findAll()).thenReturn(exportApps); + when(appNamespaceService.findAll()).thenReturn(appNamespaces); + when(permissionValidator.isAppAdmin(any())).thenReturn(true); + when(clusterService.findClusters(env, appId1)).thenReturn(app1Clusters); + when(clusterService.findClusters(env, appId2)).thenReturn(app2Clusters); + when(namespaceService.findNamespaceBOs(appId1, Env.DEV, clusterName1)).thenReturn(app1Cluster1Namespace); + when(namespaceService.findNamespaceBOs(appId1, Env.DEV, clusterName2)).thenReturn(app1Cluster2Namespace); + when(namespaceService.findNamespaceBOs(appId2, Env.DEV, clusterName1)).thenReturn(app2Cluster1Namespace); + when(namespaceService.findNamespaceBOs(appId2, Env.DEV, clusterName2)).thenReturn(app2Cluster2Namespace); + + FileOutputStream fileOutputStream = new FileOutputStream("/tmp/apollo.zip"); + + configsExportService.exportData(fileOutputStream, Lists.newArrayList(Env.DEV)); + + //import config + when(appNamespaceService.findByAppIdAndName(any(), any())).thenReturn(null); + when(appNamespaceService.importAppNamespaceInLocal(any())).thenReturn(app1Namespace1); + when(appService.load(any())).thenReturn(null); + when(appService.load(any(), any())).thenThrow(new RuntimeException()); + + when(clusterService.loadCluster(any(), any(), any())).thenThrow(new RuntimeException()); + + when(namespaceService.loadNamespaceBaseInfo(any(), any(), any(), any())).thenThrow(new RuntimeException()); + when(namespaceService.createNamespace(any(), any())).thenReturn(genNamespaceDTO(1)); + + when(itemService.findItems(any(), any(), any(), any())).thenReturn(Lists.newArrayList()); + when(itemService.loadItem(any(), any(), any(), any(), anyString())).thenThrow(new RuntimeException()); + + FileInputStream fileInputStream = new FileInputStream("/tmp/apollo.zip"); + ZipInputStream zipInputStream = new ZipInputStream(fileInputStream); + + try { + configsImportService.importDataFromZipFile(Lists.newArrayList(Env.DEV), zipInputStream, false); + } catch (Exception e) { + e.printStackTrace(); + } + + verify(appNamespaceService, times(3)).importAppNamespaceInLocal(any()); + verify(applicationEventPublisher, times(3)).publishEvent(any()); + + verify(appService, times(2)).createAppInRemote(any(), any()); + + verify(clusterService, times(4)).createCluster(any(), any()); + + verify(namespaceService, times(6)).createNamespace(any(), any()); + verify(roleInitializationService,times(6)).initNamespaceRoles(any(), any(), anyString()); + verify(roleInitializationService,times(6)).initNamespaceEnvRoles(any(), any(), anyString()); + verify(itemService, times(12)).createItem(any(), any(), any(), any(), any()); + } + + private App genApp(String name, String appId, String orgId, String orgName) { + App app = new App(); + app.setAppId(appId); + app.setName(name); + app.setOrgName("apollo"); + app.setOrgId(orgId); + app.setOrgName(orgName); + return app; + } + + private ClusterDTO genCluster(String name, String appId) { + ClusterDTO clusterDTO = new ClusterDTO(); + clusterDTO.setAppId(appId); + clusterDTO.setName(name); + return clusterDTO; + } + + private AppNamespace genAppNamespace(String appId, String name, boolean isPublic) { + AppNamespace appNamespace = new AppNamespace(); + appNamespace.setAppId(appId); + appNamespace.setPublic(isPublic); + appNamespace.setName(name); + appNamespace.setFormat(ConfigFileFormat.Properties.getValue()); + return appNamespace; + } + + private NamespaceBO genNamespace(App app, ClusterDTO clusterDTO, List itemBOS, String namespaceName) { + NamespaceBO namespaceBO = new NamespaceBO(); + + NamespaceDTO baseInfo = new NamespaceDTO(); + baseInfo.setNamespaceName(namespaceName); + baseInfo.setAppId(app.getAppId()); + baseInfo.setClusterName(clusterDTO.getName()); + + namespaceBO.setBaseInfo(baseInfo); + namespaceBO.setFormat(ConfigFileFormat.Properties.getValue()); + namespaceBO.setItems(itemBOS); + + return namespaceBO; + } + + private ItemBO genItem(String key, String value) { + ItemBO itemBO = new ItemBO(); + + ItemDTO itemDTO = new ItemDTO(); + itemDTO.setKey(key); + itemDTO.setValue(value); + + itemBO.setItem(itemDTO); + + return itemBO; + } + + private NamespaceDTO genNamespaceDTO(long id) { + NamespaceDTO dto = new NamespaceDTO(); + dto.setId(id); + return dto; + } + + private UserInfo genUser() { + UserInfo userInfo = new UserInfo(); + userInfo.setUserId("apollo"); + userInfo.setName("apollo"); + userInfo.setEmail("apollo@apollo.com"); + return userInfo; + } +} diff --git a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/NamespaceServiceTest.java b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/NamespaceServiceTest.java index f0bbeeac136..f965a83fee3 100644 --- a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/NamespaceServiceTest.java +++ b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/service/NamespaceServiceTest.java @@ -148,7 +148,7 @@ public void testDeleteNamespaceHasInstance() { AppNamespace publicNamespace = createAppNamespace(testAppId, testNamespaceName, true); when(appNamespaceService.findByAppIdAndName(testAppId, testNamespaceName)).thenReturn(publicNamespace); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, testClusterName, testNamespaceName)) + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, testClusterName, testNamespaceName)) .thenReturn(10); namespaceService.deleteNamespace(testAppId, testEnv, testClusterName, testNamespaceName); @@ -162,10 +162,10 @@ public void testDeleteNamespaceBranchHasInstance() { NamespaceDTO branch = createNamespace(testAppId, branchName, testNamespaceName); when(appNamespaceService.findByAppIdAndName(testAppId, testNamespaceName)).thenReturn(publicNamespace); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, testClusterName, testNamespaceName)) + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, testClusterName, testNamespaceName)) .thenReturn(0); when(branchService.findBranchBaseInfo(testAppId, testEnv, testClusterName, testNamespaceName)).thenReturn(branch); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(10); + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(10); namespaceService.deleteNamespace(testAppId, testEnv, testClusterName, testNamespaceName); @@ -179,10 +179,10 @@ public void testDeleteNamespaceWithAssociatedNamespace() { NamespaceDTO branch = createNamespace(testAppId, branchName, testNamespaceName); when(appNamespaceService.findByAppIdAndName(testAppId, testNamespaceName)).thenReturn(publicNamespace); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, testClusterName, testNamespaceName)) + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, testClusterName, testNamespaceName)) .thenReturn(0); when(branchService.findBranchBaseInfo(testAppId, testEnv, testClusterName, testNamespaceName)).thenReturn(branch); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(0); + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(0); when(appNamespaceService.findPublicAppNamespace(testNamespaceName)).thenReturn(publicNamespace); when(namespaceAPI.countPublicAppNamespaceAssociatedNamespaces(testEnv, testNamespaceName)).thenReturn(10); @@ -199,10 +199,10 @@ public void testDeleteEmptyNamespace() { NamespaceDTO branch = createNamespace(testAppId, branchName, testNamespaceName); when(appNamespaceService.findByAppIdAndName(testAppId, testNamespaceName)).thenReturn(publicNamespace); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, testClusterName, testNamespaceName)) + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, testClusterName, testNamespaceName)) .thenReturn(0); when(branchService.findBranchBaseInfo(testAppId, testEnv, testClusterName, testNamespaceName)).thenReturn(branch); - when(instanceService.getInstanceCountByNamepsace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(0); + when(instanceService.getInstanceCountByNamespace(testAppId, testEnv, branchName, testNamespaceName)).thenReturn(0); when(appNamespaceService.findPublicAppNamespace(testNamespaceName)).thenReturn(publicNamespace); NamespaceDTO namespace = createNamespace(testAppId, testClusterName, testNamespaceName); diff --git a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtilsTest.java b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtilsTest.java index ff665e3747d..a2dbca52a24 100644 --- a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtilsTest.java +++ b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/util/ConfigFileUtilsTest.java @@ -97,4 +97,4 @@ public void toFilename() { assertEquals("666+none+cc.yml", ymlFilename0); } -} \ No newline at end of file +}