[feat] 新增下载同一集群环境下所有服务器当天的日志

用于解决问题查看日志

v2-多线程版本/changLog.md

+## v1.1
+
+### 1、新增下载同一集群环境下所有服务器当天的日志
+
+#### 通过redis消息服务每个服务器把各自的日志打包zip文件暂存redis
+
+### 2、配置文件 spring/applicationContext-redis.xml 添加：
+
+```xml
+<bean id="sysClusterLogFileProviderImpl" class="com.hand.hls.dp.service.impl.SysClusterLogFileProviderImpl"/>
+```
+

v2-多线程版本/java-code/dp/controllers/v2/SysDpExecuteHistoryV2Controller.java

@@ -4,6 +4,7 @@ import com.hand.hap.core.IRequest;
 import com.hand.hap.core.impl.RequestHelper;
 import com.hand.hap.system.controllers.BaseController;
 import com.hand.hap.system.dto.ResponseData;
+import com.hand.hls.dp.service.SysDpDownloadLogService;
 import com.hand.hls.dp.service.SysDpExecuteHistoryService;
 import com.hand.hls.dp.util.IPUtils;
 import leaf.bean.LeafRequestData;
@@ -13,6 +14,7 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
 /**
  * @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
@@ -24,6 +26,8 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
 
     @Autowired
     private SysDpExecuteHistoryService service;
+    @Autowired
+    private SysDpDownloadLogService downloadLogService;
 
     @PostMapping(value = "/sys/dp/v2/execute")
     public ResponseData query(@ModelAttribute(LEAF_PARAM_NAME) LeafRequestData requestData,
@@ -34,4 +38,14 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
         final String dpContext = (String) requestData.get("parameter");
         return service.execute(requestContext, ip, dpContext);
     }
+
+    @PostMapping(value = "/sys/dp/v2/collect/log")
+    public void collectLog(HttpServletRequest request, HttpServletResponse response) throws Exception{
+        IRequest requestContext = createRequestContext(request);
+        // 发布消息收集日志
+        downloadLogService.collectLog(requestContext, request, response);
+        Thread.sleep(6000);
+        // 下载日志
+        downloadLogService.downloadClusterLogZipFile(requestContext, request, response);
+    }
 }

v2-多线程版本/java-code/dp/service/SysDpDownloadLogService.java

@@ -12,4 +12,6 @@ import javax.servlet.http.HttpServletResponse;
  */
 public interface SysDpDownloadLogService {
     void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
+    void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
+    void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
 }

v2-多线程版本/java-code/dp/service/impl/SysClusterLogFileProviderImpl.java

+package com.hand.hls.dp.service.impl;
+
+import com.hand.hap.message.IMessageConsumer;
+import com.hand.hap.message.TopicMonitor;
+import com.hand.hls.dp.util.LogUtils;
+import org.apache.commons.codec.binary.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
+ * @date 2020/9/3 14:32
+ * @since 1.0
+ */
+@TopicMonitor(channel = SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD)
+public class SysClusterLogFileProviderImpl implements IMessageConsumer<String>, InitializingBean {
+    private Logger logger = LoggerFactory.getLogger(SysClusterLogFileProviderImpl.class);
+    public static final String LOGFILE_DOWNLOAD = "topic:logfile:download";
+    public static final String LOGFILE_KEY = "leaf:logs:zip";
+
+    @Value("${log-path}")
+    private String logPath;
+
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    private void collectLog() {
+        ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
+        CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
+        ZipOutputStream zos = new ZipOutputStream(cos);
+        // 当天日志名称
+        String currLogFileName = logPath + "leaf.log";
+
+        // 查找日志目录下所有文件
+        List<String> files = new ArrayList<>(31);
+        LogUtils.findFiles(new File(logPath), files);
+        files.forEach(file -> {
+            if (file.equals(currLogFileName)) {
+                // 创建ZipEntry
+                ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
+                // 放入文件，防止文件名相同的附件报错
+                try {
+                    zos.putNextEntry(entry);
+                } catch (Exception e) {
+                    // ignore
+                }
+                // 写入文件
+                LogUtils.readFileToZip(file, zos);
+            }
+        });
+        try {
+            zos.closeEntry();
+            zos.close();
+            if (logger.isDebugEnabled()) {
+                logger.debug("redis key: [{}]", LOGFILE_KEY);
+            }
+            redisTemplate.opsForList().rightPush(LOGFILE_KEY, Base64.encodeBase64String(byteOutPut.toByteArray()));
+            cos.close();
+        } catch (Exception e) {
+            logger.warn(e.getMessage());
+        }
+    }
+
+    @Override
+    public void onMessage(String message, String pattern) {
+        if (LOGFILE_DOWNLOAD.equals(pattern)) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("start do [{}]", message);
+            }
+            // 收集当天日志暂存redis
+            collectLog();
+        }
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {}
+}

v2-多线程版本/java-code/dp/service/impl/SysDpDownloadLogServiceImpl.java

@@ -2,10 +2,14 @@ package com.hand.hls.dp.service.impl;
 
 import cn.hutool.core.date.DateUtil;
 import com.hand.hap.core.IRequest;
+import com.hand.hap.message.IMessagePublisher;
 import com.hand.hls.dp.service.SysDpDownloadLogService;
 import com.hand.hls.dp.util.LogUtils;
-import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
 import javax.servlet.http.HttpServletRequest;
@@ -34,6 +38,85 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
     @Value("${log-path}")
     private String logPath;
 
+    @Autowired
+    private IMessagePublisher messagePublisher;
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    /**
+     * 通知集群服务器当天日志暂存到redis
+     * @param requestContext 请求上下文
+     * @param request http请求
+     * @param response http响应
+     * @throws Exception 异常
+     */
+    @Override
+    public void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
+        messagePublisher.publish(SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD, "download logfile");
+    }
+    /**
+     * 下载集群服务器上当天日志
+     * @param requestContext 请求上下文
+     * @param request http请求
+     * @param response http响应
+     * @throws Exception 异常
+     */
+    @Override
+    public void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
+        LocalDate tmp = LocalDate.now();
+        Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
+        String zipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + ".zip";
+        response.reset();
+        response.setHeader("Pragma", "No-cache");
+        response.setHeader("Cache-Control", "no-cache,must-revalidate");
+        response.setHeader("Expires", "0");
+        response.setHeader("Content-Disposition", "attachment;filename=" + new String(zipFileName.getBytes(), StandardCharsets.ISO_8859_1));
+        response.setContentType("application/x-msdownload;");
+        List<String> zipFileList = redisTemplate.opsForList().range(SysClusterLogFileProviderImpl.LOGFILE_KEY, 0 , -1);
+        // 清空
+        while (redisTemplate.opsForList().size(SysClusterLogFileProviderImpl.LOGFILE_KEY) > 0){
+            redisTemplate.opsForList().leftPop(SysClusterLogFileProviderImpl.LOGFILE_KEY);
+        }
+        int size = zipFileList.size();
+        if (size >= 1) {
+            ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
+            CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
+            ZipOutputStream zos = new ZipOutputStream(cos);
+            for (int i = 0; i < size; i++) {
+                String zipFile = zipFileList.get(i);
+                String singleZipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + "-" + i + ".zip";
+                // 创建ZipEntry
+                ZipEntry entry = new ZipEntry(singleZipFileName);
+                // 放入文件，防止文件名相同的附件报错
+                try {
+                    zos.putNextEntry(entry);
+                } catch (Exception e) {
+                    // ignore
+                }
+                // 写入文件
+                byte[] byteArray = Base64.decodeBase64(zipFile);
+                ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray);
+                int n = 0;
+                byte[] buffer = new byte[LogUtils.DEFAULT_LARGE_BUFFER_SIZE];
+                while (LogUtils.EOF != (n = byteArrayInputStream.read(buffer))) {
+                    zos.write(buffer, 0, n);
+                }
+            }
+            zos.closeEntry();
+            zos.close();
+            response.setHeader("Content-Length", String.valueOf(byteOutPut.toByteArray().length));
+            IOUtils.copy(new ByteArrayInputStream(byteOutPut.toByteArray()), response.getOutputStream());
+            cos.close();
+        }
+    }
+
+    /**
+     * 下载当前请求的服务器上7天日志
+     * @param requestContext 请求上下文
+     * @param request http请求
+     * @param response http响应
+     * @throws Exception 异常
+     */
     @Override
     public void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
         String zipFileName = DateUtil.format(new Date(),"yyyyMMddHHmmss") + "_logs.zip";
@@ -54,14 +137,14 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
         for (int i = 1; i <= length - 1;i++){
             LocalDate tmp = LocalDate.now().minusDays(i);
             Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
-            downloadFiles[i] = logPath + "leaf-" + DateUtil.format(new Date(),"yyyy-MM-dd") + ".log";
+            downloadFiles[i] = logPath + "leaf-" + DateUtil.format(date,"yyyy-MM-dd") + ".log";
         }
         // 查找日志目录下所有文件
-        List<String> files = new ArrayList<>(30);
+        List<String> files = new ArrayList<>(31);
         LogUtils.findFiles(new File(logPath),files);
         files.forEach(file -> {
-            for (int i = 0; i < length; i++) {
-                if (file.equals(downloadFiles[i])) {
+            for (String downloadFile : downloadFiles) {
+                if (file.equals(downloadFile)) {
                     // 创建ZipEntry
                     ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
                     // 放入文件，防止文件名相同的附件报错
@@ -71,7 +154,7 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
                         // ignore
                     }
                     // 写入文件
-                    LogUtils.readFileToZip(file,zos);
+                    LogUtils.readFileToZip(file, zos);
                 }
             }
         });

v2-多线程版本/lview-code/DP/DP001/sys_dp.lview

@@ -86,6 +86,31 @@
                 }, 200, 100);
             }
 
+            function download_cluster_log() {
+                $('dp001_download_cluster_log_btn').disable();
+                Leaf.showConfirm('${l:HLS.PROMPT}', '确认下载日志吗？', function okFun() {
+                    Leaf.Masker.mask(Ext.getBody(), '正在下载中...');
+                    let form = document.createElement("form");
+                    form.target = "_export_dp_log_window";
+                    form.method = "post";
+                    let url = '${/request/@context_path}/sys/dp/v2/collect/log';
+                    let token = $jq('meta[name=_csrf]').attr('content');
+                    form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
+                    let iframe = Ext.get('_export_dp_log_window') || new Ext.Template('<iframe id ="_export_dp_log_window" name="_export_dp_log_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
+                    document.body.appendChild(form);
+                    form.submit();
+                    Ext.fly(form).remove();
+                    setTimeout(function () {
+                        $('dp001_download_cluster_log_btn').enable();
+                        Leaf.Masker.unmask(Ext.getBody());
+                    }, 10000);
+
+                }, function cancel() {
+                    $('dp001_download_cluster_log_btn').enable();
+                    Leaf.Masker.unmask(Ext.getBody());
+                }, 200, 100);
+            }
+
             function download_log() {
                 $('dp001_download_log_btn').disable();
                 Leaf.showConfirm('${l:HLS.PROMPT}', '确认下载日志吗？', function okFun() {
@@ -115,7 +140,8 @@
             <a:screenTopToolbar>
                 <a:gridButton id="dp001_query_btn" click="dp001_query" text="查询"/>
                 <a:gridButton id="dp001_exec_btn" click="dp001_exec" text="执行"/>
-                <a:gridButton id="dp001_download_log_btn" click="download_log" text="日志下载"/>
+                <a:gridButton id="dp001_download_cluster_log_btn" click="download_cluster_log" text="日志下载(当天集群环境日志)"/>
+                <a:gridButton id="dp001_download_log_btn" click="download_log" text="日志下载(7天当前请求环境日志)"/>
             </a:screenTopToolbar>
             <a:fieldSet title="待执行">
                 <a:textArea id="context_tta" name="context" marginWidth="50" marginHeight="450"/>