Skip to content

D
dp
Commit eb7fefaa authored by Spencer Chang's avatar Spencer Chang
Browse files
Options

[feat] 新增下载同一集群环境下所有服务器当天的日志 

用于解决问题查看日志
parent 35f3c618
Hide whitespace changes
Inline Side-by-side
Showing with 331 additions and 104 deletions
v2-多线程版本/changLog.md 0 → 100644
View file @ eb7fefaa
## v1.1
### 1、新增下载同一集群环境下所有服务器当天的日志
#### 通过redis消息服务每个服务器把各自的日志打包zip文件暂存redis
### 2、配置文件 spring/applicationContext-redis.xml 添加：
```xml
<bean id="sysClusterLogFileProviderImpl" class="com.hand.hls.dp.service.impl.SysClusterLogFileProviderImpl"/>
```
v2-多线程版本/java-code/dp/controllers/v2/SysDpExecuteHistoryV2Controller.java
View file @ eb7fefaa
......@@ -4,6 +4,7 @@ import com.hand.hap.core.IRequest;
import com.hand.hap.core.impl.RequestHelper;
import com.hand.hap.system.controllers.BaseController;
import com.hand.hap.system.dto.ResponseData;
import com.hand.hls.dp.service.SysDpDownloadLogService;
import com.hand.hls.dp.service.SysDpExecuteHistoryService;
import com.hand.hls.dp.util.IPUtils;
import leaf.bean.LeafRequestData;
......@@ -13,6 +14,7 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
......@@ -24,6 +26,8 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
@Autowired
private SysDpExecuteHistoryService service;
@Autowired
private SysDpDownloadLogService downloadLogService;
@PostMapping(value = "/sys/dp/v2/execute")
public ResponseData query(@ModelAttribute(LEAF_PARAM_NAME) LeafRequestData requestData,
......@@ -34,4 +38,14 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
final String dpContext = (String) requestData.get("parameter");
return service.execute(requestContext, ip, dpContext);
}
@PostMapping(value = "/sys/dp/v2/collect/log")
public void collectLog(HttpServletRequest request, HttpServletResponse response) throws Exception{
IRequest requestContext = createRequestContext(request);
// 发布消息收集日志
downloadLogService.collectLog(requestContext, request, response);
Thread.sleep(6000);
// 下载日志
downloadLogService.downloadClusterLogZipFile(requestContext, request, response);
}
}
v2-多线程版本/java-code/dp/service/SysDpDownloadLogService.java
View file @ eb7fefaa
......@@ -12,4 +12,6 @@ import javax.servlet.http.HttpServletResponse;
*/
public interface SysDpDownloadLogService {
void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
}
v2-多线程版本/java-code/dp/service/impl/SysClusterLogFileProviderImpl.java 0 → 100644
View file @ eb7fefaa
package com.hand.hls.dp.service.impl;
import com.hand.hap.message.IMessageConsumer;
import com.hand.hap.message.TopicMonitor;
import com.hand.hls.dp.util.LogUtils;
import org.apache.commons.codec.binary.Base64;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.CheckedOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
* @date 2020/9/3 14:32
* @since 1.0
*/
@TopicMonitor(channel = SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD)
public class SysClusterLogFileProviderImpl implements IMessageConsumer<String>, InitializingBean {
private Logger logger = LoggerFactory.getLogger(SysClusterLogFileProviderImpl.class);
public static final String LOGFILE_DOWNLOAD = "topic:logfile:download";
public static final String LOGFILE_KEY = "leaf:logs:zip";
@Value("${log-path}")
private String logPath;
@Autowired
private RedisTemplate<String, String> redisTemplate;
private void collectLog() {
ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
// 当天日志名称
String currLogFileName = logPath + "leaf.log";
// 查找日志目录下所有文件
List<String> files = new ArrayList<>(31);
LogUtils.findFiles(new File(logPath), files);
files.forEach(file -> {
if (file.equals(currLogFileName)) {
// 创建ZipEntry
ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
// 放入文件，防止文件名相同的附件报错
try {
zos.putNextEntry(entry);
} catch (Exception e) {
// ignore
}
// 写入文件
LogUtils.readFileToZip(file, zos);
}
});
try {
zos.closeEntry();
zos.close();
if (logger.isDebugEnabled()) {
logger.debug("redis key: [{}]", LOGFILE_KEY);
}
redisTemplate.opsForList().rightPush(LOGFILE_KEY, Base64.encodeBase64String(byteOutPut.toByteArray()));
cos.close();
} catch (Exception e) {
logger.warn(e.getMessage());
}
}
@Override
public void onMessage(String message, String pattern) {
if (LOGFILE_DOWNLOAD.equals(pattern)) {
if (logger.isDebugEnabled()) {
logger.debug("start do [{}]", message);
}
// 收集当天日志暂存redis
collectLog();
}
}
@Override
public void afterPropertiesSet() throws Exception {}
}
v2-多线程版本/java-code/dp/service/impl/SysDpDownloadLogServiceImpl.java
View file @ eb7fefaa
......@@ -2,10 +2,14 @@ package com.hand.hls.dp.service.impl;
import cn.hutool.core.date.DateUtil;
import com.hand.hap.core.IRequest;
import com.hand.hap.message.IMessagePublisher;
import com.hand.hls.dp.service.SysDpDownloadLogService;
import com.hand.hls.dp.util.LogUtils;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
......@@ -34,6 +38,85 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
@Value("${log-path}")
private String logPath;
@Autowired
private IMessagePublisher messagePublisher;
@Autowired
private RedisTemplate<String, String> redisTemplate;
/**
* 通知集群服务器当天日志暂存到redis
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override
public void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
messagePublisher.publish(SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD, "download logfile");
}
/**
* 下载集群服务器上当天日志
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override
public void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
LocalDate tmp = LocalDate.now();
Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
String zipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + ".zip";
response.reset();
response.setHeader("Pragma", "No-cache");
response.setHeader("Cache-Control", "no-cache,must-revalidate");
response.setHeader("Expires", "0");
response.setHeader("Content-Disposition", "attachment;filename=" + new String(zipFileName.getBytes(), StandardCharsets.ISO_8859_1));
response.setContentType("application/x-msdownload;");
List<String> zipFileList = redisTemplate.opsForList().range(SysClusterLogFileProviderImpl.LOGFILE_KEY, 0 , -1);
// 清空
while (redisTemplate.opsForList().size(SysClusterLogFileProviderImpl.LOGFILE_KEY) > 0){
redisTemplate.opsForList().leftPop(SysClusterLogFileProviderImpl.LOGFILE_KEY);
}
int size = zipFileList.size();
if (size >= 1) {
ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
for (int i = 0; i < size; i++) {
String zipFile = zipFileList.get(i);
String singleZipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + "-" + i + ".zip";
// 创建ZipEntry
ZipEntry entry = new ZipEntry(singleZipFileName);
// 放入文件，防止文件名相同的附件报错
try {
zos.putNextEntry(entry);
} catch (Exception e) {
// ignore
}
// 写入文件
byte[] byteArray = Base64.decodeBase64(zipFile);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray);
int n = 0;
byte[] buffer = new byte[LogUtils.DEFAULT_LARGE_BUFFER_SIZE];
while (LogUtils.EOF != (n = byteArrayInputStream.read(buffer))) {
zos.write(buffer, 0, n);
}
}
zos.closeEntry();
zos.close();
response.setHeader("Content-Length", String.valueOf(byteOutPut.toByteArray().length));
IOUtils.copy(new ByteArrayInputStream(byteOutPut.toByteArray()), response.getOutputStream());
cos.close();
}
}
/**
* 下载当前请求的服务器上7天日志
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override
public void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
String zipFileName = DateUtil.format(new Date(),"yyyyMMddHHmmss") + "_logs.zip";
......@@ -54,14 +137,14 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
for (int i = 1; i <= length - 1;i++){
LocalDate tmp = LocalDate.now().minusDays(i);
Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
downloadFiles[i] = logPath + "leaf-" + DateUtil.format(new Date(),"yyyy-MM-dd") + ".log";
downloadFiles[i] = logPath + "leaf-" + DateUtil.format(date,"yyyy-MM-dd") + ".log";
}
// 查找日志目录下所有文件
List<String> files = new ArrayList<>(30);
List<String> files = new ArrayList<>(31);
LogUtils.findFiles(new File(logPath),files);
files.forEach(file -> {
for (int i = 0; i < length; i++) {
if (file.equals(downloadFiles[i])) {
for (String downloadFile : downloadFiles) {
if (file.equals(downloadFile)) {
// 创建ZipEntry
ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
// 放入文件，防止文件名相同的附件报错
......@@ -71,7 +154,7 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
// ignore
}
// 写入文件
LogUtils.readFileToZip(file,zos);
LogUtils.readFileToZip(file, zos);
}
}
});
......
v2-多线程版本/lview-code/DP/DP001/sys_dp.lview
View file @ eb7fefaa
......@@ -3,119 +3,145 @@
<a:init-procedure/>
<a:view>
<script><![CDATA[
function dp001_query() {
$('dp001_query_btn').disable();
let dpContext = document.getElementById('context_tta').value;
Leaf.showConfirm('${l:HLS.PROMPT}', '确认执行当前语句吗？', function okFun() {
Leaf.Masker.mask(Ext.getBody(), '正在执行...');
Leaf.request({
url: '${/request/@context_path}/sys/dp/query?ck=0',
para: dpContext,
success: function () {
let form = document.createElement("form");
form.target = "_export_dp_query_window";
form.method = "post";
let url = '${/request/@context_path}/sys/dp/query?ck=1';
let token = $jq('meta[name=_csrf]').attr('content');
form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
let iframe = Ext.get('_export_dp_query_window') || new Ext.Template('<iframe id ="_export_dp_query_window" name="_export_dp_query_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
var s = document.createElement("input");
s.id = "_request_data";
s.type = 'hidden';
s.name = '_request_data';
let p = {"parameter": dpContext};
s.value = Ext.encode(p);
form.appendChild(s);
document.body.appendChild(form);
form.submit();
Ext.fly(form).remove();
setTimeout(function () {
function dp001_query() {
$('dp001_query_btn').disable();
let dpContext = document.getElementById('context_tta').value;
Leaf.showConfirm('${l:HLS.PROMPT}', '确认执行当前语句吗？', function okFun() {
Leaf.Masker.mask(Ext.getBody(), '正在执行...');
Leaf.request({
url: '${/request/@context_path}/sys/dp/query?ck=0',
para: dpContext,
success: function () {
let form = document.createElement("form");
form.target = "_export_dp_query_window";
form.method = "post";
let url = '${/request/@context_path}/sys/dp/query?ck=1';
let token = $jq('meta[name=_csrf]').attr('content');
form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
let iframe = Ext.get('_export_dp_query_window') || new Ext.Template('<iframe id ="_export_dp_query_window" name="_export_dp_query_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
var s = document.createElement("input");
s.id = "_request_data";
s.type = 'hidden';
s.name = '_request_data';
let p = {"parameter": dpContext};
s.value = Ext.encode(p);
form.appendChild(s);
document.body.appendChild(form);
form.submit();
Ext.fly(form).remove();
setTimeout(function () {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 10000);
},
failure: function () {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 10000);
},
failure: function () {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
error: function () {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
scope: this
});
}, function cancel() {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
},
error: function () {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
scope: this
});
}, function cancel() {
$('dp001_query_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
function dp001_exec() {
$('dp001_exec_btn').disable();
let dpContext = document.getElementById('context_tta').value;
Leaf.showConfirm('${l:HLS.PROMPT}', '确认执行当前语句吗？', function okFun() {
document.getElementById('result_tta').value = null;
Leaf.Masker.mask(Ext.getBody(), '正在执行...');
Leaf.request({
url: '${/request/@context_path}/sys/dp/v2/execute',
para: dpContext,
success: function (res) {
setTimeout(function () {
function dp001_exec() {
$('dp001_exec_btn').disable();
let dpContext = document.getElementById('context_tta').value;
Leaf.showConfirm('${l:HLS.PROMPT}', '确认执行当前语句吗？', function okFun() {
document.getElementById('result_tta').value = null;
Leaf.Masker.mask(Ext.getBody(), '正在执行...');
Leaf.request({
url: '${/request/@context_path}/sys/dp/v2/execute',
para: dpContext,
success: function (res) {
setTimeout(function () {
$('dp001_exec_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
if (res.success) {
Leaf.showMessage('提示', '执行完成！');
// document.getElementById('result_tta').value = res.error.message;
document.getElementById('result_tta').value = res.result.record;
}
}, 10000);
},
failure: function () {
$('dp001_exec_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
error: function () {
$('dp001_exec_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
if (res.success) {
Leaf.showMessage('提示', '执行完成！');
// document.getElementById('result_tta').value = res.error.message;
document.getElementById('result_tta').value = res.result.record;
}
}, 10000);
},
failure: function () {
$('dp001_exec_btn').enable();
},
scope: this
});
}, function cancel() {
$('dp001_exec_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
function download_cluster_log() {
$('dp001_download_cluster_log_btn').disable();
Leaf.showConfirm('${l:HLS.PROMPT}', '确认下载日志吗？', function okFun() {
Leaf.Masker.mask(Ext.getBody(), '正在下载中...');
let form = document.createElement("form");
form.target = "_export_dp_log_window";
form.method = "post";
let url = '${/request/@context_path}/sys/dp/v2/collect/log';
let token = $jq('meta[name=_csrf]').attr('content');
form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
let iframe = Ext.get('_export_dp_log_window') || new Ext.Template('<iframe id ="_export_dp_log_window" name="_export_dp_log_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
document.body.appendChild(form);
form.submit();
Ext.fly(form).remove();
setTimeout(function () {
$('dp001_download_cluster_log_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
error: function () {
$('dp001_exec_btn').enable();
}, 10000);
}, function cancel() {
$('dp001_download_cluster_log_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
function download_log() {
$('dp001_download_log_btn').disable();
Leaf.showConfirm('${l:HLS.PROMPT}', '确认下载日志吗？', function okFun() {
Leaf.Masker.mask(Ext.getBody(), '正在下载中...');
let form = document.createElement("form");
form.target = "_export_dp_log_window";
form.method = "post";
let url = '${/request/@context_path}/sys/dp/download/log';
let token = $jq('meta[name=_csrf]').attr('content');
form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
let iframe = Ext.get('_export_dp_log_window') || new Ext.Template('<iframe id ="_export_dp_log_window" name="_export_dp_log_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
document.body.appendChild(form);
form.submit();
Ext.fly(form).remove();
setTimeout(function () {
$('dp001_download_log_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
},
scope: this
});
}, function cancel() {
$('dp001_exec_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
}, 10000);
function download_log() {
$('dp001_download_log_btn').disable();
Leaf.showConfirm('${l:HLS.PROMPT}', '确认下载日志吗？', function okFun() {
Leaf.Masker.mask(Ext.getBody(), '正在下载中...');
let form = document.createElement("form");
form.target = "_export_dp_log_window";
form.method = "post";
let url = '${/request/@context_path}/sys/dp/download/log';
let token = $jq('meta[name=_csrf]').attr('content');
form.action = url + (url.indexOf('?') == -1 ? '?' : '&') + 'r=' + Math.random() + '&_csrf=' + token;
let iframe = Ext.get('_export_dp_log_window') || new Ext.Template('<iframe id ="_export_dp_log_window" name="_export_dp_log_window" style="position:absolute;left:-1000px;top:-1000px;width:1px;height:1px;display:none"></iframe>').insertFirst(document.body, {}, true)
document.body.appendChild(form);
form.submit();
Ext.fly(form).remove();
setTimeout(function () {
}, function cancel() {
$('dp001_download_log_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 10000);
}, function cancel() {
$('dp001_download_log_btn').enable();
Leaf.Masker.unmask(Ext.getBody());
}, 200, 100);
}
}, 200, 100);
}
]]></script>
<a:screenBody>
<a:screenTopToolbar>
<a:gridButton id="dp001_query_btn" click="dp001_query" text="查询"/>
<a:gridButton id="dp001_exec_btn" click="dp001_exec" text="执行"/>
<a:gridButton id="dp001_download_log_btn" click="download_log" text="日志下载"/>
<a:gridButton id="dp001_download_cluster_log_btn" click="download_cluster_log" text="日志下载(当天集群环境日志)"/>
<a:gridButton id="dp001_download_log_btn" click="download_log" text="日志下载(7天当前请求环境日志)"/>
</a:screenTopToolbar>
<a:fieldSet title="待执行">
<a:textArea id="context_tta" name="context" marginWidth="50" marginHeight="450"/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment