Commit eb7fefaa authored by Spencer Chang's avatar Spencer Chang

[feat] 新增下载同一集群环境下所有服务器当天的日志

用于解决问题查看日志
parent 35f3c618
## v1.1
### 1、新增下载同一集群环境下所有服务器当天的日志
#### 通过redis消息服务每个服务器把各自的日志打包zip文件暂存redis
### 2、配置文件 spring/applicationContext-redis.xml 添加:
```xml
<bean id="sysClusterLogFileProviderImpl" class="com.hand.hls.dp.service.impl.SysClusterLogFileProviderImpl"/>
```
...@@ -4,6 +4,7 @@ import com.hand.hap.core.IRequest; ...@@ -4,6 +4,7 @@ import com.hand.hap.core.IRequest;
import com.hand.hap.core.impl.RequestHelper; import com.hand.hap.core.impl.RequestHelper;
import com.hand.hap.system.controllers.BaseController; import com.hand.hap.system.controllers.BaseController;
import com.hand.hap.system.dto.ResponseData; import com.hand.hap.system.dto.ResponseData;
import com.hand.hls.dp.service.SysDpDownloadLogService;
import com.hand.hls.dp.service.SysDpExecuteHistoryService; import com.hand.hls.dp.service.SysDpExecuteHistoryService;
import com.hand.hls.dp.util.IPUtils; import com.hand.hls.dp.util.IPUtils;
import leaf.bean.LeafRequestData; import leaf.bean.LeafRequestData;
...@@ -13,6 +14,7 @@ import org.springframework.web.bind.annotation.PostMapping; ...@@ -13,6 +14,7 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/** /**
* @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a> * @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
...@@ -24,6 +26,8 @@ public class SysDpExecuteHistoryV2Controller extends BaseController { ...@@ -24,6 +26,8 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
@Autowired @Autowired
private SysDpExecuteHistoryService service; private SysDpExecuteHistoryService service;
@Autowired
private SysDpDownloadLogService downloadLogService;
@PostMapping(value = "/sys/dp/v2/execute") @PostMapping(value = "/sys/dp/v2/execute")
public ResponseData query(@ModelAttribute(LEAF_PARAM_NAME) LeafRequestData requestData, public ResponseData query(@ModelAttribute(LEAF_PARAM_NAME) LeafRequestData requestData,
...@@ -34,4 +38,14 @@ public class SysDpExecuteHistoryV2Controller extends BaseController { ...@@ -34,4 +38,14 @@ public class SysDpExecuteHistoryV2Controller extends BaseController {
final String dpContext = (String) requestData.get("parameter"); final String dpContext = (String) requestData.get("parameter");
return service.execute(requestContext, ip, dpContext); return service.execute(requestContext, ip, dpContext);
} }
@PostMapping(value = "/sys/dp/v2/collect/log")
public void collectLog(HttpServletRequest request, HttpServletResponse response) throws Exception{
IRequest requestContext = createRequestContext(request);
// 发布消息收集日志
downloadLogService.collectLog(requestContext, request, response);
Thread.sleep(6000);
// 下载日志
downloadLogService.downloadClusterLogZipFile(requestContext, request, response);
}
} }
...@@ -12,4 +12,6 @@ import javax.servlet.http.HttpServletResponse; ...@@ -12,4 +12,6 @@ import javax.servlet.http.HttpServletResponse;
*/ */
public interface SysDpDownloadLogService { public interface SysDpDownloadLogService {
void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception; void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception;
} }
package com.hand.hls.dp.service.impl;
import com.hand.hap.message.IMessageConsumer;
import com.hand.hap.message.TopicMonitor;
import com.hand.hls.dp.util.LogUtils;
import org.apache.commons.codec.binary.Base64;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.CheckedOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* @author <a href="mailto:zhangnet14@gmail.com">Spencer Chang</a>
* @date 2020/9/3 14:32
* @since 1.0
*/
@TopicMonitor(channel = SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD)
public class SysClusterLogFileProviderImpl implements IMessageConsumer<String>, InitializingBean {
private Logger logger = LoggerFactory.getLogger(SysClusterLogFileProviderImpl.class);
public static final String LOGFILE_DOWNLOAD = "topic:logfile:download";
public static final String LOGFILE_KEY = "leaf:logs:zip";
@Value("${log-path}")
private String logPath;
@Autowired
private RedisTemplate<String, String> redisTemplate;
private void collectLog() {
ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
// 当天日志名称
String currLogFileName = logPath + "leaf.log";
// 查找日志目录下所有文件
List<String> files = new ArrayList<>(31);
LogUtils.findFiles(new File(logPath), files);
files.forEach(file -> {
if (file.equals(currLogFileName)) {
// 创建ZipEntry
ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
// 放入文件,防止文件名相同的附件报错
try {
zos.putNextEntry(entry);
} catch (Exception e) {
// ignore
}
// 写入文件
LogUtils.readFileToZip(file, zos);
}
});
try {
zos.closeEntry();
zos.close();
if (logger.isDebugEnabled()) {
logger.debug("redis key: [{}]", LOGFILE_KEY);
}
redisTemplate.opsForList().rightPush(LOGFILE_KEY, Base64.encodeBase64String(byteOutPut.toByteArray()));
cos.close();
} catch (Exception e) {
logger.warn(e.getMessage());
}
}
@Override
public void onMessage(String message, String pattern) {
if (LOGFILE_DOWNLOAD.equals(pattern)) {
if (logger.isDebugEnabled()) {
logger.debug("start do [{}]", message);
}
// 收集当天日志暂存redis
collectLog();
}
}
@Override
public void afterPropertiesSet() throws Exception {}
}
...@@ -2,10 +2,14 @@ package com.hand.hls.dp.service.impl; ...@@ -2,10 +2,14 @@ package com.hand.hls.dp.service.impl;
import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUtil;
import com.hand.hap.core.IRequest; import com.hand.hap.core.IRequest;
import com.hand.hap.message.IMessagePublisher;
import com.hand.hls.dp.service.SysDpDownloadLogService; import com.hand.hls.dp.service.SysDpDownloadLogService;
import com.hand.hls.dp.util.LogUtils; import com.hand.hls.dp.util.LogUtils;
import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
...@@ -34,6 +38,85 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService { ...@@ -34,6 +38,85 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
@Value("${log-path}") @Value("${log-path}")
private String logPath; private String logPath;
@Autowired
private IMessagePublisher messagePublisher;
@Autowired
private RedisTemplate<String, String> redisTemplate;
/**
* 通知集群服务器当天日志暂存到redis
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override
public void collectLog(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
messagePublisher.publish(SysClusterLogFileProviderImpl.LOGFILE_DOWNLOAD, "download logfile");
}
/**
* 下载集群服务器上当天日志
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override
public void downloadClusterLogZipFile(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
LocalDate tmp = LocalDate.now();
Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
String zipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + ".zip";
response.reset();
response.setHeader("Pragma", "No-cache");
response.setHeader("Cache-Control", "no-cache,must-revalidate");
response.setHeader("Expires", "0");
response.setHeader("Content-Disposition", "attachment;filename=" + new String(zipFileName.getBytes(), StandardCharsets.ISO_8859_1));
response.setContentType("application/x-msdownload;");
List<String> zipFileList = redisTemplate.opsForList().range(SysClusterLogFileProviderImpl.LOGFILE_KEY, 0 , -1);
// 清空
while (redisTemplate.opsForList().size(SysClusterLogFileProviderImpl.LOGFILE_KEY) > 0){
redisTemplate.opsForList().leftPop(SysClusterLogFileProviderImpl.LOGFILE_KEY);
}
int size = zipFileList.size();
if (size >= 1) {
ByteArrayOutputStream byteOutPut = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(byteOutPut, new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
for (int i = 0; i < size; i++) {
String zipFile = zipFileList.get(i);
String singleZipFileName = "leaf-" + DateUtil.format(date, "yyyy-MM-dd") + "-" + i + ".zip";
// 创建ZipEntry
ZipEntry entry = new ZipEntry(singleZipFileName);
// 放入文件,防止文件名相同的附件报错
try {
zos.putNextEntry(entry);
} catch (Exception e) {
// ignore
}
// 写入文件
byte[] byteArray = Base64.decodeBase64(zipFile);
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray);
int n = 0;
byte[] buffer = new byte[LogUtils.DEFAULT_LARGE_BUFFER_SIZE];
while (LogUtils.EOF != (n = byteArrayInputStream.read(buffer))) {
zos.write(buffer, 0, n);
}
}
zos.closeEntry();
zos.close();
response.setHeader("Content-Length", String.valueOf(byteOutPut.toByteArray().length));
IOUtils.copy(new ByteArrayInputStream(byteOutPut.toByteArray()), response.getOutputStream());
cos.close();
}
}
/**
* 下载当前请求的服务器上7天日志
* @param requestContext 请求上下文
* @param request http请求
* @param response http响应
* @throws Exception 异常
*/
@Override @Override
public void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception { public void download(IRequest requestContext, HttpServletRequest request, HttpServletResponse response) throws Exception {
String zipFileName = DateUtil.format(new Date(),"yyyyMMddHHmmss") + "_logs.zip"; String zipFileName = DateUtil.format(new Date(),"yyyyMMddHHmmss") + "_logs.zip";
...@@ -54,14 +137,14 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService { ...@@ -54,14 +137,14 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
for (int i = 1; i <= length - 1;i++){ for (int i = 1; i <= length - 1;i++){
LocalDate tmp = LocalDate.now().minusDays(i); LocalDate tmp = LocalDate.now().minusDays(i);
Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); Date date = Date.from(tmp.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
downloadFiles[i] = logPath + "leaf-" + DateUtil.format(new Date(),"yyyy-MM-dd") + ".log"; downloadFiles[i] = logPath + "leaf-" + DateUtil.format(date,"yyyy-MM-dd") + ".log";
} }
// 查找日志目录下所有文件 // 查找日志目录下所有文件
List<String> files = new ArrayList<>(30); List<String> files = new ArrayList<>(31);
LogUtils.findFiles(new File(logPath),files); LogUtils.findFiles(new File(logPath),files);
files.forEach(file -> { files.forEach(file -> {
for (int i = 0; i < length; i++) { for (String downloadFile : downloadFiles) {
if (file.equals(downloadFiles[i])) { if (file.equals(downloadFile)) {
// 创建ZipEntry // 创建ZipEntry
ZipEntry entry = new ZipEntry(file.substring(logPath.length())); ZipEntry entry = new ZipEntry(file.substring(logPath.length()));
// 放入文件,防止文件名相同的附件报错 // 放入文件,防止文件名相同的附件报错
...@@ -71,7 +154,7 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService { ...@@ -71,7 +154,7 @@ public class SysDpDownloadLogServiceImpl implements SysDpDownloadLogService {
// ignore // ignore
} }
// 写入文件 // 写入文件
LogUtils.readFileToZip(file,zos); LogUtils.readFileToZip(file, zos);
} }
} }
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment