From 3c932648b915b3466fceea6cb70d96128accdd74 Mon Sep 17 00:00:00 2001 From: kcq Date: Sun, 19 Jul 2020 14:05:05 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + README.md | 12 +- ReleaseNotes.md | 6 +- doc/docker/Dockerfile | 2 +- frostmourne-core/pom.xml | 16 +- .../core/log4j2/AbstractFieldParser.java | 10 + .../core/log4j2/Autolog4jCsvLayout.java | 218 +++++++++++++++ .../core/log4j2/DefaultLog4jFieldFactory.java | 62 +++++ .../frostmourne/core/log4j2/EnumFilter.java | 71 +++++ .../frostmourne/core/log4j2/FieldFactory.java | 8 + .../frostmourne/core/log4j2/FieldName.java | 31 +++ .../frostmourne/core/log4j2/LayoutField.java | 259 ++++++++++++++++++ frostmourne-monitor/pom.xml | 4 +- frostmourne-monitor/src/assembly/config/env | 1 + .../src/assembly/scripts/startup.sh | 2 +- .../jobhandler/FrostmourneJobHandler.java | 2 +- .../src/main/resources/application.properties | 2 + .../src/main/resources/log4j2/log4j2.xml | 64 +++++ frostmourne-spi-starter/pom.xml | 4 +- frostmourne-spi/pom.xml | 3 +- frostmourne-spi/src/assembly/config/env | 1 + .../src/assembly/scripts/startup.sh | 2 +- .../src/main/resources/application.properties | 1 + .../src/main/resources/log4j2/log4j2.xml | 64 +++++ frostmourne-vue/pom.xml | 2 +- frostmourne-vue/vue.config.js | 2 +- pom.xml | 51 +++- 27 files changed, 875 insertions(+), 27 deletions(-) create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/AbstractFieldParser.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/Autolog4jCsvLayout.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/DefaultLog4jFieldFactory.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/EnumFilter.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldFactory.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldName.java create mode 100644 frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/LayoutField.java create mode 100644 frostmourne-monitor/src/main/resources/log4j2/log4j2.xml create mode 100644 frostmourne-spi/src/main/resources/log4j2/log4j2.xml diff --git a/.gitignore b/.gitignore index 5d9693f2..8ca809cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ HELP.md +.flattened-pom.xml application-autohome.properties target/ +logs/ !.mvn/wrapper/maven-wrapper.jar !**/src/main/** !**/src/test/** diff --git a/README.md b/README.md index 2d48968a..51a8b84b 100644 --- a/README.md +++ b/README.md @@ -324,9 +324,16 @@ dwz45.token=t8HGzRNv9TmvqUFICNoW3SaYNA1C9OAC * ~~增加企业微信机器人消息发送方式~~ [2020-07-05] * ~~用户信息，团队信息，部门信息外部文件增加定期重新加载~~ [2020-07-05] * ~~增加账号信息管理功能模块~~ [2020-07-11] +* ~~引入[maven-ci-friendly](https://maven.apache.org/maven-ci-friendly.html)实践~~ [2020-07-18] +* ~~使用[autolog4j](https://github.com/AutohomeCorp/autolog4j)程序日志格式~~ [2020-07-19] +* ~~文本日志按天滚动~~ [2020-07-19] +* 补充更详细的部署文档和使用指南 +* 更新在线demo至最新 * 新功能暂时停止，代码优化改进，稳定一段时间后发布0.2-RELEASE +* 监控列表增加按团队查询；监控列表增加权限控制，监控按部门隔离 * 增加企业钉钉发消息默认实现(本地没有环境，需要帮助，欢迎有环境的同僚联系，先行谢过) * 内置实现一个短链接功能，移除外部短链接服务依赖 +* 移除SPI模块，经过一系列优化后，spi模块存在的必要性可能很低了，考虑移除掉，降低部署难度 * Elasticsearch监控数值实现同比监控 * Elasticsearch监控数值实现环比监控 * Elasticsearch数据源更新免重启加载 @@ -334,13 +341,9 @@ dwz45.token=t8HGzRNv9TmvqUFICNoW3SaYNA1C9OAC * Elasticsearch查询数据柱状图可点击并自动变更时间范围 * Elasticsearch数据监控增加更多聚合类型(unique_count, percentiles)数值监控 * 增加系统配置功能模块，将启动非必要的配置用功能管理起来，减轻启动配置负担 -* 移除SPI模块，经过一系列优化后，spi模块存在的必要性可能很低了，考虑移除掉，降低部署难度 -* 补充更详细的部署文档 * README简化为文档目录索引形式，具体内容分散到各个文档中，方便查找 -* 监控列表增加按团队查询；监控列表增加权限控制，监控按部门隔离 * 监控调度配置后显示预计调度时间 * 报警接收人可以设置为组 -* 使用autolog4j程序日志格式 * 报警消息模板存库管理 * Elasticsearch数据名配置时自动提示索引名称 * Elasticsearch索引字段自动获取 @@ -355,6 +358,7 @@ dwz45.token=t8HGzRNv9TmvqUFICNoW3SaYNA1C9OAC * 国际化 * 移除xxl-job依赖，内置实现监控调度，减小部署难度(待定) * 发布1.0-RELEASE +* 加入更为复杂的时序数据异常检测算法规则(需要实验可行性) ## 主要技术栈 diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 704cb755..6007c905 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -36,12 +36,16 @@ * 增加query string简易教程 * 增加docker启动说明文档 -* 引入[mybatis-dynamic-sql](https://github.com/mybatis/mybatis-dynamic-sql) ### Others * Elasticsearch-Rest-Client升级至6.6.2 * 升级guava至28.2-jre +* 引入[mybatis-dynamic-sql](https://github.com/mybatis/mybatis-dynamic-sql) +* springboot升级至2.3.1-RELEASE +* 引入[maven-ci-friendly](https://maven.apache.org/maven-ci-friendly.html)实践 [2020-07-18] +* 使用[autolog4j](https://github.com/AutohomeCorp/autolog4j)程序日志格式 [2020-07-19] +* 文本日志按天滚动 [2020-07-19] ## 0.1-RELEASE diff --git a/doc/docker/Dockerfile b/doc/docker/Dockerfile index c2b02832..dd1a082c 100644 --- a/doc/docker/Dockerfile +++ b/doc/docker/Dockerfile @@ -2,9 +2,9 @@ FROM maven:3.6.3-ibmjava-8-alpine LABEL description="Frostmourne监控平台" WORKDIR /opt/frostmourne EXPOSE 9999 +EXPOSE 10052 EXPOSE 10053 EXPOSE 10054 -EXPOSE 10055 ENV XXL_JOB_VERSION=2.1.0 ENV FROSTMOURNE_VERSION=0.2-SNAPSHOT COPY xxl-job-admin-${XXL_JOB_VERSION}.zip /opt/frostmourne/xxl-job-admin.zip diff --git a/frostmourne-core/pom.xml b/frostmourne-core/pom.xml index 8e46e5ba..d24c3aaa 100644 --- a/frostmourne-core/pom.xml +++ b/frostmourne-core/pom.xml @@ -5,10 +5,9 @@ frostmourne com.autohome - 0.2-SNAPSHOT + ${revision} 4.0.0 - frostmourne-core @@ -49,5 +48,18 @@ org.slf4j slf4j-api + + org.apache.logging.log4j + log4j-core + + + org.apache.commons + commons-csv + 1.4 + + + joda-time + joda-time + \ No newline at end of file diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/AbstractFieldParser.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/AbstractFieldParser.java new file mode 100644 index 00000000..85c9a167 --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/AbstractFieldParser.java @@ -0,0 +1,10 @@ +package com.autohome.frostmourne.core.log4j2; + +import org.apache.logging.log4j.core.LogEvent; + +/** + * Created by kcq on 2017/6/8. + */ +public abstract class AbstractFieldParser { + public abstract String parse(String fieldName, LogEvent logEvent); +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/Autolog4jCsvLayout.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/Autolog4jCsvLayout.java new file mode 100644 index 00000000..12c48ce9 --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/Autolog4jCsvLayout.java @@ -0,0 +1,218 @@ +package com.autohome.frostmourne.core.log4j2; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import com.google.common.base.Strings; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.QuoteMode; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.CsvLogEventLayout; +import org.apache.logging.log4j.status.StatusLogger; + +/** + * Created by kcq on 2017/6/8. + */ +@Plugin(name = "Autolog4jCsvLayout", + category = Node.CATEGORY, + elementType = Layout.ELEMENT_TYPE, + printObject = true) +public class Autolog4jCsvLayout extends CsvLogEventLayout { + + private static final List defaultFieldNames = Arrays.asList(FieldName.LOG_AT, + FieldName.LEVEL, + FieldName.LOGGER, + FieldName.LINE, + FieldName.EXCEPTION_TYPE, + FieldName.EXCEPTION_MESSAGE, + FieldName.CUSTOM_MESSAGE, + FieldName.CLASS_NAME, + FieldName.METHOD_NAME, + FieldName.METHOD_PARAMS, + FieldName.TRACE_ID, + FieldName.DEPARTMENT, + FieldName.TEAM, + FieldName.PROJECT, + FieldName.APP_ID, + FieldName.THREAD_ID, + FieldName.THREAD_NAME, + FieldName.HOST, + FieldName.SERVER_IP, + FieldName.URI_STEM, + FieldName.QUERY_STRING, + FieldName.FORM_STRING, + FieldName.USER_AGENT, + FieldName.STACK_TRACE + ); + + private String team; + private String department; + private String project; + private String appId; + + private List fields; + private FieldFactory fieldFactory; + + protected Autolog4jCsvLayout(Configuration config, Charset charset, CSVFormat csvFormat, String header, String footer, + String department, String team, String project, String appId, FieldFactory fieldFactoryIn) { + super(config, charset, csvFormat, header, footer); + this.department = department; + this.team = team; + this.project = project; + this.appId = appId; + if (fieldFactoryIn == null) { + this.fieldFactory = new DefaultLog4jFieldFactory(); + } else { + this.fieldFactory = fieldFactoryIn; + } + } + + @PluginFactory + public static Autolog4jCsvLayout createLayout( + @PluginConfiguration final Configuration config, + @PluginAttribute(value = "format", defaultString = DEFAULT_FORMAT) final String format, + @PluginAttribute(value = "delimiter", defaultChar = '\t') final Character delimiter, + @PluginAttribute(value = "escape", defaultChar = '\\') final Character escape, + @PluginAttribute(value = "quote", defaultChar = '\"') final Character quote, + @PluginAttribute(value = "quoteMode", defaultString = "ALL") final QuoteMode quoteMode, + @PluginAttribute(value = "nullString", defaultString = "\"-\"") final String nullString, + @PluginAttribute(value = "recordSeparator", defaultString = "\r\n") final String recordSeparator, + @PluginAttribute(value = "charset", defaultString = DEFAULT_CHARSET) final Charset charset, + @PluginAttribute("header") final String header, + @PluginAttribute("footer") final String footer, + @PluginAttribute("department") final String department, + @PluginAttribute("team") final String team, + @PluginAttribute("project") final String project, + @PluginAttribute("appId") final String appId, + @PluginElement("FieldFactory") final FieldFactory fieldFactory) { + final CSVFormat csvFormat = createFormat(format, delimiter, escape, quote, quoteMode, nullString, recordSeparator); + return new Autolog4jCsvLayout(config, charset, csvFormat, header, footer, + department, team, project, appId, fieldFactory); + } + + @Override + public String toSerializable(final LogEvent event) { + final StringBuilder buffer = getStringBuilder(); + final CSVFormat format = getFormat(); + if (this.fields == null || this.fields.size() == 0) { + this.fields = getDefaultFields(); + } + try { + Iterator fieldIterator = this.fields.iterator(); + if (fieldIterator.hasNext()) { + format.print(handleEscapeChar(fieldIterator.next().format(event)), buffer, true); + while (fieldIterator.hasNext()) { + format.print(handleEscapeChar(fieldIterator.next().format(event)), buffer, false); + } + } + format.println(buffer); + return buffer.toString(); + } catch (final IOException e) { + StatusLogger.getLogger().error(event.toString(), e); + return format.getCommentMarker() + " " + e; + } + } + + public String getTeam() { + return team; + } + + public void setTeam(String team) { + this.team = team; + } + + public String getProject() { + return project; + } + + public void setProject(String project) { + this.project = project; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + private LayoutField createProjectField() { + if (!Strings.isNullOrEmpty(this.project)) { + this.project = this.project.toLowerCase(); + } + return new LayoutField("Project", this.project); + } + + private LayoutField createTeamField() { + if (!Strings.isNullOrEmpty(this.team)) { + this.team = this.team.toLowerCase(); + } + return new LayoutField("Team", this.team); + } + + private LayoutField createAppIdField() { + return new LayoutField("HawkKey", this.appId); + } + + private List getDefaultFields() { + List fields = new ArrayList<>(); + + for (String fieldName : defaultFieldNames) { + switch (fieldName) { + case FieldName.DEPARTMENT: + fields.add(createDepartmentField()); + break; + case FieldName.TEAM: + fields.add(createTeamField()); + break; + case FieldName.PROJECT: + fields.add(createProjectField()); + break; + case FieldName.APP_ID: + fields.add(createAppIdField()); + break; + default: + fields.add(this.fieldFactory.fetchField(fieldName)); + break; + } + } + + return fields; + } + + private String handleEscapeChar(String value) { + if (Strings.isNullOrEmpty(value) || !value.contains("\\\"")) { + return value; + } + + return value.replace("\\\"", "\""); + } + + private LayoutField createDepartmentField() { + if (!Strings.isNullOrEmpty(this.department)) { + this.department = this.department.toLowerCase(); + } + return new LayoutField("Department", this.department); + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/DefaultLog4jFieldFactory.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/DefaultLog4jFieldFactory.java new file mode 100644 index 00000000..6133f7cd --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/DefaultLog4jFieldFactory.java @@ -0,0 +1,62 @@ +package com.autohome.frostmourne.core.log4j2; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.core.AbstractLifeCycle; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +/** + * Created by kcq on 2018/1/3. + */ +@Plugin(name = "DefaultLog4jFieldFactory", + category = Node.CATEGORY, + printObject = true) +public class DefaultLog4jFieldFactory extends AbstractLifeCycle implements FieldFactory { + private static final Map defaultFieldMap = new HashMap() { + { + put(FieldName.LOG_AT, LayoutField.LOG_AT); + put(FieldName.TRACE_ID, LayoutField.TRACE_ID); + put(FieldName.HOST, LayoutField.HOST); + put(FieldName.SERVER_IP, LayoutField.SERVER_IP); + put(FieldName.URI_STEM, LayoutField.URI_STEM); + put(FieldName.QUERY_STRING, LayoutField.QUERY_STRING); + put(FieldName.FORM_STRING, LayoutField.FORM_STRING); + put(FieldName.USER_AGENT, LayoutField.USER_AGENT); + put(FieldName.LEVEL, LayoutField.LEVEL); + put(FieldName.CLASS_NAME, LayoutField.CLASS_NAME); + put(FieldName.METHOD_NAME, LayoutField.METHOD_NAME); + put(FieldName.METHOD_PARAMS, LayoutField.METHOD_PARAMS); + put(FieldName.LINE, LayoutField.LINE); + put(FieldName.LOGGER, LayoutField.LOGGER); + put(FieldName.EXCEPTION_TYPE, LayoutField.EXCEPTION_TYPE); + put(FieldName.EXCEPTION_MESSAGE, LayoutField.EXCEPTION_MESSAGE); + put(FieldName.CUSTOM_MESSAGE, LayoutField.CUSTOM_MESSAGE); + put(FieldName.STACK_TRACE, LayoutField.STACK_TRACE); + put(FieldName.THREAD_ID, LayoutField.THREAD_ID); + put(FieldName.THREAD_NAME, LayoutField.THREAD_NAME); + } + }; + + @PluginFactory + public static DefaultLog4jFieldFactory createDefaultFieldFactory() { + return new DefaultLog4jFieldFactory(); + } + + protected Map fetchCustomFieldMap() { + return null; + } + + @Override + public LayoutField fetchField(String name) { + Map customMaps = fetchCustomFieldMap(); + if (customMaps != null && customMaps.size() > 0 + && customMaps.containsKey(name)) { + return customMaps.get(name); + } else { + return defaultFieldMap.get(name); + } + } +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/EnumFilter.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/EnumFilter.java new file mode 100644 index 00000000..f3676296 --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/EnumFilter.java @@ -0,0 +1,71 @@ +package com.autohome.frostmourne.core.log4j2; + +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.base.Splitter; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.filter.AbstractFilter; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.util.PerformanceSensitive; + +@Plugin(name = "EnumFilter", category = Node.CATEGORY, elementType = Filter.ELEMENT_TYPE, printObject = true) +@PerformanceSensitive("allocation") +public class EnumFilter extends AbstractFilter { + + private final List allowLevelList; + + private EnumFilter(List allowLevelList) { + super(Result.ACCEPT, Result.DENY); + this.allowLevelList = allowLevelList; + } + + @PluginFactory + public static EnumFilter createFilter( + @PluginAttribute("allowLevels") final String allowLevels) { + List allowLevelList = Splitter.on(",").trimResults().splitToList(allowLevels) + .stream().map(Level::getLevel).collect(Collectors.toList()); + return new EnumFilter(allowLevelList); + } + + @Override + public Result filter(final Logger logger, final Level level, final Marker marker, final String msg, + final Object... params) { + return filter(level); + } + + @Override + public Result filter(final Logger logger, final Level level, final Marker marker, final Object msg, + final Throwable t) { + return filter(level); + } + + @Override + public Result filter(final Logger logger, final Level level, final Marker marker, final Message msg, + final Throwable t) { + return filter(level); + } + + @Override + public Result filter(final LogEvent event) { + return filter(event.getLevel()); + } + + private Result filter(final Level level) { + if(allowLevelList.contains(Level.OFF)) { + return onMismatch; + } + if (allowLevelList.contains(level)) { + return onMatch; + } + return onMismatch; + } +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldFactory.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldFactory.java new file mode 100644 index 00000000..24f2833b --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldFactory.java @@ -0,0 +1,8 @@ +package com.autohome.frostmourne.core.log4j2; + +/** + * Created by kcq on 2018/1/3. + */ +public interface FieldFactory { + LayoutField fetchField(String name); +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldName.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldName.java new file mode 100644 index 00000000..3e96daf2 --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/FieldName.java @@ -0,0 +1,31 @@ +package com.autohome.frostmourne.core.log4j2; + +/** + * Created by kcq on 2018/1/3. + */ +public class FieldName { + public static final String LOG_AT = "LogAt"; + public static final String TRACE_ID = "TraceId"; + public static final String DEPARTMENT = "Department"; + public static final String TEAM = "Team"; + public static final String PROJECT = "Project"; + public static final String HOST = "Host"; + public static final String SERVER_IP = "ServerIP"; + public static final String URI_STEM = "UriStem"; + public static final String QUERY_STRING = "QueryString"; + public static final String FORM_STRING = "FormString"; + public static final String USER_AGENT = "UserAgent"; + public static final String LEVEL = "Level"; + public static final String CLASS_NAME = "Class"; + public static final String METHOD_NAME = "Method"; + public static final String METHOD_PARAMS = "MethodParams"; + public static final String LINE = "Line"; + public static final String LOGGER = "Logger"; + public static final String EXCEPTION_TYPE = "ExceptionType"; + public static final String EXCEPTION_MESSAGE = "ExceptionMessage"; + public static final String CUSTOM_MESSAGE = "CustomMessage"; + public static final String STACK_TRACE = "StackTrace"; + public static final String APP_ID = "AppId"; + public static final String THREAD_NAME = "ThreadName"; + public static final String THREAD_ID = "ThreadId"; +} diff --git a/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/LayoutField.java b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/LayoutField.java new file mode 100644 index 00000000..5153adae --- /dev/null +++ b/frostmourne-core/src/main/java/com/autohome/frostmourne/core/log4j2/LayoutField.java @@ -0,0 +1,259 @@ +package com.autohome.frostmourne.core.log4j2; + +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.UnknownHostException; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.util.StringBuilderWriter; +import org.joda.time.DateTime; + +/** + * Created by kcq on 2017/6/8. + */ +public class LayoutField { + private static String LOCAL_IP_ADDR; + + private final String name; + private AbstractFieldParser parser; + private String value; + + public LayoutField(String name, AbstractFieldParser parser) { + this.name = name; + this.parser = parser; + } + + public LayoutField(String name, String value) { + this.name = name; + this.value = value; + } + + public String format(LogEvent logEvent) { + if (!Strings.isNullOrEmpty(this.value)) { + return this.value; + } + if (this.parser == null) { + return null; + } + return this.parser.parse(this.name, logEvent); + } + + public String getName() { + return this.name; + } + + + public static LayoutField create(String name, AbstractFieldParser fieldParser) { + return new LayoutField(name, fieldParser); + } + + public static final LayoutField LOG_AT = create(FieldName.LOG_AT, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return new DateTime(logEvent.getTimeMillis()).toDateTimeISO().toString(); + } + }); + + public static final LayoutField CLASS_NAME = create(FieldName.CLASS_NAME, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getSource() != null ? logEvent.getSource().getClassName() : null; + } + }); + + public static final LayoutField LINE = create(FieldName.LINE, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getSource() != null ? logEvent.getSource().getLineNumber() + "" : "0"; + } + }); + + public static final LayoutField METHOD_NAME = create(FieldName.METHOD_NAME, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getSource() != null ? logEvent.getSource().getMethodName() : null; + } + }); + + public static final LayoutField EXCEPTION_TYPE = create(FieldName.EXCEPTION_TYPE, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getThrown() == null) { + return null; + } + Throwable ex = logEvent.getThrown(); + return ex.getClass().getCanonicalName(); + } + }); + + public static final LayoutField EXCEPTION_MESSAGE = create(FieldName.EXCEPTION_MESSAGE, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getThrown() == null) { + return null; + } + return logEvent.getThrown().getMessage(); + } + }); + + + public static final LayoutField STACK_TRACE = create(FieldName.STACK_TRACE, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getThrown() == null) { + return null; + } + StringBuilderWriter sbw = new StringBuilderWriter(); + try (PrintWriter pw = new PrintWriter(sbw)) { + logEvent.getThrown().printStackTrace(pw); + return sbw.toString(); + } + } + }); + + public static final LayoutField CUSTOM_MESSAGE = create(FieldName.CUSTOM_MESSAGE, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getMessage() != null ? logEvent.getMessage().getFormattedMessage() : null; + } + }); + + public static final LayoutField LOGGER = create(FieldName.LOGGER, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getLoggerName(); + } + }); + + public static final LayoutField TRACE_ID = create(FieldName.TRACE_ID, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey("Custom-TraceId")) { + return logEvent.getContextData().getValue("Custom-TraceId"); + } + if (logEvent.getContextData().containsKey("X-B3-TraceId")) { + return logEvent.getContextData().getValue("X-B3-TraceId"); + } + if (logEvent.getContextData().containsKey("tid")) { + //skywalking traceid + return logEvent.getContextData().getValue("tid"); + } + if (logEvent.getContextData().containsKey("SW-TraceId")) { + return logEvent.getContextData().getValue("SW-TraceId"); + } + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField HOST = create(FieldName.HOST, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField SERVER_IP = create(FieldName.SERVER_IP, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return LayoutField.findServerIP(); + } + }); + + public static final LayoutField URI_STEM = create(FieldName.URI_STEM, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField QUERY_STRING = create(FieldName.QUERY_STRING, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField FORM_STRING = create(FieldName.FORM_STRING, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getLevel().isMoreSpecificThan(Level.ERROR) && logEvent.getContextData().containsKey(fieldName)) { + String formString = logEvent.getContextData().getValue(fieldName); + if (formString.length() > 2000) { + return formString.substring(0, 2000); + } + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField LEVEL = create(FieldName.LEVEL, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return logEvent.getLevel().name().toUpperCase(); + } + }); + + public static final LayoutField METHOD_PARAMS = create(FieldName.METHOD_PARAMS, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getThrown() == null) { + return null; + } + return null; + } + }); + + public static final LayoutField USER_AGENT = create(FieldName.USER_AGENT, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + if (logEvent.getContextData().containsKey(fieldName)) { + return logEvent.getContextData().getValue(fieldName); + } + return null; + } + }); + + public static final LayoutField THREAD_NAME = create(FieldName.THREAD_NAME, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return Thread.currentThread().getName(); + } + }); + + public static final LayoutField THREAD_ID = create(FieldName.THREAD_ID, new AbstractFieldParser() { + @Override + public String parse(String fieldName, LogEvent logEvent) { + return String.valueOf(Thread.currentThread().getId()); + } + }); + + + public static String findServerIP() { + if (Strings.isNullOrEmpty(LOCAL_IP_ADDR)) { + try { + LOCAL_IP_ADDR = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + return null; + } + } + return LOCAL_IP_ADDR; + } +} diff --git a/frostmourne-monitor/pom.xml b/frostmourne-monitor/pom.xml index 96da016a..ab9d6cd8 100644 --- a/frostmourne-monitor/pom.xml +++ b/frostmourne-monitor/pom.xml @@ -5,10 +5,9 @@ com.autohome frostmourne - 0.2-SNAPSHOT + ${revision} frostmourne-monitor - 0.2-SNAPSHOT frostmourne-monitor frostmourne-monitor @@ -84,7 +83,6 @@ joda-time joda-time - 2.9.9 com.google.guava diff --git a/frostmourne-monitor/src/assembly/config/env b/frostmourne-monitor/src/assembly/config/env index df4cd112..989e2b43 100644 --- a/frostmourne-monitor/src/assembly/config/env +++ b/frostmourne-monitor/src/assembly/config/env @@ -2,6 +2,7 @@ MODE=service JAVA_HOME=/usr/local/java/jdk1.8.0_05 #PID_FOLDER=. #LOG_DIR=logs +LOG_CONSOLE_LEVEL=OFF ## Adjust server port if necessary SERVER_PORT=10054 PROFILE=dev \ No newline at end of file diff --git a/frostmourne-monitor/src/assembly/scripts/startup.sh b/frostmourne-monitor/src/assembly/scripts/startup.sh index dccc825d..de9e7cd9 100644 --- a/frostmourne-monitor/src/assembly/scripts/startup.sh +++ b/frostmourne-monitor/src/assembly/scripts/startup.sh @@ -35,7 +35,7 @@ export JAVA_OPTS="-Xms1024m -Xmx1024m -Xss256k -XX:MetaspaceSize=64m -XX:MaxMeta #export JAVA_OPTS="$JAVA_OPTS -server -XX:-ReduceInitialCardMarks" export JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC -XX:ParallelGCThreads=4 -XX:MaxTenuringThreshold=9 -XX:+UseConcMarkSweepGC -XX:+DisableExplicitGC -XX:+UseCMSInitiatingOccupancyOnly -XX:+ScavengeBeforeFullGC -XX:+UseCMSCompactAtFullCollection -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSClassUnloadingEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSPermGenSweepingEnabled -XX:CMSInitiatingPermOccupancyFraction=70 -XX:+ExplicitGCInvokesConcurrent -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" -export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dspring.profiles.active=$PROFILE -Dlogs.dir=$LOG_DIR -Xloggc:$LOG_DIR/heap_trace.txt -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/" +export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dspring.profiles.active=$PROFILE -Dlogs.dir=$LOG_DIR -Dlog.console.level=${LOG_CONSOLE_LEVEL} -Xloggc:$LOG_DIR/heap_trace.txt -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/" PATH_TO_JAR=$SERVICE_NAME".jar" SERVER_URL="http://localhost:$SERVER_PORT" diff --git a/frostmourne-monitor/src/main/java/com/autohome/frostmourne/monitor/jobhandler/FrostmourneJobHandler.java b/frostmourne-monitor/src/main/java/com/autohome/frostmourne/monitor/jobhandler/FrostmourneJobHandler.java index 0044c4bf..7377649e 100644 --- a/frostmourne-monitor/src/main/java/com/autohome/frostmourne/monitor/jobhandler/FrostmourneJobHandler.java +++ b/frostmourne-monitor/src/main/java/com/autohome/frostmourne/monitor/jobhandler/FrostmourneJobHandler.java @@ -43,7 +43,7 @@ public ReturnT execute(String param) throws Exception { Map paramMap = JacksonObjectMapper.getCommonObjectMapper().readValue(param, typeRef); Long alarmId = new Long(paramMap.get("alarmId").toString()); alarmService.run("system", alarmId, false); - XxlJobLogger.log("frostmourne, job begin."); + XxlJobLogger.log("frostmourne, job end."); return SUCCESS; } catch (Exception ex) { LOGGER.error("error when execute, param: " + param, ex); diff --git a/frostmourne-monitor/src/main/resources/application.properties b/frostmourne-monitor/src/main/resources/application.properties index 95a99954..1292c3fa 100644 --- a/frostmourne-monitor/src/main/resources/application.properties +++ b/frostmourne-monitor/src/main/resources/application.properties @@ -1,4 +1,6 @@ server.port=10054 +logging.config=classpath:log4j2/log4j2.xml + initial.password=${initial_password:#{null}} alarm.log.reserve.days=${alarmlog_reserve_days:30} diff --git a/frostmourne-monitor/src/main/resources/log4j2/log4j2.xml b/frostmourne-monitor/src/main/resources/log4j2/log4j2.xml new file mode 100644 index 00000000..6de201ed --- /dev/null +++ b/frostmourne-monitor/src/main/resources/log4j2/log4j2.xml @@ -0,0 +1,64 @@ + + + + ???? + %xwEx + %5p + yyyy-MM-dd HH:mm:ss.SSS + %clr{%d{${LOG_DATEFORMAT_PATTERN}}}{faint} %clr{${LOG_LEVEL_PATTERN}} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n${sys:LOG_EXCEPTION_CONVERSION_WORD} + + logs + tech + frostmourne + frostmourne.monitor + OFF + INFO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frostmourne-spi-starter/pom.xml b/frostmourne-spi-starter/pom.xml index bd936fe2..782d220c 100644 --- a/frostmourne-spi-starter/pom.xml +++ b/frostmourne-spi-starter/pom.xml @@ -5,10 +5,9 @@ com.autohome frostmourne - 0.2-SNAPSHOT + ${revision} frostmourne-spi-starter - 0.2-SNAPSHOT frostmourne-spi-starter frostmourne-spi-starter @@ -21,7 +20,6 @@ com.autohome frostmourne-core - 0.2-SNAPSHOT org.springframework.boot diff --git a/frostmourne-spi/pom.xml b/frostmourne-spi/pom.xml index 4b7ca31b..53967658 100644 --- a/frostmourne-spi/pom.xml +++ b/frostmourne-spi/pom.xml @@ -5,10 +5,9 @@ com.autohome frostmourne - 0.2-SNAPSHOT + ${revision} frostmourne-spi - 0.2-SNAPSHOT frostmourne-spi frostmourne spi diff --git a/frostmourne-spi/src/assembly/config/env b/frostmourne-spi/src/assembly/config/env index 3e34b40b..3122b4a4 100644 --- a/frostmourne-spi/src/assembly/config/env +++ b/frostmourne-spi/src/assembly/config/env @@ -2,6 +2,7 @@ MODE=service #JAVA_HOME=/usr/local/java/jdk1.8.0_05 #PID_FOLDER=. #LOG_DIR=logs +LOG_CONSOLE_LEVEL=OFF ## Adjust server port if necessary SERVER_PORT=10053 PROFILE=default \ No newline at end of file diff --git a/frostmourne-spi/src/assembly/scripts/startup.sh b/frostmourne-spi/src/assembly/scripts/startup.sh index c206e853..24dcf13f 100644 --- a/frostmourne-spi/src/assembly/scripts/startup.sh +++ b/frostmourne-spi/src/assembly/scripts/startup.sh @@ -35,7 +35,7 @@ export JAVA_OPTS="-Xms1024m -Xmx1024m -Xss256k -XX:MetaspaceSize=64m -XX:MaxMeta #export JAVA_OPTS="$JAVA_OPTS -server -XX:-ReduceInitialCardMarks" export JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC -XX:ParallelGCThreads=4 -XX:MaxTenuringThreshold=9 -XX:+UseConcMarkSweepGC -XX:+DisableExplicitGC -XX:+UseCMSInitiatingOccupancyOnly -XX:+ScavengeBeforeFullGC -XX:+UseCMSCompactAtFullCollection -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSClassUnloadingEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSPermGenSweepingEnabled -XX:CMSInitiatingPermOccupancyFraction=70 -XX:+ExplicitGCInvokesConcurrent -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" -export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dspring.profiles.active=$PROFILE -Dlogs.dir=$LOG_DIR -Xloggc:$LOG_DIR/heap_trace.txt -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/" +export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dspring.profiles.active=$PROFILE -Dlogs.dir=$LOG_DIR -Dlog.console.level=${LOG_CONSOLE_LEVEL} -Xloggc:$LOG_DIR/heap_trace.txt -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/" PATH_TO_JAR=$SERVICE_NAME".jar" SERVER_URL="http://localhost:$SERVER_PORT" diff --git a/frostmourne-spi/src/main/resources/application.properties b/frostmourne-spi/src/main/resources/application.properties index 87deecd5..da85abe2 100644 --- a/frostmourne-spi/src/main/resources/application.properties +++ b/frostmourne-spi/src/main/resources/application.properties @@ -1,4 +1,5 @@ server.port=10053 +logging.config=classpath:log4j2/log4j2.xml email.smtp.host=${your.email.smtp.host:#{null}} email.smtp.port=${your.email.smtp.port:#{null}} diff --git a/frostmourne-spi/src/main/resources/log4j2/log4j2.xml b/frostmourne-spi/src/main/resources/log4j2/log4j2.xml new file mode 100644 index 00000000..d707be32 --- /dev/null +++ b/frostmourne-spi/src/main/resources/log4j2/log4j2.xml @@ -0,0 +1,64 @@ + + + + ???? + %xwEx + %5p + yyyy-MM-dd HH:mm:ss.SSS + %clr{%d{${LOG_DATEFORMAT_PATTERN}}}{faint} %clr{${LOG_LEVEL_PATTERN}} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n${sys:LOG_EXCEPTION_CONVERSION_WORD} + + logs + tech + frostmourne + frostmourne.spi + OFF + INFO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frostmourne-vue/pom.xml b/frostmourne-vue/pom.xml index 4b16443c..e1899619 100644 --- a/frostmourne-vue/pom.xml +++ b/frostmourne-vue/pom.xml @@ -5,7 +5,7 @@ frostmourne com.autohome - 0.2-SNAPSHOT + ${revision} 4.0.0 frostmourne-vue diff --git a/frostmourne-vue/vue.config.js b/frostmourne-vue/vue.config.js index f8e8e479..09230683 100644 --- a/frostmourne-vue/vue.config.js +++ b/frostmourne-vue/vue.config.js @@ -47,7 +47,7 @@ module.exports = { } }, '/monitor-api/': { - target: `http://localhost:8080/`, + target: `http://localhost:10054/`, changeOrigin: true, pathRewrite: { '^/api/monitor-api/': '' diff --git a/pom.xml b/pom.xml index 0d74394f..52cc59c7 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,12 @@ org.springframework.boot spring-boot-starter-parent - 2.3.0.RELEASE + 2.3.1.RELEASE com.autohome frostmourne - 0.2-SNAPSHOT + ${revision} pom frostmourne frostmourne for Spring Boot @@ -27,8 +27,8 @@ 1.8 UTF-8 UTF-8 + 0.2-RELEASE 1.7.12 - 2.10.3 28.2-jre @@ -45,19 +45,19 @@ org.springframework.boot spring-boot-dependencies - 2.3.0.RELEASE + 2.3.1.RELEASE pom import com.autohome frostmourne-core - 0.2-SNAPSHOT + ${project.version} com.autohome frostmourne-spi-starter - 0.2-SNAPSHOT + ${project.version} com.google.guava @@ -85,6 +85,45 @@ jackson-core ${jackson.version} + + joda-time + joda-time + 2.9.9 + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.apache.maven.plugins + maven-source-plugin + + +