From b1ec1dfa6fc34e5192168bbaaf54a1e8c6382449 Mon Sep 17 00:00:00 2001 From: Aidaho Date: Thu, 28 Nov 2024 15:43:07 +0300 Subject: [PATCH] v1.2.3: Add filtering and sorting for checks in RMON API Introduced the `AllChecksViewWithFilters` method to support filtering and sorting of checks based on criteria such as check group, name, status, and type. Modified the database queries in `smon.py` and updated the API and data models accordingly to accommodate these enhancements. Additionally, refactored the checks view for improved code organization and functionality. --- app/api/v1/routes/rmon/routes.py | 4 +- app/modules/db/smon.py | 45 +++++ app/modules/roxywi/class_models.py | 12 ++ app/views/check/checks_view.py | 258 ++++++++++++++++++++++++----- app/views/check/views.py | 6 +- 5 files changed, 280 insertions(+), 45 deletions(-) diff --git a/app/api/v1/routes/rmon/routes.py b/app/api/v1/routes/rmon/routes.py index 17f28cd..a7d5aef 100644 --- a/app/api/v1/routes/rmon/routes.py +++ b/app/api/v1/routes/rmon/routes.py @@ -3,7 +3,8 @@ from app.views.agent.region_views import RegionView, RegionListView from app.views.agent.country_views import CountryView, CountryListView from app.views.check.views import CheckHttpView, CheckTcpView, CheckDnsView, CheckPingView, CheckSmtpView, CheckRabbitView -from app.views.check.checks_view import ChecksViewHttp, ChecksViewDns, ChecksViewTcp, ChecksViewPing, ChecksViewSmtp, ChecksViewRabbit +from app.views.check.checks_view import (ChecksViewHttp, ChecksViewDns, ChecksViewTcp, ChecksViewPing, ChecksViewSmtp, + ChecksViewRabbit, AllChecksViewWithFilters) from app.views.check.check_metric_view import (ChecksMetricViewHttp, ChecksMetricViewTcp, ChecksMetricViewDNS, ChecksMetricViewPing, ChecksMetricViewSMTP, ChecksMetricViewRabbitmq, CheckStatusesView, CheckStatusView) @@ -22,6 +23,7 @@ def register_api(view, endpoint, url, pk='check_id', pk_type='int'): bp.add_url_rule('/regions', view_func=RegionListView.as_view('regions')) bp.add_url_rule('/countries', view_func=CountryListView.as_view('countries')) bp.add_url_rule('/checks/http', view_func=ChecksViewHttp.as_view('http_checks')) +bp.add_url_rule('/checks', view_func=AllChecksViewWithFilters.as_view('checks')) bp.add_url_rule('/checks/dns', view_func=ChecksViewDns.as_view('dns_checks')) bp.add_url_rule('/checks/tcp', view_func=ChecksViewTcp.as_view('tcp_checks')) bp.add_url_rule('/checks/ping', view_func=ChecksViewPing.as_view('ping_checks')) diff --git a/app/modules/db/smon.py b/app/modules/db/smon.py index 848f5db..acda92d 100644 --- a/app/modules/db/smon.py +++ b/app/modules/db/smon.py @@ -11,6 +11,7 @@ from app.modules.db.common import out_error, resource_not_empty import app.modules.roxy_wi_tools as roxy_wi_tools import app.modules.tools.common as tool_common +from app.modules.roxywi.class_models import CheckFiltersQuery from app.modules.roxywi.exception import RoxywiResourceNotFound @@ -374,6 +375,11 @@ def select_multi_checks(group_id: int) -> SMON: def select_multi_checks_with_type(check_type: int, group_id: int) -> SMON: try: + if pgsql_enable: + return SMON.select().join(MultiCheck).where( + (SMON.group_id == group_id) & + (SMON.check_type == check_type) + ).distinct(SMON.multi_check_id) return SMON.select().join(MultiCheck).where( (SMON.group_id == group_id) & (SMON.check_type == check_type) @@ -382,6 +388,45 @@ def select_multi_checks_with_type(check_type: int, group_id: int) -> SMON: out_error(e) +def select_multi_check_with_filters(group_id: int, query: CheckFiltersQuery) -> SMON: + where_query = (SMON.group_id == group_id) + sort_query = None + if any((query.check_status, query.check_name, query.check_group, query.check_type)): + if query.check_status: + where_query = where_query & (SMON.status == query.check_status) + if query.check_name: + where_query = where_query & (SMON.name == query.check_name) + if query.check_type: + where_query = where_query & (SMON.check_type == query.check_type) + if query.check_group: + check_group_id = get_smon_group_by_name(group_id, query.check_group) + where_query = where_query & (MultiCheck.check_group_id == check_group_id) + if any((query.sort_by_check_name, query.sort_by_check_status, query.sort_by_check_type)): + if query.sort_by_check_name: + sort_query = SMON.name + elif query.sort_by_check_status: + sort_query = SMON.status + elif query.sort_by_check_type: + sort_query = SMON.check_type + try: + if pgsql_enable: + if sort_query: + query = SMON.select().join(MultiCheck).where(where_query).distinct(SMON.multi_check_id).order_by(SMON.multi_check_id, sort_query).paginate(query.offset, query.limit) + else: + query = SMON.select().join(MultiCheck).where(where_query).distinct(SMON.multi_check_id).paginate(query.offset, query.limit) + return query + else: + if sort_query: + query = SMON.select().join(MultiCheck).where(where_query).group_by(SMON.multi_check_id).order_by(sort_query).paginate(query.offset, query.limit) + else: + query = SMON.select().join(MultiCheck).where(where_query).group_by(SMON.multi_check_id).paginate(query.offset, query.limit) + return query + except SMON.DoesNotExist: + raise RoxywiResourceNotFound + except Exception as e: + out_error(e) + + def select_one_multi_check_join(multi_check_id: int, check_type_id: int) -> SMON: correct_model = tool_common.get_model_for_check(check_type_id=check_type_id) try: diff --git a/app/modules/roxywi/class_models.py b/app/modules/roxywi/class_models.py index d683816..8fe373b 100644 --- a/app/modules/roxywi/class_models.py +++ b/app/modules/roxywi/class_models.py @@ -200,6 +200,18 @@ class GroupQuery(BaseModel): max_depth: Optional[int] = 1 +class CheckFiltersQuery(GroupQuery): + offset: int = 1 + limit: int = 25 + check_group: Optional[EscapedString] = None + check_name: Optional[EscapedString] = None + check_status: Optional[int] = None + check_type: Optional[Literal['http', 'tcp', 'ping', 'dns', 'rabbitmq', 'smtp']] = None + sort_by_check_name: Optional[bool] = False + sort_by_check_status: Optional[bool] = False + sort_by_check_type: Optional[bool] = False + + class UserSearchRequest(GroupQuery): username: Optional[str] = None email: Optional[str] = None diff --git a/app/views/check/checks_view.py b/app/views/check/checks_view.py index 58e75bd..2d8984b 100644 --- a/app/views/check/checks_view.py +++ b/app/views/check/checks_view.py @@ -1,5 +1,3 @@ -from typing import Union - from flask.views import MethodView from flask_jwt_extended import jwt_required from flask import jsonify @@ -10,8 +8,48 @@ import app.modules.tools.smon as smon_mod from app.modules.common.common_classes import SupportClass from app.middleware import get_user_params, check_group -from app.modules.db.db_model import SmonTcpCheck, SmonHttpCheck, SmonDnsCheck, SmonPingCheck, SmonSMTPCheck, SmonRabbitCheck -from app.modules.roxywi.class_models import GroupQuery +from app.modules.roxywi.class_models import GroupQuery, CheckFiltersQuery +from app.modules.db.db_model import SMON + + +def _return_checks(checks: SMON, check_type_id: int = None) -> list: + entities = [] + check_list = [] + + for m in checks: + check_json = {'checks': []} + place = m.multi_check_id.entity_type + check_id = m.id + if m.multi_check_id.check_group_id: + group_name = smon_sql.get_smon_group_by_id(m.multi_check_id.check_group_id).name + group_name = group_name.replace("'", "") + else: + group_name = None + check_json['check_group'] = group_name + if m.country_id: + entities.append(m.country_id.id) + elif m.region_id: + entities.append(m.region_id.id) + elif m.agent_id: + entities.append(m.agent_id.id) + checks = smon_sql.select_one_smon(check_id, check_type_id=check_type_id) + i = 0 + for check in checks: + check_dict = model_to_dict(check, max_depth=1) + check_json['checks'].append(check_dict) + check_json['entities'] = entities + check_json['place'] = place + smon_id = model_to_dict(check, max_depth=1) + check_json.update(smon_id['smon_id']) + check_json.update(model_to_dict(check, recurse=False)) + check_json['name'] = check_json['name'].replace("'", "") + check_json['checks'][i]['smon_id']['name'] = check.smon_id.name.replace("'", "") + if check_json['checks'][i]['smon_id']['check_type'] == 'http': + check_json['checks'][i]['accepted_status_codes'] = int(check_json['checks'][i]['accepted_status_codes']) + check_json['accepted_status_codes'] = int(check_json['accepted_status_codes']) + i += 1 + check_list.append(check_json) + return check_list class ChecksView(MethodView): @@ -21,47 +59,12 @@ class ChecksView(MethodView): def __init__(self): self.check_type = None - def get(self, query: GroupQuery) -> Union[SmonTcpCheck, SmonHttpCheck, SmonDnsCheck, SmonPingCheck]: + def get(self, query: GroupQuery) -> list: group_id = SupportClass.return_group_id(query) check_type_id = smon_mod.get_check_id_by_name(self.check_type) checks = smon_sql.select_multi_checks_with_type(self.check_type, group_id) - entities = [] - check_list = [] - - for m in checks: - check_json = {'checks': []} - place = m.multi_check_id.entity_type - check_id = m.id - if m.multi_check_id.check_group_id: - group_name = smon_sql.get_smon_group_by_id(m.multi_check_id.check_group_id).name - group_name = group_name.replace("'", "") - else: - group_name = None - check_json['check_group'] = group_name - if m.country_id: - entities.append(m.country_id.id) - elif m.region_id: - entities.append(m.region_id.id) - elif m.agent_id: - entities.append(m.agent_id.id) - checks = smon_sql.select_one_smon(check_id, check_type_id=check_type_id) - i = 0 - for check in checks: - check_dict = model_to_dict(check, max_depth=1) - check_json['checks'].append(check_dict) - check_json['entities'] = entities - check_json['place'] = place - smon_id = model_to_dict(check, max_depth=1) - check_json.update(smon_id['smon_id']) - check_json.update(model_to_dict(check, recurse=False)) - check_json['name'] = check_json['name'].replace("'", "") - check_json['checks'][i]['smon_id']['name'] = check.smon_id.name.replace("'", "") - if check_json['checks'][i]['smon_id']['check_type'] == 'http': - check_json['checks'][i]['accepted_status_codes'] = int(check_json['checks'][i]['accepted_status_codes']) - check_json['accepted_status_codes'] = int(check_json['accepted_status_codes']) - i += 1 - check_list.append(check_json) + check_list = _return_checks(checks, check_type_id) return check_list @@ -943,3 +946,176 @@ def get(self, query: GroupQuery): """ checks = super().get(query) return jsonify(checks) + + +class AllChecksViewWithFilters(MethodView): + methods = ["GET"] + decorators = [jwt_required(), get_user_params(), check_group()] + + @validate(query=CheckFiltersQuery) + def get(self, query: CheckFiltersQuery): + """ + Get all checks with filters. + + --- + tags: + - 'Checks' + parameters: + - name: check_group + in: query + description: 'Filter by check group.' + required: false + type: string + - name: check_name + in: query + description: 'Filter by check name.' + required: false + type: string + - name: check_status + in: query + description: 'Filter by check status.' + required: false + type: integer + - name: check_type + in: query + description: 'Filter by check type. Available values: `http`, `tcp`, `ping`, `dns`, `rabbitmq`, `smtp`.' + required: false + type: string + - name: sort_by_check_name + in: query + description: 'Sort checks by check name.' + required: false + type: bool + - name: sort_by_check_status + in: query + description: 'Sort checks by check status.' + required: false + type: bool + - name: sort_by_check_type + in: query + description: 'Sort checks by check type.' + required: false + type: bool + - name: offset + in: query + type: integer + description: 'Offset for pagination.' + default: 1 + - name: limit + in: query + type: integer + description: 'Limit for pagination.' + default: 25 + responses: + '200': + description: 'Successful Operation' + schema: + type: array + items: + type: object + properties: + agent_id: + type: integer + description: 'ID of the agent.' + body_status: + type: integer + description: 'Status of the body content.' + check_group: + type: string + description: 'Name of the check group.' + check_timeout: + type: integer + description: 'Timeout interval of the check.' + check_type: + type: string + description: 'Type of the check.' + country_id: + type: integer + description: 'ID of the country.' + created_at: + type: string + format: date-time + description: 'Creation time of the check.' + description: + type: string + description: 'Description of the check.' + enabled: + type: integer + description: 'Enabled status of the check.' + group_id: + type: integer + description: 'ID of the group.' + id: + type: integer + description: 'ID of the check.' + mm_channel_id: + type: integer + description: 'MM Channel ID.' + multi_check_id: + type: integer + description: 'Multi-check ID.' + name: + type: string + description: 'Name of the check.' + pd_channel_id: + type: integer + description: 'PD Channel ID.' + place: + type: string + description: 'Place related to the check.' + region_id: + type: integer + description: 'ID of the region.' + response_time: + type: string + description: 'Response time of the check.' + slack_channel_id: + type: integer + description: 'Slack Channel ID.' + ssl_expire_critical_alert: + type: integer + description: 'Critical alert for SSL expiry.' + ssl_expire_date: + type: string + format: date-time + description: 'SSL expiry date.' + ssl_expire_warning_alert: + type: integer + description: 'Warning alert for SSL expiry.' + status: + type: integer + description: 'Status of the check.' + telegram_channel_id: + type: integer + description: 'Telegram Channel ID.' + time_state: + type: string + format: date-time + description: 'Time state of the check.' + updated_at: + type: string + format: date-time + description: 'Last updated time of the check.' + """ + group_id = SupportClass.return_group_id(query) + checks = smon_sql.select_multi_check_with_filters(group_id, query) + entities = [] + check_list = [] + + for m in checks: + check_json = {} + place = m.multi_check_id.entity_type + if m.multi_check_id.check_group_id: + group_name = smon_sql.get_smon_group_by_id(m.multi_check_id.check_group_id).name + group_name = group_name.replace("'", "") + else: + group_name = None + + check_json['check_group'] = group_name + check_json['entities'] = entities + check_json['place'] = place + smon_id = model_to_dict(m, recurse=False, exclude={SMON.agent_id, SMON.region_id, SMON.country_id}) + check_json.update(smon_id) + check_json['name'] = check_json['name'].replace("'", "") + check_list.append(check_json) + return jsonify(check_list) diff --git a/app/views/check/views.py b/app/views/check/views.py index 8590264..0d662af 100644 --- a/app/views/check/views.py +++ b/app/views/check/views.py @@ -443,7 +443,7 @@ def post(self, body: HttpCheckRequest) -> Union[dict, tuple]: - name - enabled - url - - http_method + - method - place - entities properties: @@ -486,7 +486,7 @@ def post(self, body: HttpCheckRequest) -> Union[dict, tuple]: mm_channel_id: type: 'integer' description: 'Mattermost channel ID (optional)' - http_method: + method: type: 'string' description: 'HTTP method' enum: ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'] @@ -587,7 +587,7 @@ def put(self, check_id: int, body: HttpCheckRequest) -> Union[dict, tuple]: mm_channel_id: type: 'integer' description: 'Mattermost channel ID (optional)' - http_method: + method: type: 'string' description: 'HTTP method' enum: ['get', 'post', 'put', 'patch', 'delete', 'head', 'options']