From e2843082c6bb2c31a8c75e5e4b6356541c07e5d0 Mon Sep 17 00:00:00 2001 From: m-o-d-e-r Date: Tue, 22 Aug 2023 22:30:58 +0300 Subject: [PATCH 1/2] try to implement new preprocessor --- burrito/__main__.py | 2 +- burrito/models/basic_model.py | 1 + burrito/preprocessor/Dockerfile | 31 +++++++++ burrito/preprocessor/__init__.py | 0 burrito/preprocessor/__main__.py | 0 burrito/preprocessor/conf.py | 114 +++++++++++++++++++++++++++++++ burrito/preprocessor/core.py | 109 +++++++++++++++++++++++++++++ burrito/utils/app_util.py | 4 ++ docker-compose.yml | 26 ------- 9 files changed, 260 insertions(+), 27 deletions(-) create mode 100644 burrito/preprocessor/Dockerfile create mode 100644 burrito/preprocessor/__init__.py create mode 100644 burrito/preprocessor/__main__.py create mode 100644 burrito/preprocessor/conf.py create mode 100644 burrito/preprocessor/core.py diff --git a/burrito/__main__.py b/burrito/__main__.py index 8ffbf515..0c8c29b8 100644 --- a/burrito/__main__.py +++ b/burrito/__main__.py @@ -18,7 +18,7 @@ error_attempt_delta=3 ) init_manager.add_task(CheckDBTask(attempt_count=100)) -init_manager.add_task(PreProcessorTask(attempt_count=100)) +#init_manager.add_task(PreProcessorTask(attempt_count=100)) init_manager.run_cycle() diff --git a/burrito/models/basic_model.py b/burrito/models/basic_model.py index 1a1dcc8d..403da9c3 100644 --- a/burrito/models/basic_model.py +++ b/burrito/models/basic_model.py @@ -7,3 +7,4 @@ class BurritoBasicModel(Model): class Meta: database = get_database_cursor() legacy_table_names = False + table_settings = ['ENGINE=InnoDB', 'DEFAULT CHARSET=utf8'] diff --git a/burrito/preprocessor/Dockerfile b/burrito/preprocessor/Dockerfile new file mode 100644 index 00000000..17b7d1bb --- /dev/null +++ b/burrito/preprocessor/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.10-slim-buster as burrito-build-base + +ENV POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=true \ + PATH="/opt/pysetup/.venv/bin:$PATH" + +RUN apt-get update +RUN apt-get install --no-install-recommends -y build-essential + +WORKDIR /opt/pysetup + +RUN pip3 install poetry + +COPY pyproject.toml ./ + +RUN poetry install --only main + + +FROM python:3.10-slim-buster + +ENV PATH="/opt/pysetup/.venv/bin:$PATH" + +COPY --from=burrito-build-base /opt/pysetup/ /opt/pysetup/ +COPY ./preprocessor_config.json /preprocessor_config.json +COPY ./CONTRIBUTORS.md /CONTRIBUTORS.md +COPY ./CHANGELOG.md /CHANGELOG.md +COPY ./burrito /burrito + +WORKDIR ./ + +#CMD ["python", "-m", "burrito.apps.tickets"] \ No newline at end of file diff --git a/burrito/preprocessor/__init__.py b/burrito/preprocessor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/burrito/preprocessor/__main__.py b/burrito/preprocessor/__main__.py new file mode 100644 index 00000000..e69de29b diff --git a/burrito/preprocessor/conf.py b/burrito/preprocessor/conf.py new file mode 100644 index 00000000..cc3ea321 --- /dev/null +++ b/burrito/preprocessor/conf.py @@ -0,0 +1,114 @@ +from burrito.models.group_model import Groups +from burrito.models.statuses_model import Statuses +from burrito.models.faculty_model import Faculties +from burrito.models.queues_model import Queues +from burrito.models.permissions_model import Permissions +from burrito.models.roles_model import Roles +from burrito.models.role_permissions_model import RolePermissions + + +MODEL_KEYS = { + "groups": Groups, + "faculties": Faculties, + "statuses": Statuses, + "queues": Queues, + "permissions": Permissions, + "roles": Roles, + "role_permissions": RolePermissions +} + +DEFAULT_CONFIG = { + "__tables_option": { + "groups": "SELECT * FROM `groups`;", + "faculties": "SELECT * FROM `faculties`;", + "statuses": "SELECT * FROM `statuses`;", + "queues": "SELECT * FROM `queues`;", + "permissions": "SELECT * FROM `permissions`;", + "roles": "SELECT * FROM `roles`;", + "role_permissions": "SELECT * FROM `role_permissions`;" + }, + "groups": [], # will be updated automatic + "faculties": [], # will be updated automatic + "statuses": [ + {"status_id": 1, "name": "NEW"}, + {"status_id": 2, "name": "ACCEPTED"}, + {"status_id": 3, "name": "OPEN"}, + {"status_id": 4, "name": "WAITING"}, + {"status_id": 5, "name": "REJECTED"}, + {"status_id": 6, "name": "CLOSE"} + ], + "queues": [ + {"queue_id": 1, "name": "Lecturers", "faculty_id": 1, "scope": "Reports"}, + {"queue_id": 2, "name": "Food", "faculty_id": 1, "scope": "Reports"}, + {"queue_id": 3, "name": "Academic integrity", "faculty_id": 2, "scope": "Reports"}, + {"queue_id": 4, "name": "Scholarship", "faculty_id": 2, "scope": "Q/A"}, + {"queue_id": 5, "name": "Scholarship", "faculty_id": 1, "scope": "Q/A"}, + {"queue_id": 6, "name": "Food", "faculty_id": 2, "scope": "Q/A"}, + {"queue_id": 7, "name": "Dormitory", "faculty_id": 1, "scope": "Q/A"}, + {"queue_id": 8, "name": "Dormitory", "faculty_id": 2, "scope": "Q/A"}, + {"queue_id": 9, "name": "Dormitory", "faculty_id": 1, "scope": "Reports"}, + {"queue_id": 10, "name": "Dormitory", "faculty_id": 2, "scope": "Reports"} + ], + "permissions": [ + {"permission_id": 1, "name": "UPDATE_PROFILE"}, + {"permission_id": 2, "name": "CREATE_TICKET"}, + {"permission_id": 3, "name": "READ_TICKET"}, + {"permission_id": 4, "name": "SEND_MESSAGE"}, + {"permission_id": 5, "name": "ADMIN"}, + {"permission_id": 6, "name": "GOD_MODE"} + ], + "roles": [ + {"role_id": 1, "name": "USER_ALL"}, + {"role_id": 2, "name": "USER_NO_M"}, + {"role_id": 3, "name": "USER_NO_CT"}, + {"role_id": 4, "name": "USER_NO_P"}, + {"role_id": 5, "name": "USER_NO_PCT"}, + {"role_id": 6, "name": "USER_NO_CTM"}, + {"role_id": 7, "name": "USER_NO_PM"}, + {"role_id": 8, "name": "USER_NO_PCTM"}, + {"role_id": 9, "name": "ADMIN"}, + {"role_id": 10, "name": "CHIEF_ADMIN"} + ], + "role_permissions": [ + {"role_id": 1, "permission_id": 1}, + {"role_id": 1, "permission_id": 2}, + {"role_id": 1, "permission_id": 3}, + {"role_id": 1, "permission_id": 4}, + + {"role_id": 2, "permission_id": 1}, + {"role_id": 2, "permission_id": 2}, + {"role_id": 2, "permission_id": 3}, + + {"role_id": 3, "permission_id": 1}, + {"role_id": 3, "permission_id": 3}, + {"role_id": 3, "permission_id": 4}, + + {"role_id": 4, "permission_id": 2}, + {"role_id": 4, "permission_id": 3}, + {"role_id": 4, "permission_id": 4}, + + {"role_id": 5, "permission_id": 3}, + {"role_id": 5, "permission_id": 4}, + + {"role_id": 6, "permission_id": 1}, + {"role_id": 6, "permission_id": 3}, + + {"role_id": 7, "permission_id": 2}, + {"role_id": 7, "permission_id": 3}, + + {"role_id": 8, "permission_id": 3}, + + {"role_id": 9, "permission_id": 1}, + {"role_id": 9, "permission_id": 2}, + {"role_id": 9, "permission_id": 3}, + {"role_id": 9, "permission_id": 4}, + {"role_id": 9, "permission_id": 5}, + + {"role_id": 10, "permission_id": 1}, + {"role_id": 10, "permission_id": 2}, + {"role_id": 10, "permission_id": 3}, + {"role_id": 10, "permission_id": 4}, + {"role_id": 10, "permission_id": 5}, + {"role_id": 10, "permission_id": 6} + ] +} diff --git a/burrito/preprocessor/core.py b/burrito/preprocessor/core.py new file mode 100644 index 00000000..4f0aef58 --- /dev/null +++ b/burrito/preprocessor/core.py @@ -0,0 +1,109 @@ +import pymysql.cursors +import requests + + +from burrito.utils.config_reader import get_config +from burrito.utils.logger import get_logger + +from burrito.preprocessor.conf import MODEL_KEYS, DEFAULT_CONFIG + + +SSU_KEY = get_config().BURRITO_SSU_KEY +SSU_GROUPS_URL = get_config().BURRITO_SSU_GROUPS_URL +SSU_FACULTIES_URL = get_config().BURRITO_SSU_FACULTIES_URL + + +async def pull_faculties_from_ssu() -> list: + try: + raw_faculties_data = requests.get( + f"{SSU_FACULTIES_URL}?key={SSU_KEY}", + timeout=30 + ) + if raw_faculties_data.status_code != 200: + get_logger().warning( + f"{SSU_FACULTIES_URL} status code is {raw_faculties_data.status_code}" + ) + raw_faculties_data = raw_faculties_data.json()["result"] + + return [ + { + "faculty_id": raw_group["ID_DIV"], + "name": raw_group["ABBR_DIV"] + } for raw_group in raw_faculties_data if ( + raw_group["ID_DIV"] and raw_group["ABBR_DIV"] and raw_group["KOD_TYPE"] in (7, 9) + ) + ] + except Exception as e: + get_logger().error(e) + + return [] + + +async def pull_groups_from_ssu() -> list: + try: + raw_groups_data = requests.get( + f"{SSU_GROUPS_URL}?key={SSU_KEY}", + timeout=30 + ) + if raw_groups_data.status_code != 200: + get_logger().warning( + f"{SSU_GROUPS_URL} status code is {raw_groups_data.status_code}" + ) + raw_groups_data = raw_groups_data.json()["result"] + + return [ + { + "group_id": raw_group["ID_GROUP"], + "name": raw_group["NAME_GROUP"] + } for raw_group in raw_groups_data if raw_group["ID_GROUP"] and raw_group["NAME_GROUP"] + ] + except Exception as e: + get_logger().error(e) + + return [] + + +async def preprocessor_task(): + get_logger().info("Preprocessor is started") + + conn = pymysql.connect( + database=get_config().BURRITO_DB_NAME, + user=get_config().BURRITO_DB_USER, + password=get_config().BURRITO_DB_PASSWORD, + host=get_config().BURRITO_DB_HOST, + port=int(get_config().BURRITO_DB_PORT), + cursorclass=pymysql.cursors.DictCursor + ) + + with conn: + __sql_commands: dict = {} + __config_data: dict = {} + data: dict = DEFAULT_CONFIG + + data["groups"] = await pull_groups_from_ssu() + data["faculties"] = await pull_faculties_from_ssu() + + for key, value in data.items(): + if not key.startswith("__"): + __config_data[key] = value + continue + + if key == "__tables_option": + __sql_commands = value + + for table, config_values in __config_data.items(): + with conn.cursor() as cursor: + cursor.execute(__sql_commands[table]) + conn.commit() + + config_filtered_values: set = {tuple(i.values()) for i in config_values} + db_filtered_values: set = {tuple(i.values()) for i in cursor.fetchall()} + + if config_filtered_values.difference(db_filtered_values): + for value in config_values: + try: + MODEL_KEYS[table].create(**value) + except: + ... + + get_logger().info("Preprocessor is finished") diff --git a/burrito/utils/app_util.py b/burrito/utils/app_util.py index d11f159e..fc998ba7 100644 --- a/burrito/utils/app_util.py +++ b/burrito/utils/app_util.py @@ -5,6 +5,8 @@ # from burrito.middlewares.user_agent import UserAgentMiddleware +from burrito.preprocessor.core import preprocessor_task + from .singleton_pattern import singleton from .task_manager import get_async_manager from .logger import get_logger @@ -51,6 +53,8 @@ async def startup_event(): """ task_manager = get_async_manager() + task_manager.add_task(preprocessor_task()) + # task_manager.add_task(get_pubsub_manager().run()) def test1(): diff --git a/docker-compose.yml b/docker-compose.yml index 53287f36..7d44fc04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,16 +10,6 @@ services: - "8080:8080" links: - "db" -# deploy: -# mode: replicated -# replicas: 5 -# resources: -# limits: -# cpus: '0.70' -# memory: 500M -# reservations: -# cpus: '0.25' -# memory: 20M db: container_name: burrito_db @@ -42,19 +32,3 @@ services: image: mongo:4.4.6 ports: - "27017:27017" - -# prometheus: -# container_name: burrito_prometheus -# image: prom/prometheus -# ports: -# - 9090:9090 -# volumes: -# - ./prometheus.yml:/etc/prometheus/prometheus.yml -# command: -# - '--config.file=/etc/prometheus/prometheus.yml' -# -# grafana: -# container_name: burrito_grafana -# image: grafana/grafana -# ports: -# - 3000:3000 From 0e940092730a13e98d7879ce89420c6445c8b7b5 Mon Sep 17 00:00:00 2001 From: m-o-d-e-r Date: Sun, 27 Aug 2023 00:04:02 +0300 Subject: [PATCH 2/2] version 0.8.0 & created scheduler --- Makefile | 3 + README.md | 12 ++- burrito/__init__.py | 2 +- burrito/__main__.py | 52 ++++-------- burrito/apps/about/__main__.py | 8 +- burrito/apps/admin/__main__.py | 8 +- burrito/apps/anon/__main__.py | 8 +- burrito/apps/auth/__main__.py | 8 +- burrito/apps/comments/__main__.py | 8 +- burrito/apps/iofiles/__main__.py | 8 +- burrito/apps/meta/__main__.py | 8 +- burrito/apps/notifications/__main__.py | 8 +- burrito/apps/profile/__main__.py | 8 +- burrito/apps/registration/__main__.py | 8 +- .../scheduler}/Dockerfile | 2 +- burrito/{init => apps/scheduler}/__init__.py | 0 burrito/apps/scheduler/__main__.py | 4 + burrito/apps/scheduler/core.py | 12 +++ .../scheduler}/preprocessor/__init__.py | 0 .../scheduler}/preprocessor/__main__.py | 0 .../{ => apps/scheduler}/preprocessor/conf.py | 15 ++-- .../{ => apps/scheduler}/preprocessor/core.py | 59 ++++++++------ burrito/apps/tickets/__main__.py | 8 +- burrito/containers.py | 20 ----- burrito/init/init_system.py | 46 ----------- burrito/init/init_task.py | 36 --------- burrito/init/tasks/check_db_task.py | 38 --------- burrito/init/tasks/preprocessor_task.py | 80 ------------------- burrito/utils/app_util.py | 28 +------ burrito/utils/db_utils.py | 9 ++- burrito/utils/file_manager.py | 4 +- burrito/utils/logger.py | 2 +- burrito/utils/query_util.py | 17 +--- burrito/utils/task_manager.py | 70 ++++++++-------- docker-compose.yml => docker-compose-dbs.yml | 14 +--- docker-compose-separated.yml | 67 ++-------------- preprocessor_config.json | 13 +-- pyproject.toml | 4 +- scripts/burrito_cluster_run.sh | 2 +- scripts/prepare_db.py | 6 ++ tests/meta_test.py | 2 +- tests/profile_test.py | 4 +- tests/registration_test.py | 7 +- tests/run_tests.py | 2 +- tests/tickets_test.py | 10 +-- 45 files changed, 195 insertions(+), 535 deletions(-) rename burrito/{preprocessor => apps/scheduler}/Dockerfile (93%) rename burrito/{init => apps/scheduler}/__init__.py (100%) create mode 100644 burrito/apps/scheduler/__main__.py create mode 100644 burrito/apps/scheduler/core.py rename burrito/{ => apps/scheduler}/preprocessor/__init__.py (100%) rename burrito/{ => apps/scheduler}/preprocessor/__main__.py (100%) rename burrito/{ => apps/scheduler}/preprocessor/conf.py (82%) rename burrito/{ => apps/scheduler}/preprocessor/core.py (63%) delete mode 100644 burrito/init/init_system.py delete mode 100644 burrito/init/init_task.py delete mode 100644 burrito/init/tasks/check_db_task.py delete mode 100644 burrito/init/tasks/preprocessor_task.py rename docker-compose.yml => docker-compose-dbs.yml (71%) create mode 100644 scripts/prepare_db.py diff --git a/Makefile b/Makefile index b31b897f..123d4f21 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,6 @@ burrito_cluster_ps: changelog: scripts/generate_changelog.sh + +prepare_db: + $(PYTHON) scripts/prepare_db.py diff --git a/README.md b/README.md index 53bd5c64..3429a057 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,13 @@ touch .env - `MYSQL_USER` - `MYSQL_PASSWORD` - `burrito_redis` - contains access/refresh tokens (no variables are needed) +- Build Burrito API +```bash +docker build . -t burrito +``` - Launch Burrito API ```bash -docker-compose up +docker run --rm burrito ``` - Run tests ```bash @@ -85,9 +89,13 @@ touch .env - `burrito_db` - contain MySQL database (you can find more information about MySql docker container and its environment variables [here](https://hub.docker.com/_/mysql) - `burrito_nginx` - proxy-server and load balancer for `burrito cluster` (no variables are needed) - `burrito_redis` - contains access/refresh tokens (no variables are needed) +- Launch needed databases (if you would not setup own) in docker-compose +```bash +docker-compose -f docker-compose-dbs.yml up +``` - Launch Burrito API ```bash -make burrito_cluster_run +docker-compose -f docker-compose-separated.yml up ``` - Run tests ```bash diff --git a/burrito/__init__.py b/burrito/__init__.py index ec9883ac..3d813712 100644 --- a/burrito/__init__.py +++ b/burrito/__init__.py @@ -1 +1 @@ -__version__ = "0.7.2 indev" +__version__ = "0.8.0 indev" diff --git a/burrito/__main__.py b/burrito/__main__.py index 0c8c29b8..eb9ddbf0 100644 --- a/burrito/__main__.py +++ b/burrito/__main__.py @@ -5,49 +5,27 @@ """ -from burrito.utils.config_reader import get_config - -from burrito.init.init_system import InitManager, get_logger -from burrito.init.tasks.check_db_task import CheckDBTask -from burrito.init.tasks.preprocessor_task import PreProcessorTask - - -get_config() # read configs - -init_manager = InitManager( - error_attempt_delta=3 -) -init_manager.add_task(CheckDBTask(attempt_count=100)) -#init_manager.add_task(PreProcessorTask(attempt_count=100)) +import uvicorn -init_manager.run_cycle() +from burrito.apps.registration.router import registration_router +from burrito.apps.about.router import about_router +from burrito.apps.auth.router import auth_router +from burrito.apps.profile.router import profile_router -if not init_manager.critical: - import uvicorn +from burrito.apps.tickets.router import tickets_router +from burrito.apps.admin.router import admin_router - from burrito.apps.registration.router import registration_router - from burrito.apps.about.router import about_router +from burrito.apps.anon.router import anon_router +from burrito.apps.meta.router import meta_router - from burrito.apps.auth.router import auth_router - from burrito.apps.profile.router import profile_router +from burrito.apps.iofiles.router import iofiles_router +from burrito.apps.comments.router import comments_router - from burrito.apps.tickets.router import tickets_router - from burrito.apps.admin.router import admin_router +from burrito.apps.notifications.router import notifications_router - from burrito.apps.anon.router import anon_router - from burrito.apps.meta.router import meta_router - - from burrito.apps.iofiles.router import iofiles_router - from burrito.apps.comments.router import comments_router - - from burrito.apps.notifications.router import notifications_router - - from burrito.utils.app_util import connect_app, get_current_app -else: - print() - get_logger().critical("Some critical error was ocurred before") - exit(1) +from burrito.utils.app_util import connect_app, get_current_app +from burrito.utils.config_reader import get_config app = get_current_app() @@ -71,6 +49,4 @@ host="0.0.0.0", port=int(get_config().BURRITO_PORT), proxy_headers=bool(get_config().BURRITO_PROXY_HEADERS), - reload=True, - reload_dirs="burrito" ) diff --git a/burrito/apps/about/__main__.py b/burrito/apps/about/__main__.py index f68d50cd..9b891353 100644 --- a/burrito/apps/about/__main__.py +++ b/burrito/apps/about/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.about.router import about_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.about.router import about_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/admin/__main__.py b/burrito/apps/admin/__main__.py index 42ef6e90..90d5af54 100644 --- a/burrito/apps/admin/__main__.py +++ b/burrito/apps/admin/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.admin.router import admin_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.admin.router import admin_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/anon/__main__.py b/burrito/apps/anon/__main__.py index cd539e79..158bf89d 100644 --- a/burrito/apps/anon/__main__.py +++ b/burrito/apps/anon/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.anon.router import anon_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.anon.router import anon_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/auth/__main__.py b/burrito/apps/auth/__main__.py index 8d9b2270..ff3d4823 100644 --- a/burrito/apps/auth/__main__.py +++ b/burrito/apps/auth/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.auth.router import auth_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.auth.router import auth_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/comments/__main__.py b/burrito/apps/comments/__main__.py index d0de461c..3b3fd673 100644 --- a/burrito/apps/comments/__main__.py +++ b/burrito/apps/comments/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.comments.router import comments_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.comments.router import comments_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/iofiles/__main__.py b/burrito/apps/iofiles/__main__.py index 1b46f4f8..531c0d26 100644 --- a/burrito/apps/iofiles/__main__.py +++ b/burrito/apps/iofiles/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.iofiles.router import iofiles_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.iofiles.router import iofiles_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/meta/__main__.py b/burrito/apps/meta/__main__.py index f507108d..52d80afa 100644 --- a/burrito/apps/meta/__main__.py +++ b/burrito/apps/meta/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.meta.router import meta_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.meta.router import meta_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/notifications/__main__.py b/burrito/apps/notifications/__main__.py index fa82e6e2..6f448659 100644 --- a/burrito/apps/notifications/__main__.py +++ b/burrito/apps/notifications/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.notifications.router import notifications_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.notifications.router import notifications_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/profile/__main__.py b/burrito/apps/profile/__main__.py index 5f26782f..21ba002f 100644 --- a/burrito/apps/profile/__main__.py +++ b/burrito/apps/profile/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.profile.router import profile_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.profile.router import profile_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/apps/registration/__main__.py b/burrito/apps/registration/__main__.py index abb4da62..404b8bff 100644 --- a/burrito/apps/registration/__main__.py +++ b/burrito/apps/registration/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.registration.router import registration_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.registration.router import registration_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/preprocessor/Dockerfile b/burrito/apps/scheduler/Dockerfile similarity index 93% rename from burrito/preprocessor/Dockerfile rename to burrito/apps/scheduler/Dockerfile index 17b7d1bb..ee076c9f 100644 --- a/burrito/preprocessor/Dockerfile +++ b/burrito/apps/scheduler/Dockerfile @@ -28,4 +28,4 @@ COPY ./burrito /burrito WORKDIR ./ -#CMD ["python", "-m", "burrito.apps.tickets"] \ No newline at end of file +CMD ["python", "-m", "burrito.apps.scheduler"] \ No newline at end of file diff --git a/burrito/init/__init__.py b/burrito/apps/scheduler/__init__.py similarity index 100% rename from burrito/init/__init__.py rename to burrito/apps/scheduler/__init__.py diff --git a/burrito/apps/scheduler/__main__.py b/burrito/apps/scheduler/__main__.py new file mode 100644 index 00000000..ef6eafd2 --- /dev/null +++ b/burrito/apps/scheduler/__main__.py @@ -0,0 +1,4 @@ +from .core import start_scheduler + + +start_scheduler() diff --git a/burrito/apps/scheduler/core.py b/burrito/apps/scheduler/core.py new file mode 100644 index 00000000..8588b27c --- /dev/null +++ b/burrito/apps/scheduler/core.py @@ -0,0 +1,12 @@ +import schedule +import time + +from .preprocessor.core import preprocessor_task + + +def start_scheduler(): + schedule.every().day.at("00:30").do(preprocessor_task) + + while True: + schedule.run_pending() + time.sleep(1) diff --git a/burrito/preprocessor/__init__.py b/burrito/apps/scheduler/preprocessor/__init__.py similarity index 100% rename from burrito/preprocessor/__init__.py rename to burrito/apps/scheduler/preprocessor/__init__.py diff --git a/burrito/preprocessor/__main__.py b/burrito/apps/scheduler/preprocessor/__main__.py similarity index 100% rename from burrito/preprocessor/__main__.py rename to burrito/apps/scheduler/preprocessor/__main__.py diff --git a/burrito/preprocessor/conf.py b/burrito/apps/scheduler/preprocessor/conf.py similarity index 82% rename from burrito/preprocessor/conf.py rename to burrito/apps/scheduler/preprocessor/conf.py index cc3ea321..59442071 100644 --- a/burrito/preprocessor/conf.py +++ b/burrito/apps/scheduler/preprocessor/conf.py @@ -38,16 +38,11 @@ {"status_id": 6, "name": "CLOSE"} ], "queues": [ - {"queue_id": 1, "name": "Lecturers", "faculty_id": 1, "scope": "Reports"}, - {"queue_id": 2, "name": "Food", "faculty_id": 1, "scope": "Reports"}, - {"queue_id": 3, "name": "Academic integrity", "faculty_id": 2, "scope": "Reports"}, - {"queue_id": 4, "name": "Scholarship", "faculty_id": 2, "scope": "Q/A"}, - {"queue_id": 5, "name": "Scholarship", "faculty_id": 1, "scope": "Q/A"}, - {"queue_id": 6, "name": "Food", "faculty_id": 2, "scope": "Q/A"}, - {"queue_id": 7, "name": "Dormitory", "faculty_id": 1, "scope": "Q/A"}, - {"queue_id": 8, "name": "Dormitory", "faculty_id": 2, "scope": "Q/A"}, - {"queue_id": 9, "name": "Dormitory", "faculty_id": 1, "scope": "Reports"}, - {"queue_id": 10, "name": "Dormitory", "faculty_id": 2, "scope": "Reports"} + {"queue_id": 1, "name": "Lecturers", "faculty_id": 414, "scope": "Reports"}, + {"queue_id": 2, "name": "Food", "faculty_id": 414, "scope": "Reports"}, + {"queue_id": 3, "name": "Scholarship", "faculty_id": 414, "scope": "Q/A"}, + {"queue_id": 4, "name": "Dormitory", "faculty_id": 414, "scope": "Q/A"}, + {"queue_id": 5, "name": "Dormitory", "faculty_id": 414, "scope": "Reports"}, ], "permissions": [ {"permission_id": 1, "name": "UPDATE_PROFILE"}, diff --git a/burrito/preprocessor/core.py b/burrito/apps/scheduler/preprocessor/core.py similarity index 63% rename from burrito/preprocessor/core.py rename to burrito/apps/scheduler/preprocessor/core.py index 4f0aef58..f0d72558 100644 --- a/burrito/preprocessor/core.py +++ b/burrito/apps/scheduler/preprocessor/core.py @@ -1,11 +1,11 @@ -import pymysql.cursors import requests +import pymysql.cursors - +from burrito.utils.task_manager import get_task_manager from burrito.utils.config_reader import get_config from burrito.utils.logger import get_logger - -from burrito.preprocessor.conf import MODEL_KEYS, DEFAULT_CONFIG +from burrito.utils.db_cursor_object import get_database_cursor +from .conf import MODEL_KEYS, DEFAULT_CONFIG SSU_KEY = get_config().BURRITO_SSU_KEY @@ -13,7 +13,7 @@ SSU_FACULTIES_URL = get_config().BURRITO_SSU_FACULTIES_URL -async def pull_faculties_from_ssu() -> list: +def pull_faculties_from_ssu() -> list: try: raw_faculties_data = requests.get( f"{SSU_FACULTIES_URL}?key={SSU_KEY}", @@ -39,7 +39,7 @@ async def pull_faculties_from_ssu() -> list: return [] -async def pull_groups_from_ssu() -> list: +def pull_groups_from_ssu() -> list: try: raw_groups_data = requests.get( f"{SSU_GROUPS_URL}?key={SSU_KEY}", @@ -63,25 +63,31 @@ async def pull_groups_from_ssu() -> list: return [] -async def preprocessor_task(): +def preprocessor_task(): get_logger().info("Preprocessor is started") - conn = pymysql.connect( - database=get_config().BURRITO_DB_NAME, - user=get_config().BURRITO_DB_USER, - password=get_config().BURRITO_DB_PASSWORD, - host=get_config().BURRITO_DB_HOST, - port=int(get_config().BURRITO_DB_PORT), - cursorclass=pymysql.cursors.DictCursor - ) + conn = None + + try: + conn = pymysql.connect( + database=get_config().BURRITO_DB_NAME, + user=get_config().BURRITO_DB_USER, + password=get_config().BURRITO_DB_PASSWORD, + host=get_config().BURRITO_DB_HOST, + port=int(get_config().BURRITO_DB_PORT), + cursorclass=pymysql.cursors.DictCursor + ) + except Exception as e: + get_logger().warning(e) + return with conn: __sql_commands: dict = {} __config_data: dict = {} - data: dict = DEFAULT_CONFIG - data["groups"] = await pull_groups_from_ssu() - data["faculties"] = await pull_faculties_from_ssu() + data: dict = DEFAULT_CONFIG + data["groups"] = pull_groups_from_ssu() + data["faculties"] = pull_faculties_from_ssu() for key, value in data.items(): if not key.startswith("__"): @@ -102,8 +108,15 @@ async def preprocessor_task(): if config_filtered_values.difference(db_filtered_values): for value in config_values: try: - MODEL_KEYS[table].create(**value) - except: - ... - - get_logger().info("Preprocessor is finished") + if table in ("groups", "faculties"): + get_task_manager().add_task(MODEL_KEYS[table].create, **value) + else: + if table == "queues": + get_database_cursor().execute_sql("SET FOREIGN_KEY_CHECKS=0") + MODEL_KEYS[table].create(**value) + if table == "queues": + get_database_cursor().execute_sql("SET FOREIGN_KEY_CHECKS=1") + except Exception as e: + get_logger().warning(f"Preprocessor error: {e}") + + get_logger().info("Preprocessor sub-tasks pushed to task manager") diff --git a/burrito/apps/tickets/__main__.py b/burrito/apps/tickets/__main__.py index 8ecfea5f..fe514619 100644 --- a/burrito/apps/tickets/__main__.py +++ b/burrito/apps/tickets/__main__.py @@ -1,15 +1,11 @@ import uvicorn +from burrito.containers import get_current_app_name +from burrito.apps.tickets.router import tickets_router from burrito.utils.config_reader import get_config -from burrito.containers import prepare_app, get_current_app_name from burrito.utils.app_util import get_current_app, connect_app -if prepare_app(): - from burrito.apps.tickets.router import tickets_router -else: - print("App preparation failed") - _APP_NAME = get_current_app_name() app = get_current_app(docs_url=f"/{_APP_NAME}/", openapi_url=f"/{_APP_NAME}/openapi.json") diff --git a/burrito/containers.py b/burrito/containers.py index 053cb9a8..fc2d3173 100644 --- a/burrito/containers.py +++ b/burrito/containers.py @@ -1,26 +1,6 @@ from pathlib import Path import inspect -from burrito.utils.config_reader import get_config - -from burrito.init.init_system import InitManager -from burrito.init.tasks.check_db_task import CheckDBTask -from burrito.init.tasks.preprocessor_task import PreProcessorTask - def get_current_app_name() -> str: return Path(inspect.getouterframes(inspect.currentframe(), 2)[1][1]).parent.name - - -def prepare_app(): - get_config() # read configs - - init_manager = InitManager( - error_attempt_delta=3 - ) - init_manager.add_task(CheckDBTask(attempt_count=100)) - init_manager.add_task(PreProcessorTask(attempt_count=100)) - - init_manager.run_cycle() - - return True diff --git a/burrito/init/init_system.py b/burrito/init/init_system.py deleted file mode 100644 index 284d17f5..00000000 --- a/burrito/init/init_system.py +++ /dev/null @@ -1,46 +0,0 @@ -from queue import Queue -from time import sleep as init_sleep - -from burrito.init.init_task import InitTask - -from burrito.utils.logger import get_logger - - -class InitManager: - def __init__(self, error_attempt_delta: int = 3) -> None: - self.__init_tasks: Queue = Queue() - self.__error_attempt_delta = error_attempt_delta - self.__error_count = 0 - self.__critical_error_count = 0 - - @property - def errors(self) -> int: - return self.__error_count - - @property - def critical(self) -> int: - return self.__critical_error_count - - def add_task(self, task: InitTask): - self.__init_tasks.put(task) - - def run_cycle(self): - while not self.__init_tasks.empty(): - task: InitTask = self.__init_tasks.get() - - get_logger().info(f"Run task\t'{task.__class__.__name__}'") - - if issubclass(task.__class__, InitTask): - attempts_remain = task.attempt_count - - while attempts_remain > 0: - if task._run_task(): - break - - get_logger().error(f"Error in task\t'{task.__class__.__name__}'") - init_sleep(self.__error_attempt_delta) - - attempts_remain -= 1 - - if attempts_remain <= 0: - self.__error_count += 1 diff --git a/burrito/init/init_task.py b/burrito/init/init_task.py deleted file mode 100644 index 58c64b38..00000000 --- a/burrito/init/init_task.py +++ /dev/null @@ -1,36 +0,0 @@ -from burrito.utils.logger import get_logger - - -class InitTask: - def __init__( - self, - wait_time: int = 30, - attempt_count: int = 2, - can_skip: bool = False, - ) -> None: - self.__wait_time = wait_time if wait_time >= 1 else 30 - self.__attempt_count = attempt_count if attempt_count >= 1 else 1 - self.__can_skip = can_skip if isinstance(can_skip, bool) else bool(can_skip) - - @property - def wait_time(self) -> int: - return self.__wait_time - - @property - def attempt_count(self) -> int: - return self.__attempt_count - - @property - def can_skip(self) -> bool: - return self.__can_skip - - def _run_task(self) -> bool | None: - try: - self.run() - return True - except Exception as exc: - get_logger().error(exc) - return None - - def run(self): - raise NotImplementedError("You can't call method of abstract class") diff --git a/burrito/init/tasks/check_db_task.py b/burrito/init/tasks/check_db_task.py deleted file mode 100644 index 593da7c1..00000000 --- a/burrito/init/tasks/check_db_task.py +++ /dev/null @@ -1,38 +0,0 @@ -import socket - -from burrito.init.init_task import InitTask - -from burrito.utils.config_reader import get_config -from burrito.utils.db_utils import create_tables - - -class CheckDBTask(InitTask): - def __init__(self, wait_time: int = 30, attempt_count: int = 2, can_skip: bool = False) -> None: - super().__init__(wait_time, attempt_count, can_skip) - - def run(self): - db_host: str = get_config().BURRITO_DB_HOST - db_port: str | int = get_config().BURRITO_DB_PORT - - if not db_host: - raise Exception("Database host is not defined") - - if not db_port: - raise Exception("Database port is not defined") - - if not db_port.isdigit(): - raise Exception("Database port should be integer") - - db_port = int(db_port) - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(1) - - try: - sock.connect((db_host, db_port)) - except Exception as exc: - raise Exception(f"No connection with database: {exc}") - finally: - sock.close() - - create_tables() diff --git a/burrito/init/tasks/preprocessor_task.py b/burrito/init/tasks/preprocessor_task.py deleted file mode 100644 index 60edc583..00000000 --- a/burrito/init/tasks/preprocessor_task.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -from pathlib import Path -import os - -import pymysql.cursors - -from burrito.init.init_task import InitTask -from burrito.utils.config_reader import get_config - -from burrito.models.group_model import Groups -from burrito.models.statuses_model import Statuses -from burrito.models.faculty_model import Faculties -from burrito.models.queues_model import Queues -from burrito.models.permissions_model import Permissions -from burrito.models.roles_model import Roles -from burrito.models.role_permissions_model import RolePermissions - - -MODEL_KEYS = { - "groups": Groups, - "faculties": Faculties, - "statuses": Statuses, - "queues": Queues, - "permissions": Permissions, - "roles": Roles, - "role_permissions": RolePermissions -} - - -class PreProcessorTask(InitTask): - def __init__(self, wait_time: int = 30, attempt_count: int = 2, can_skip: bool = False) -> None: - super().__init__(wait_time, attempt_count, can_skip) - - def run(self): - __config_path = Path( - os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")) - ) / "preprocessor_config.json" - - if not os.path.exists(__config_path): - raise FileNotFoundError("preprocessor_config.json is not found") - - conn = pymysql.connect( - database=get_config().BURRITO_DB_NAME, - user=get_config().BURRITO_DB_USER, - password=get_config().BURRITO_DB_PASSWORD, - host=get_config().BURRITO_DB_HOST, - port=int(get_config().BURRITO_DB_PORT), - cursorclass=pymysql.cursors.DictCursor - ) - - with conn: - __sql_commands: dict = {} - __config_data: dict = {} - data: dict = None - - with open(__config_path, "r", encoding="utf-8") as file: - data = json.loads(file.read()) - - for key, value in data.items(): - if not key.startswith("__"): - __config_data[key] = value - continue - - if key == "__tables_option": - __sql_commands = value - - for table, config_values in __config_data.items(): - with conn.cursor() as cursor: - cursor.execute(__sql_commands[table]) - conn.commit() - - config_filtered_values: set = {tuple(i.values()) for i in config_values} - db_filtered_values: set = {tuple(i.values()) for i in cursor.fetchall()} - - if config_filtered_values.difference(db_filtered_values): - for value in config_values: - try: - MODEL_KEYS[table].create(**value) - except: - ... diff --git a/burrito/utils/app_util.py b/burrito/utils/app_util.py index fc998ba7..da5b97f3 100644 --- a/burrito/utils/app_util.py +++ b/burrito/utils/app_util.py @@ -1,15 +1,9 @@ from fastapi import FastAPI, APIRouter - -# from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.middleware.cors import CORSMiddleware -# from burrito.middlewares.user_agent import UserAgentMiddleware - -from burrito.preprocessor.core import preprocessor_task - from .singleton_pattern import singleton -from .task_manager import get_async_manager -from .logger import get_logger +from burrito.utils.task_manager import get_task_manager +from burrito.apps.scheduler.core import start_scheduler @singleton @@ -52,23 +46,7 @@ async def startup_event(): Setup task when when server is started """ - task_manager = get_async_manager() - task_manager.add_task(preprocessor_task()) - -# task_manager.add_task(get_pubsub_manager().run()) - - def test1(): - print("test1") - - async def test2(): - print("test2") - -# get_pubsub_manager().add_callback("test1", test1) -# get_pubsub_manager().add_callback("test2", test2) - - task_manager.run() - - get_logger().info("All tasks was started") + get_task_manager().run() def connect_app(fast_api_object: FastAPI, prefix: str, router: APIRouter): diff --git a/burrito/utils/db_utils.py b/burrito/utils/db_utils.py index fe00ce93..aa597154 100644 --- a/burrito/utils/db_utils.py +++ b/burrito/utils/db_utils.py @@ -39,9 +39,12 @@ def create_tables(): Comments, Queues, Bookmarks ] - get_database_cursor().create_tables(all_models) - - get_logger().info("All tables was created") + try: + get_database_cursor().create_tables(all_models) + except Exception as e: + get_logger().warning(f"{e}") + else: + get_logger().info("All tables was created") def drop_tables(use: bool = False): diff --git a/burrito/utils/file_manager.py b/burrito/utils/file_manager.py index 7ed184a7..063d970e 100644 --- a/burrito/utils/file_manager.py +++ b/burrito/utils/file_manager.py @@ -6,7 +6,7 @@ import aiofiles from burrito.utils.singleton_pattern import singleton -from burrito.utils.task_manager import get_async_manager +from burrito.utils.task_manager import get_task_manager @singleton @@ -43,7 +43,7 @@ async def push_file(self, file_data: UploadFile) -> None: await file.write("123") async def push_files(self, file_data_list: list[UploadFile]) -> None: - get_async_manager().add_multiply_task( + get_task_manager().add_multiply_task( [self.push_file(item) for item in file_data_list] ) diff --git a/burrito/utils/logger.py b/burrito/utils/logger.py index 88d1d9d0..a61e4b3c 100644 --- a/burrito/utils/logger.py +++ b/burrito/utils/logger.py @@ -12,7 +12,7 @@ class BurritoFormatter(logging.Formatter): red = "\x1b[31;20m" magenta = "\u001b[35m" reset = "\x1b[0m" - _format = "[ %(asctime)s ] | %(name)s | %(levelname)s: %(message)s (%(filename)s:%(lineno)d)" + _format = "[ %(asctime)s ] | %(name)s (%(process)d) | %(levelname)s: %(message)s (%(filename)s:%(lineno)d)" # Defining formats FORMATS = { diff --git a/burrito/utils/query_util.py b/burrito/utils/query_util.py index a7dafa6e..c843891b 100644 --- a/burrito/utils/query_util.py +++ b/burrito/utils/query_util.py @@ -1,4 +1,3 @@ -from fastapi import HTTPException from peewee import Expression from burrito.models.bookmarks_model import Bookmarks @@ -11,23 +10,15 @@ from burrito.utils.converter import ( StatusConverter, FacultyConverter, - QueueConverter ) -_PROTECTED_STATUSES: tuple[int] = ( - StatusConverter.convert(1).status_id, -) - -ADMIN_ROLES: list[int] = [9] +_PROTECTED_STATUSES: tuple[int] = (1,) -STATUSES_FOR_USER: list[int] = [] -STATUSES_FOR_ADMIN: list[int] = [] +ADMIN_ROLES: list[int] = (9, 10) -for i in Statuses.select(): - if i.status_id not in _PROTECTED_STATUSES: - STATUSES_FOR_USER.append(i.status_id) - STATUSES_FOR_ADMIN.append(i.status_id) +STATUSES_FOR_USER: list[int] = [i.status_id for i in Statuses.select() if i.status_id not in _PROTECTED_STATUSES] +STATUSES_FOR_ADMIN: list[int] = [i.status_id for i in Statuses.select()] def q_is_creator(value) -> Expression: diff --git a/burrito/utils/task_manager.py b/burrito/utils/task_manager.py index 2627670d..9cb2b2b3 100644 --- a/burrito/utils/task_manager.py +++ b/burrito/utils/task_manager.py @@ -3,29 +3,16 @@ """ -from threading import get_native_id +from concurrent.futures import ThreadPoolExecutor from typing import Any import asyncio -import sys +import inspect from burrito.utils.logger import get_logger +from burrito.utils.singleton_pattern import singleton -def thread_singleton(class_) -> Any: - class_instance: dict[int, _TaskManager] = {} - - def get_class_instance(*args, **kwargs): - instance_key = (class_, get_native_id()) - - if not class_instance.get(instance_key): - class_instance[instance_key] = class_(*args, **kwargs) - - return class_instance[instance_key] - - return get_class_instance - - -@thread_singleton +@singleton class _TaskManager: def __init__(self) -> None: """_summary_ @@ -34,11 +21,15 @@ def __init__(self) -> None: """ - self.__loop = self.__get_running_loop() + self._thread_pool = ThreadPoolExecutor(max_workers=25) + self._loop = self._get_running_loop() + + asyncio.set_event_loop(self._loop) - asyncio.set_event_loop(self.__loop) + def __del__(self): + self._thread_pool.shutdown(wait=True) - def __get_running_loop(self) -> asyncio.AbstractEventLoop: + def _get_running_loop(self) -> asyncio.AbstractEventLoop: """_summary_ Create or return running event loop @@ -55,8 +46,8 @@ def __get_running_loop(self) -> asyncio.AbstractEventLoop: except Exception as e: get_logger().critical(f"Unexpected error {e}") - get_logger().info("Exit program") - sys.exit(1) + get_logger().warning("Async loop is not running...") + return None @property def loop(self) -> asyncio.AbstractEventLoop: @@ -68,30 +59,37 @@ def loop(self) -> asyncio.AbstractEventLoop: asyncio.AbstractEventLoop: event loop object """ - return self.__loop + return self._loop - def add_task(self, coro) -> None: + @property + def pool(self) -> ThreadPoolExecutor: + return self._thread_pool + + def add_task(self, task, *args, **kwargs) -> None: """_summary_ Add task to execute in event loop Args: - coro (_type_): coroutine object + task (_type_): function/coroutine object """ - self.__loop.create_task(coro) + if inspect.iscoroutine(task): + self._loop.create_task(task) + else: + self._thread_pool.submit(task, *args, **kwargs) - def add_multiply_task(self, coro_list: tuple[Any]) -> None: + def add_multiply_task(self, task_list: tuple[Any]) -> None: """_summary_ Create few tasks using Args: - coro_list (tuple[Any]): coroutines tuple + task_list (tuple[Any]): function/coroutine tuple """ - for coro in coro_list: - self.add_task(coro) + for task in task_list: + self.add_task(task) def run(self, *, forever: bool = True) -> None: """_summary_ @@ -103,15 +101,15 @@ def run(self, *, forever: bool = True) -> None: If this option is True cycle run forever else until complete. Defaults to True. """ - if self.__loop.is_running(): # exit function if loop is running + if self._loop.is_running(): # exit function if loop is running return if forever: - self.__loop.run_forever() + self._loop.run_forever() else: - self.__loop.run_until_complete( + self._loop.run_until_complete( asyncio.gather( - *asyncio.all_tasks(self.__loop) # unpack task list + *asyncio.all_tasks(self._loop) # unpack task list ) ) @@ -121,10 +119,10 @@ def stop(self): Stop running event loop """ - self.__loop.stop() + self._loop.stop() -def get_async_manager() -> _TaskManager: +def get_task_manager() -> _TaskManager: """_summary_ Interface to get access to AsyncManager diff --git a/docker-compose.yml b/docker-compose-dbs.yml similarity index 71% rename from docker-compose.yml rename to docker-compose-dbs.yml index 7d44fc04..94a1247c 100644 --- a/docker-compose.yml +++ b/docker-compose-dbs.yml @@ -1,16 +1,6 @@ version: "3.0" services: - burrito_app: - container_name: burrito_app - build: . - env_file: - - .env - ports: - - "8080:8080" - links: - - "db" - db: container_name: burrito_db image: mysql @@ -18,6 +8,10 @@ services: - .env ports: - "3306:3306" + ulimits: + nofile: + soft: 20000 + hard: 40000 redis: container_name: burrito_redis diff --git a/docker-compose-separated.yml b/docker-compose-separated.yml index 276f7ee8..ce6d61d3 100644 --- a/docker-compose-separated.yml +++ b/docker-compose-separated.yml @@ -24,46 +24,17 @@ services: - burrito_registration - burrito_tickets - db: - container_name: burrito_db - image: mysql - env_file: - - .env - ports: - - "3306:3306" - - redis: - container_name: burrito_redis - image: redis - ports: - - "6379:6379" - - mongo: - container_name: burrito_mongo - image: mongo:4.4.6 - ports: - - "27017:27017" - -# prometheus: -# container_name: burrito_prometheus -# image: prom/prometheus -# ports: -# - 9090:9090 -# volumes: -# - ./prometheus.yml:/etc/prometheus/prometheus.yml -# command: -# - '--config.file=/etc/prometheus/prometheus.yml' -# -# grafana: -# container_name: burrito_grafana -# image: grafana/grafana -# ports: -# - 3000:3000 - - # APPS SECTION # INFO: port forwarding is provided only for testing, this option will be deleted soon + burrito_scheduler: + container_name: burrito_scheduler + build: + context: . + dockerfile: burrito/apps/scheduler/Dockerfile + env_file: + - .env + burrito_about: container_name: burrito_about build: @@ -73,8 +44,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_admin: container_name: burrito_admin @@ -85,8 +54,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_anon: container_name: burrito_anon @@ -97,8 +64,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_auth: container_name: burrito_auth @@ -109,8 +74,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_comments: container_name: burrito_comments @@ -121,8 +84,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_iofiles: container_name: burrito_iofiles @@ -133,8 +94,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_meta: container_name: burrito_meta @@ -145,8 +104,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_notifications: container_name: burrito_notifications @@ -157,8 +114,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_profile: container_name: burrito_profile @@ -169,8 +124,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_registration: container_name: burrito_registration @@ -181,8 +134,6 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" burrito_tickets: container_name: burrito_tickets @@ -193,5 +144,3 @@ services: - .env ports: - "$BURRITO_PORT" - links: - - "db" diff --git a/preprocessor_config.json b/preprocessor_config.json index bcf0f6eb..f0b88a77 100644 --- a/preprocessor_config.json +++ b/preprocessor_config.json @@ -8,17 +8,8 @@ "roles": "SELECT * FROM `roles`;", "role_permissions": "SELECT * FROM `role_permissions`;" }, - "groups": [ - {"group_id": 1, "name": "IT-11"}, - {"group_id": 2, "name": "LOL-11"}, - {"group_id": 3, "name": "LOL-12"}, - {"group_id": 1002512, "name": "KB-01"} - ], - "faculties": [ - {"faculty_id": 1, "name": "EliT"}, - {"faculty_id": 414, "name": "ElIT"}, - {"faculty_id": 2, "name": "Biem"} - ], + "groups": [], + "faculties": [], "statuses": [ {"status_id": 1, "name": "NEW"}, {"status_id": 2, "name": "ACCEPTED"}, diff --git a/pyproject.toml b/pyproject.toml index d4f9be7c..6574ea9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Burrito" -version = "0.7.2.dev2" +version = "0.8.0.dev2" description = "API for the issue tracker" authors = ["DimonBor", "m-o-d-e-r"] readme = "README.md" @@ -31,6 +31,8 @@ cryptography = "^41.0.2" tabulate = "^0.9.0" redis = "^4.6.0" pymongo = "^4.4.1" +schedule = "^1.2.0" +pytz = "^2023.3" [tool.poetry.group.dev.dependencies] diff --git a/scripts/burrito_cluster_run.sh b/scripts/burrito_cluster_run.sh index a9315aab..de124fa1 100644 --- a/scripts/burrito_cluster_run.sh +++ b/scripts/burrito_cluster_run.sh @@ -1 +1 @@ -docker-compose -f docker-compose-separated.yml up +docker-compose -f docker-compose-dbs.yml up && docker-compose -f docker-compose-separated.yml up diff --git a/scripts/prepare_db.py b/scripts/prepare_db.py new file mode 100644 index 00000000..322360b0 --- /dev/null +++ b/scripts/prepare_db.py @@ -0,0 +1,6 @@ +from burrito.utils.db_utils import create_tables +from burrito.apps.scheduler.preprocessor.core import preprocessor_task + + +create_tables() +preprocessor_task() diff --git a/tests/meta_test.py b/tests/meta_test.py index d5a26b40..b6c8f1cf 100644 --- a/tests/meta_test.py +++ b/tests/meta_test.py @@ -58,7 +58,7 @@ def test_queues_list(self): response = requests.post( f"http://{get_config().BURRITO_HOST}:{get_config().BURRITO_PORT}/meta/get_queues", json={ - "faculty": 1 + "faculty": 414 }, timeout=TIMEOUT ) diff --git a/tests/profile_test.py b/tests/profile_test.py index 45773717..da52d085 100644 --- a/tests/profile_test.py +++ b/tests/profile_test.py @@ -98,8 +98,8 @@ def test_update_profile_with_auth(self): "lastname": "".join(random.sample(string.ascii_letters, 5)) if random.randint(0, 10) % 2 == 0 else None, "email": "".join(random.sample(string.ascii_letters, 5)) if random.randint(0, 10) % 2 == 0 else None, "phone": "".join(random.sample(string.ascii_letters, 5)) if random.randint(0, 10) % 2 == 0 else None, - "faculty": random.choice([1, 2]), - "group": random.choice([1, 2]), + "faculty": random.choice([414, 1675]), + "group": random.choice([1003254, 1003565]), }, timeout=0.5 ) diff --git a/tests/registration_test.py b/tests/registration_test.py index 136bd1f4..44af2cc8 100644 --- a/tests/registration_test.py +++ b/tests/registration_test.py @@ -13,8 +13,8 @@ def make_user_registration( login: str = "".join(random.sample(string.ascii_letters, 5)), password: str = "".join(random.sample(string.ascii_letters, 8)), - group: int = random.choice([1, 2]), - faculty: int = random.choice([1, 2]) + group: int = 1003254, + faculty: int = 414 ): response = requests.post( f"http://{get_config().BURRITO_HOST}:{get_config().BURRITO_PORT}/registration/", @@ -60,7 +60,8 @@ def test_do_registration(self): { "obj": user_data, "cls": int - } + }, + user_data ) RegistrationTestCase.user_id = user_data diff --git a/tests/run_tests.py b/tests/run_tests.py index 5b12f380..d2a07bdc 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -25,7 +25,7 @@ unittest.TestLoader().loadTestsFromTestCase(ProfileTestCase), unittest.TestLoader().loadTestsFromTestCase(TicketsTestCase), unittest.TestLoader().loadTestsFromTestCase(AboutTestCase), -# unittest.TestLoader().loadTestsFromTestCase(AdminTestCase), +## unittest.TestLoader().loadTestsFromTestCase(AdminTestCase), unittest.TestLoader().loadTestsFromTestCase(AnonTestCase), unittest.TestLoader().loadTestsFromTestCase(MetaTestCase), #unittest.TestLoader().loadTestsFromTestCase(IOFilesTestCase) diff --git a/tests/tickets_test.py b/tests/tickets_test.py index 141335a4..f60aa970 100644 --- a/tests/tickets_test.py +++ b/tests/tickets_test.py @@ -25,7 +25,7 @@ def create_ticket_get_id(subject: str) -> int: "hidden": True if random.randint(0, 9) % 2 == 0 else False, "anonymous": True if random.randint(0, 9) % 2 == 0 else False, "queue": 1, - "faculty": 1, + "faculty": 414, }, timeout=TIMEOUT ).json()["ticket_id"] @@ -50,7 +50,7 @@ def test_001_create_ticket(self): "hidden": True if random.randint(0, 9) % 2 == 0 else False, "anonymous": True if random.randint(0, 9) % 2 == 0 else False, # "queue": 1, - "faculty": 1, + "faculty": 414 }, timeout=TIMEOUT ) @@ -128,7 +128,7 @@ def test_004_like_ticket(self): "hidden": False, "anonymous": True if random.randint(0, 9) % 2 == 0 else False, "queue": 1, - "faculty": random.choice([1, 2]), + "faculty": random.choice([414, 1675]), }, timeout=TIMEOUT ).json()["ticket_id"] @@ -190,7 +190,7 @@ def test_006_bookmark_ticket(self): "hidden": False, "anonymous": True if random.randint(0, 9) % 2 == 0 else False, "queue": 1, - "faculty": random.choice([1, 2]), + "faculty": random.choice([414, 1675]), }, timeout=TIMEOUT ).json()["ticket_id"] @@ -271,7 +271,7 @@ def test_009_ticket_detail_view(self): "hidden": False, "anonymous": False, "queue": 1, - "faculty": 1, + "faculty": 414 }, timeout=TIMEOUT ).json()["ticket_id"]