From b3655f439c8f944b0eb909f8a5f1b859929b6d59 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 22 Jun 2023 13:32:03 -0500 Subject: [PATCH 1/2] Parallelize Synapse worker template render --- .../tasks/synapse/workers/setup_install.yml | 8 ++ .../workers/util/setup_files_for_worker.yml | 74 ++++++++++++++++--- .../workers/util/worker_setup_job_cleanup.yml | 44 +++++++++++ 3 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml index 85ddf3b94c2..840f1886fc3 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml @@ -45,3 +45,11 @@ with_items: "{{ matrix_synapse_workers_enabled_list }}" loop_control: loop_var: matrix_synapse_worker_details + index_var: worker_template_creation_index + +- name: Check status of worker systemd service files and configuration files creation + ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/worker_setup_job_cleanup.yml" + with_items: "{{ matrix_synapse_workers_enabled_list }}" + loop_control: + loop_var: matrix_synapse_worker_details + index_var: worker_template_job_status_index diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml index d350941297d..03bac63ed4f 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml @@ -5,16 +5,70 @@ matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_details.name }}" matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.name }}.yaml" +# ansible.builtin.template does not support async, so instead run async commands +# that launch ad hoc template tasks on the controller. - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }} - ansible.builtin.template: - src: "{{ role_path }}/templates/synapse/worker.yaml.j2" - dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}" - mode: 0644 - owner: "{{ matrix_synapse_uid }}" - group: "{{ matrix_synapse_gid }}" + ansible.builtin.command: > + ansible matrix_servers -i inventory/hosts -m template --become + -e "{ + 'matrix_synapse_worker_details':{{ matrix_synapse_worker_details }}, + 'matrix_server_fqn_matrix':'{{ matrix_server_fqn_matrix }}', + 'matrix_synapse_replication_listener_enabled':'{{ matrix_synapse_replication_listener_enabled }}', + 'matrix_synapse_replication_http_port':'{{ matrix_synapse_replication_http_port }}', + 'matrix_synapse_metrics_enabled':'{{ matrix_synapse_metrics_enabled }}' + }" + -a " + src={{ role_path }}/templates/synapse/worker.yaml.j2 + dest={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }} + mode=0644 + owner={{ matrix_synapse_uid }} + group={{ matrix_synapse_gid }} + " + register: "configuration_result" + delegate_to: localhost + become: false + async: 60 + poll: 0 - name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }} - ansible.builtin.template: - src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2" - dest: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service" - mode: 0644 + ansible.builtin.command: > + ansible matrix_servers -i inventory/hosts -m template --become + -e "{ + 'matrix_synapse_worker_details':{{ matrix_synapse_worker_details }}, + 'matrix_synapse_worker_container_name':'{{ matrix_synapse_worker_container_name }}', + 'matrix_synapse_config_dir_path':'{{ matrix_synapse_config_dir_path }}', + 'matrix_synapse_worker_config_file_name':'{{ matrix_synapse_worker_config_file_name }}', + 'devture_systemd_docker_base_systemd_unit_home_path':'{{ devture_systemd_docker_base_systemd_unit_home_path }}', + 'devture_systemd_docker_base_host_command_sh':'{{ devture_systemd_docker_base_host_command_sh }}', + 'devture_systemd_docker_base_host_command_docker':'{{ devture_systemd_docker_base_host_command_docker }}', + 'matrix_host_command_sleep':'{{ matrix_host_command_sleep }}', + 'matrix_synapse_uid':'{{ matrix_synapse_uid }}', + 'matrix_synapse_gid':'{{ matrix_synapse_gid }}', + 'matrix_synapse_tmp_directory_size_mb':'{{ matrix_synapse_tmp_directory_size_mb }}', + 'matrix_synapse_container_network':'{{ matrix_synapse_container_network }}', + 'matrix_synapse_workers_enabled':'{{ matrix_synapse_workers_enabled }}', + 'matrix_synapse_workers_container_host_bind_address':'{{ matrix_synapse_workers_container_host_bind_address }}', + 'matrix_synapse_storage_path':'{{ matrix_synapse_storage_path }}', + 'matrix_synapse_container_additional_volumes':{{ matrix_synapse_container_additional_volumes }}, + 'matrix_synapse_container_arguments':{{ matrix_synapse_container_arguments }}, + 'matrix_synapse_docker_image_final':'{{ matrix_synapse_docker_image_final }}', + 'matrix_synapse_container_additional_networks':{{ matrix_synapse_container_additional_networks }} + }" + -a " + src={{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2 + dest={{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service + mode=0644 + " + register: "service_result" + delegate_to: localhost + become: false + async: 60 + poll: 0 + +# Create unique variable identifiers for checking job status later +- ansible.builtin.set_fact: + "{{ item.name }}": "{{ item.val }}" + when: item.name not in vars + with_items: + - { name: "configuration_result_{{ worker_template_creation_index }}", val: "{{ configuration_result }}" } + - { name: "service_result_{{ worker_template_creation_index }}", val: "{{ service_result }}" } diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml new file mode 100644 index 00000000000..2e683947275 --- /dev/null +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml @@ -0,0 +1,44 @@ +--- + +# Clean up Ansible controller temp files as a result of spawning async tasks +- name: Check job status for configuration file {{ matrix_synapse_worker_details.name }} + vars: + worker_configuration_result: "{{ lookup('ansible.builtin.vars', 'configuration_result_' + worker_template_job_status_index|string) }}" + ansible.builtin.async_status: + jid: "{{ worker_configuration_result.ansible_job_id }}" + register: configuration_status + until: configuration_status.finished + retries: 60 + delay: 1 + delegate_to: localhost + become: false + +- name: Check job status for service file {{ matrix_synapse_worker_details.name }} + vars: + worker_service_result: "{{ lookup('ansible.builtin.vars', 'service_result_' + worker_template_job_status_index|string) }}" + ansible.builtin.async_status: + jid: "{{ worker_service_result.ansible_job_id }}" + register: service_status + until: service_status.finished + retries: 60 + delay: 1 + delegate_to: localhost + become: false + +- name: Cleanup job result for configuration file + vars: + worker_configuration_result: "{{ lookup('ansible.builtin.vars', 'configuration_result_' + worker_template_job_status_index|string) }}" + ansible.builtin.async_status: + jid: "{{ worker_configuration_result.ansible_job_id }}" + mode: "cleanup" + delegate_to: localhost + become: false + +- name: Cleanup job result for service file + vars: + worker_service_result: "{{ lookup('ansible.builtin.vars', 'service_result_' + worker_template_job_status_index|string) }}" + ansible.builtin.async_status: + jid: "{{ worker_service_result.ansible_job_id }}" + mode: "cleanup" + delegate_to: localhost + become: false From 56880ad220d534c6e8745eb4a61857d911ef3a4a Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 22 Jun 2023 19:32:22 -0500 Subject: [PATCH 2/2] Fixed linting errors --- .../tasks/synapse/workers/setup_install.yml | 7 +++---- .../workers/util/setup_files_for_worker.yml | 13 +++++++----- .../workers/util/worker_setup_job_cleanup.yml | 20 ++++++------------- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml index 840f1886fc3..3aa7b333208 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml @@ -9,6 +9,7 @@ - ansible.builtin.set_fact: matrix_synapse_enabled_worker_names: "{{ matrix_synapse_workers_enabled_list | map(attribute='name') }}" + matrix_synapse_worker_template_job_status_result_list: [] # This also deletes some things which we need. They will be recreated below. - name: Ensure unnecessary worker configs are cleaned @@ -45,11 +46,9 @@ with_items: "{{ matrix_synapse_workers_enabled_list }}" loop_control: loop_var: matrix_synapse_worker_details - index_var: worker_template_creation_index - name: Check status of worker systemd service files and configuration files creation ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/worker_setup_job_cleanup.yml" - with_items: "{{ matrix_synapse_workers_enabled_list }}" + with_items: "{{ matrix_synapse_worker_template_job_status_result_list }}" loop_control: - loop_var: matrix_synapse_worker_details - index_var: worker_template_job_status_index + loop_var: matrix_synapse_worker_template_job_status diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml index 03bac63ed4f..31d8bf240af 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml @@ -29,6 +29,7 @@ become: false async: 60 poll: 0 + changed_when: false - name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }} ansible.builtin.command: > @@ -64,11 +65,13 @@ become: false async: 60 poll: 0 + changed_when: false -# Create unique variable identifiers for checking job status later +# Store job status results for checking later - ansible.builtin.set_fact: - "{{ item.name }}": "{{ item.val }}" - when: item.name not in vars + matrix_synapse_worker_template_job_status_result_list: "{{ matrix_synapse_worker_template_job_status_result_list + [item] }}" with_items: - - { name: "configuration_result_{{ worker_template_creation_index }}", val: "{{ configuration_result }}" } - - { name: "service_result_{{ worker_template_creation_index }}", val: "{{ service_result }}" } + - result: + name: "{{ matrix_synapse_worker_details.name }}" + configuration: "{{ configuration_result }}" + service: "{{ service_result }}" diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml index 2e683947275..917ae8d82da 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml @@ -1,11 +1,9 @@ --- # Clean up Ansible controller temp files as a result of spawning async tasks -- name: Check job status for configuration file {{ matrix_synapse_worker_details.name }} - vars: - worker_configuration_result: "{{ lookup('ansible.builtin.vars', 'configuration_result_' + worker_template_job_status_index|string) }}" +- name: Check job status for configuration file {{ matrix_synapse_worker_template_job_status.result.name }} ansible.builtin.async_status: - jid: "{{ worker_configuration_result.ansible_job_id }}" + jid: "{{ matrix_synapse_worker_template_job_status.result.configuration.ansible_job_id }}" register: configuration_status until: configuration_status.finished retries: 60 @@ -13,11 +11,9 @@ delegate_to: localhost become: false -- name: Check job status for service file {{ matrix_synapse_worker_details.name }} - vars: - worker_service_result: "{{ lookup('ansible.builtin.vars', 'service_result_' + worker_template_job_status_index|string) }}" +- name: Check job status for service file {{ matrix_synapse_worker_template_job_status.result.name }} ansible.builtin.async_status: - jid: "{{ worker_service_result.ansible_job_id }}" + jid: "{{ matrix_synapse_worker_template_job_status.result.service.ansible_job_id }}" register: service_status until: service_status.finished retries: 60 @@ -26,19 +22,15 @@ become: false - name: Cleanup job result for configuration file - vars: - worker_configuration_result: "{{ lookup('ansible.builtin.vars', 'configuration_result_' + worker_template_job_status_index|string) }}" ansible.builtin.async_status: - jid: "{{ worker_configuration_result.ansible_job_id }}" + jid: "{{ matrix_synapse_worker_template_job_status.result.configuration.ansible_job_id }}" mode: "cleanup" delegate_to: localhost become: false - name: Cleanup job result for service file - vars: - worker_service_result: "{{ lookup('ansible.builtin.vars', 'service_result_' + worker_template_job_status_index|string) }}" ansible.builtin.async_status: - jid: "{{ worker_service_result.ansible_job_id }}" + jid: "{{ matrix_synapse_worker_template_job_status.result.service.ansible_job_id }}" mode: "cleanup" delegate_to: localhost become: false