Parallelize Synapse worker template render

roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml

@@ -9,6 +9,7 @@
 
 - ansible.builtin.set_fact:
     matrix_synapse_enabled_worker_names: "{{ matrix_synapse_workers_enabled_list | map(attribute='name') }}"
+    matrix_synapse_worker_template_job_status_result_list: []
 
 # This also deletes some things which we need. They will be recreated below.
 - name: Ensure unnecessary worker configs are cleaned
@@ -45,3 +46,9 @@
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
   loop_control:
     loop_var: matrix_synapse_worker_details
+
+- name: Check status of worker systemd service files and configuration files creation
+  ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/worker_setup_job_cleanup.yml"
+  with_items: "{{ matrix_synapse_worker_template_job_status_result_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_template_job_status

roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml

@@ -5,16 +5,73 @@
     matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_details.name }}"
     matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.name }}.yaml"
 
+# ansible.builtin.template does not support async, so instead run async commands
+# that launch ad hoc template tasks on the controller.
 - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
-  ansible.builtin.template:
-    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
-    dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}"
-    mode: 0644
-    owner: "{{ matrix_synapse_uid }}"
-    group: "{{ matrix_synapse_gid }}"
+  ansible.builtin.command: >
+    ansible matrix_servers -i inventory/hosts -m template --become
+    -e "{
+    'matrix_synapse_worker_details':{{ matrix_synapse_worker_details }},
+    'matrix_server_fqn_matrix':'{{ matrix_server_fqn_matrix }}',
+    'matrix_synapse_replication_listener_enabled':'{{ matrix_synapse_replication_listener_enabled }}',
+    'matrix_synapse_replication_http_port':'{{ matrix_synapse_replication_http_port }}',
+    'matrix_synapse_metrics_enabled':'{{ matrix_synapse_metrics_enabled }}'
+    }"
+    -a "
+    src={{ role_path }}/templates/synapse/worker.yaml.j2
+    dest={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}
+    mode=0644
+    owner={{ matrix_synapse_uid }}
+    group={{ matrix_synapse_gid }}
+    "
+  register: "configuration_result"
+  delegate_to: localhost
+  become: false
+  async: 60
+  poll: 0
+  changed_when: false
 
 - name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }}
-  ansible.builtin.template:
-    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2"
-    dest: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service"
-    mode: 0644
+  ansible.builtin.command: >
+    ansible matrix_servers -i inventory/hosts -m template --become
+    -e "{
+    'matrix_synapse_worker_details':{{ matrix_synapse_worker_details }},
+    'matrix_synapse_worker_container_name':'{{ matrix_synapse_worker_container_name }}',
+    'matrix_synapse_config_dir_path':'{{ matrix_synapse_config_dir_path }}',
+    'matrix_synapse_worker_config_file_name':'{{ matrix_synapse_worker_config_file_name }}',
+    'devture_systemd_docker_base_systemd_unit_home_path':'{{ devture_systemd_docker_base_systemd_unit_home_path }}',
+    'devture_systemd_docker_base_host_command_sh':'{{ devture_systemd_docker_base_host_command_sh }}',
+    'devture_systemd_docker_base_host_command_docker':'{{ devture_systemd_docker_base_host_command_docker }}',
+    'matrix_host_command_sleep':'{{ matrix_host_command_sleep }}',
+    'matrix_synapse_uid':'{{ matrix_synapse_uid }}',
+    'matrix_synapse_gid':'{{ matrix_synapse_gid }}',
+    'matrix_synapse_tmp_directory_size_mb':'{{ matrix_synapse_tmp_directory_size_mb }}',
+    'matrix_synapse_container_network':'{{ matrix_synapse_container_network }}',
+    'matrix_synapse_workers_enabled':'{{ matrix_synapse_workers_enabled }}',
+    'matrix_synapse_workers_container_host_bind_address':'{{ matrix_synapse_workers_container_host_bind_address }}',
+    'matrix_synapse_storage_path':'{{ matrix_synapse_storage_path }}',
+    'matrix_synapse_container_additional_volumes':{{ matrix_synapse_container_additional_volumes }},
+    'matrix_synapse_container_arguments':{{ matrix_synapse_container_arguments }},
+    'matrix_synapse_docker_image_final':'{{ matrix_synapse_docker_image_final }}',
+    'matrix_synapse_container_additional_networks':{{ matrix_synapse_container_additional_networks }}
+    }"
+    -a "
+    src={{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2
+    dest={{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service
+    mode=0644
+    "
+  register: "service_result"
+  delegate_to: localhost
+  become: false
+  async: 60
+  poll: 0
+  changed_when: false
+
+# Store job status results for checking later
+- ansible.builtin.set_fact:
+    matrix_synapse_worker_template_job_status_result_list: "{{ matrix_synapse_worker_template_job_status_result_list + [item] }}"
+  with_items:
+    - result:
+        name: "{{ matrix_synapse_worker_details.name }}"
+        configuration: "{{ configuration_result }}"
+        service: "{{ service_result }}"

roles/custom/matrix-synapse/tasks/synapse/workers/util/worker_setup_job_cleanup.yml

@@ -0,0 +1,36 @@
+---
+
+# Clean up Ansible controller temp files as a result of spawning async tasks
+- name: Check job status for configuration file {{ matrix_synapse_worker_template_job_status.result.name }}
+  ansible.builtin.async_status:
+    jid: "{{ matrix_synapse_worker_template_job_status.result.configuration.ansible_job_id }}"
+  register: configuration_status
+  until: configuration_status.finished
+  retries: 60
+  delay: 1
+  delegate_to: localhost
+  become: false
+
+- name: Check job status for service file {{ matrix_synapse_worker_template_job_status.result.name }}
+  ansible.builtin.async_status:
+    jid: "{{ matrix_synapse_worker_template_job_status.result.service.ansible_job_id }}"
+  register: service_status
+  until: service_status.finished
+  retries: 60
+  delay: 1
+  delegate_to: localhost
+  become: false
+
+- name: Cleanup job result for configuration file
+  ansible.builtin.async_status:
+    jid: "{{ matrix_synapse_worker_template_job_status.result.configuration.ansible_job_id }}"
+    mode: "cleanup"
+  delegate_to: localhost
+  become: false
+
+- name: Cleanup job result for service file
+  ansible.builtin.async_status:
+    jid: "{{ matrix_synapse_worker_template_job_status.result.service.ansible_job_id }}"
+    mode: "cleanup"
+  delegate_to: localhost
+  become: false