Fix missed service restarts with Synapse workers enabled

With Synapse workers enabled, inject_worker.yml appends each worker's
service to devture_systemd_service_manager_services_list_auto with
set_fact. That templates the whole list, including every service's
restart_necessary value, when the matrix-synapse role initializes, and
the resulting fact outranks group_vars. Services whose restart flag is
set after that point are then only started, not restarted, on install-*
runs even when their configuration changed. That includes the
matrix-synapse role's own reverse-proxy companion, Goofys and S3 storage
provider timer, and the services of every later role that sets a restart
flag (Element, Grafana, Prometheus, Traefik and others).

Collect worker services in matrix_synapse_worker_systemd_services_list
and append it to the auto list in group_vars/matrix_servers, like the
other entries there, so the list stays lazy.

Include inject_worker.yml once and loop inside each task instead of
including it once per worker. With 19 workers this cuts the step from
26.2 s to 5.3 s on ansible-core 2.20.1 (controller-only, one run). Also
fix two stream-writer failure messages that referenced the undefined
`item`, and report all missing worker properties in one message.

group_vars/matrix_servers

@@ -958,6 +958,8 @@ devture_systemd_service_manager_services_list_auto: |
       'restart_necessary': (traefik_certs_dumper_restart_necessary | bool),
       'groups': ['matrix', 'traefik-certs-dumper'],
     }] if traefik_certs_dumper_enabled else [])
+    +
+    (matrix_synapse_worker_systemd_services_list if matrix_synapse_enabled else [])
   }}
 
 ########################################################################

roles/custom/matrix-synapse/tasks/init.yml

@@ -26,7 +26,4 @@
 
 - name: Ensure workers are injected into various places
   ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/inject_worker.yml"
-  with_items: "{{ matrix_synapse_workers_enabled_list }}"
-  loop_control:
-    loop_var: matrix_synapse_worker_details
   when: matrix_synapse_workers_enabled | bool

roles/custom/matrix-synapse/tasks/synapse/workers/util/inject_worker.yml

@@ -3,6 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 ---
+# Each task below loops over all workers.
+# Including this file once per worker instead (10 tasks x N workers) was measurably slower.
+
 # The tasks below run before `validate_config.yml`.
 # To avoid failing with a cryptic error message, we'll do validation here.
 #
@@ -13,15 +16,14 @@
 # Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right.
 - name: Fail if required property not defined for worker
   ansible.builtin.fail:
-    msg: "Synapse workers (like {{ matrix_synapse_worker_details | to_json }}) need to define a `{{ item }}` property"
-  with_items:
-    - id
-    - name
-    - type
-    - app
-    - port
-    - webserving
-  when: "item not in matrix_synapse_worker_details"
+    msg: >-
+      Synapse workers (like {{ matrix_synapse_worker_details | to_json }}) need to define these properties:
+      {{ ['id', 'name', 'type', 'app', 'port', 'webserving'] | reject('in', matrix_synapse_worker_details) | join(', ') }}
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name | default(matrix_synapse_worker_details.id | default('?')) }}"
+  when: "['id', 'name', 'type', 'app', 'port', 'webserving'] | reject('in', matrix_synapse_worker_details) | length > 0"
 
 # Names are used for container names and systemd services.
 # Routing happens based on container names, so Synapse processes that try to route to workers with underscores in the name will complain. Example:
@@ -29,46 +31,83 @@
 - name: Fail if worker name includes underscore
   ansible.builtin.fail:
     msg: "Unrecognized Synapse worker `name`: `{{ matrix_synapse_worker_details.name }}`. It must not include underscores"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: "'_' in matrix_synapse_worker_details.name"
 
 - name: Fail if worker type unknown
   ansible.builtin.fail:
     msg: "Unrecognized Synapse worker `type`: `{{ matrix_synapse_worker_details.type }}`. Supported types are: {{ matrix_synapse_known_worker_types | join(', ') }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: "matrix_synapse_worker_details.type not in matrix_synapse_known_worker_types"
 
 - name: Fail if worker app unknown
   ansible.builtin.fail:
     msg: "Unrecognized Synapse worker `app`: `{{ matrix_synapse_worker_details.app }}`. Supported types are: {{ matrix_synapse_workers_avail_list | join(', ') }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: "matrix_synapse_worker_details.app not in matrix_synapse_workers_avail_list"
 
-- when: "matrix_synapse_worker_details.type == 'stream_writer'"
-  block:
-    - name: Fail if stream_writer_stream not defined for stream_writer worker
-      ansible.builtin.fail:
-        msg: >-
-          Synapse stream_writer workers (such as {{ item }}) need to define a valid `stream_writer_stream` property
-          (not `{{ matrix_synapse_worker_details.stream_writer_stream | default('undefined') }}`).
-          Supported types are: {{ matrix_synapse_workers_known_stream_writer_stream_types | join(', ') }}
-      when: "'stream_writer_stream' not in matrix_synapse_worker_details or matrix_synapse_worker_details.stream_writer_stream not in matrix_synapse_workers_known_stream_writer_stream_types"
+- name: Fail if stream_writer_stream not defined for stream_writer worker
+  ansible.builtin.fail:
+    msg: >-
+      Synapse stream_writer workers (such as {{ matrix_synapse_worker_details.name }}) need to define a valid `stream_writer_stream` property
+      (not `{{ matrix_synapse_worker_details.stream_writer_stream | default('undefined') }}`).
+      Supported types are: {{ matrix_synapse_workers_known_stream_writer_stream_types | join(', ') }}
+  loop: "{{ matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'stream_writer') }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
+  when: "'stream_writer_stream' not in matrix_synapse_worker_details or matrix_synapse_worker_details.stream_writer_stream not in matrix_synapse_workers_known_stream_writer_stream_types"
 
-    - name: Fail if replication_port not defined for stream_writer worker
-      ansible.builtin.fail:
-        msg: "Synapse background workers of type stream_writer (such as {{ item }}) need to define a valid `replication_port` property"
-      when: "'replication_port' not in matrix_synapse_worker_details"
+- name: Fail if replication_port not defined for stream_writer worker
+  ansible.builtin.fail:
+    msg: "Synapse background workers of type stream_writer (such as {{ matrix_synapse_worker_details.name }}) need to define a valid `replication_port` property"
+  loop: "{{ matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'stream_writer') }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
+  when: "'replication_port' not in matrix_synapse_worker_details"
 
+# Workers go into their own list, which `group_vars/matrix_servers` appends to `devture_systemd_service_manager_services_list_auto`.
+# Using `set_fact` on `devture_systemd_service_manager_services_list_auto` itself would template the whole list right now,
+# freezing every service's `restart_necessary` flag before the roles that set it have run.
 - ansible.builtin.set_fact:
-    devture_systemd_service_manager_services_list_auto: "{{ devture_systemd_service_manager_services_list_auto + [{'name': (matrix_synapse_worker_details.name + '.service'), 'priority': 1100, 'groups': ['matrix', 'homeservers', 'synapse']}] }}"
+    matrix_synapse_worker_systemd_services_list: "{{ matrix_synapse_worker_systemd_services_list + [{'name': (matrix_synapse_worker_details.name + '.service'), 'priority': 1100, 'groups': ['matrix', 'homeservers', 'synapse']}] }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
 
 - ansible.builtin.set_fact:
     matrix_synapse_webserving_workers_systemd_services_list: "{{ matrix_synapse_webserving_workers_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: matrix_synapse_worker_details.webserving | bool
 
 # Inject stream writers into the instance map.
 - ansible.builtin.set_fact:
     matrix_synapse_instance_map: "{{ matrix_synapse_instance_map | combine({matrix_synapse_worker_details.name: {'host': matrix_synapse_worker_details.name, 'port': matrix_synapse_worker_details.replication_port | int}}) }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: matrix_synapse_worker_details.type in matrix_synapse_known_instance_map_eligible_worker_types
 
 # Inject pusher instances.
 - ansible.builtin.set_fact:
     matrix_synapse_federation_pusher_instances: "{{ matrix_synapse_federation_pusher_instances + [matrix_synapse_worker_details.name] }}"
+  loop: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+    label: "{{ matrix_synapse_worker_details.name }}"
   when: matrix_synapse_worker_details.type == 'pusher'

roles/custom/matrix-synapse/vars/main.yml

@@ -166,6 +166,10 @@ matrix_synapse_workers_webserving_stream_writer_types: ['typing', 'to_device', '
 # Not all workers serve web requests. Those that don't won't be injected here.
 matrix_synapse_webserving_workers_systemd_services_list: []
 
+# Populated with systemd service definitions for workers (see `util/inject_worker.yml`) and wired into
+# `devture_systemd_service_manager_services_list_auto` via `group_vars/matrix_servers`.
+matrix_synapse_worker_systemd_services_list: []
+
 # matrix_synapse_known_worker_types contains the list of known worker types.
 #
 # A worker type is different than a worker app (e.g. `generic_worker`).