gitlord
/
synapse
mirror of https://github.com/matrix-org/synapse.git
1
0
Fork 0
Code Issues 0 Releases 671 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
23441 Commits
842 Branches
799 MiB
 
 
 
 
 
 
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
synapse/docker/configure_workers_and_start.py

1085 lines
43 KiB
Raw Permalink Blame History 

  1. #!/usr/bin/env python

  2. # Copyright 2021 The Matrix.org Foundation C.I.C.

  3. #

  4. # Licensed under the Apache License, Version 2.0 (the "License");

  5. # you may not use this file except in compliance with the License.

  6. # You may obtain a copy of the License at

  7. #

  8. #     http://www.apache.org/licenses/LICENSE-2.0

  9. #

  10. # Unless required by applicable law or agreed to in writing, software

  11. # distributed under the License is distributed on an "AS IS" BASIS,

  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13. # See the License for the specific language governing permissions and

  14. # limitations under the License.

  15. 

  16. # This script reads environment variables and generates a shared Synapse worker,

  17. # nginx and supervisord configs depending on the workers requested.

  18. #

  19. # The environment variables it reads are:

  20. #   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.

  21. #   * SYNAPSE_REPORT_STATS: Whether to report stats.

  22. #   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKERS_CONFIG

  23. #         below. Leave empty for no workers. Add a ':' and a number at the end to

  24. #         multiply that worker. Append multiple worker types with '+' to merge the

  25. #         worker types into a single worker. Add a name and a '=' to the front of a

  26. #         worker type to give this instance a name in logs and nginx.

  27. #         Examples:

  28. #         SYNAPSE_WORKER_TYPES='event_persister, federation_sender, client_reader'

  29. #         SYNAPSE_WORKER_TYPES='event_persister:2, federation_sender:2, client_reader'

  30. #         SYNAPSE_WORKER_TYPES='stream_writers=account_data+presence+typing'

  31. #   * SYNAPSE_AS_REGISTRATION_DIR: If specified, a directory in which .yaml and .yml files

  32. #         will be treated as Application Service registration files.

  33. #   * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format.

  34. #   * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified,

  35. #         Nginx will be configured to serve TLS on port 8448.

  36. #   * SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER: Whether to use the forking launcher,

  37. #         only intended for usage in Complement at the moment.

  38. #         No stability guarantees are provided.

  39. #   * SYNAPSE_LOG_LEVEL: Set this to DEBUG, INFO, WARNING or ERROR to change the

  40. #         log level. INFO is the default.

  41. #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,

  42. #         regardless of the SYNAPSE_LOG_LEVEL setting.

  43. #   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful

  44. #     for testing.

  45. #

  46. # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined

  47. # in the project's README), this script may be run multiple times, and functionality should

  48. # continue to work if so.

  49. 

  50. import os

  51. import platform

  52. import re

  53. import subprocess

  54. import sys

  55. from collections import defaultdict

  56. from itertools import chain

  57. from pathlib import Path

  58. from typing import (

  59.     Any,

  60.     Dict,

  61.     List,

  62.     Mapping,

  63.     MutableMapping,

  64.     NoReturn,

  65.     Optional,

  66.     Set,

  67.     SupportsIndex,

  68. )

  69. 

  70. import yaml

  71. from jinja2 import Environment, FileSystemLoader

  72. 

  73. MAIN_PROCESS_HTTP_LISTENER_PORT = 8080

  74. MAIN_PROCESS_INSTANCE_NAME = "main"

  75. MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"

  76. MAIN_PROCESS_REPLICATION_PORT = 9093

  77. # Obviously, these would only be used with the UNIX socket option

  78. MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"

  79. MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"

  80. 

  81. # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced

  82. # during processing with the name of the worker.

  83. WORKER_PLACEHOLDER_NAME = "placeholder_name"

  84. 

  85. # Workers with exposed endpoints needs either "client", "federation", or "media" listener_resources

  86. # Watching /_matrix/client needs a "client" listener

  87. # Watching /_matrix/federation needs a "federation" listener

  88. # Watching /_matrix/media and related needs a "media" listener

  89. # Stream Writers require "client" and "replication" listeners because they

  90. #   have to attach by instance_map to the master process and have client endpoints.

  91. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {

  92.     "pusher": {

  93.         "app": "synapse.app.generic_worker",

  94.         "listener_resources": [],

  95.         "endpoint_patterns": [],

  96.         "shared_extra_conf": {},

  97.         "worker_extra_conf": "",

  98.     },

  99.     "user_dir": {

  100.         "app": "synapse.app.generic_worker",

  101.         "listener_resources": ["client"],

  102.         "endpoint_patterns": [

  103.             "^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"

  104.         ],

  105.         "shared_extra_conf": {

  106.             "update_user_directory_from_worker": WORKER_PLACEHOLDER_NAME

  107.         },

  108.         "worker_extra_conf": "",

  109.     },

  110.     "media_repository": {

  111.         "app": "synapse.app.generic_worker",

  112.         "listener_resources": ["media"],

  113.         "endpoint_patterns": [

  114.             "^/_matrix/media/",

  115.             "^/_synapse/admin/v1/purge_media_cache$",

  116.             "^/_synapse/admin/v1/room/.*/media.*$",

  117.             "^/_synapse/admin/v1/user/.*/media.*$",

  118.             "^/_synapse/admin/v1/media/.*$",

  119.             "^/_synapse/admin/v1/quarantine_media/.*$",

  120.         ],

  121.         # The first configured media worker will run the media background jobs

  122.         "shared_extra_conf": {

  123.             "enable_media_repo": False,

  124.             "media_instance_running_background_jobs": WORKER_PLACEHOLDER_NAME,

  125.         },

  126.         "worker_extra_conf": "enable_media_repo: true",

  127.     },

  128.     "appservice": {

  129.         "app": "synapse.app.generic_worker",

  130.         "listener_resources": [],

  131.         "endpoint_patterns": [],

  132.         "shared_extra_conf": {

  133.             "notify_appservices_from_worker": WORKER_PLACEHOLDER_NAME

  134.         },

  135.         "worker_extra_conf": "",

  136.     },

  137.     "federation_sender": {

  138.         "app": "synapse.app.generic_worker",

  139.         "listener_resources": [],

  140.         "endpoint_patterns": [],

  141.         "shared_extra_conf": {},

  142.         "worker_extra_conf": "",

  143.     },

  144.     "synchrotron": {

  145.         "app": "synapse.app.generic_worker",

  146.         "listener_resources": ["client"],

  147.         "endpoint_patterns": [

  148.             "^/_matrix/client/(v2_alpha|r0|v3)/sync$",

  149.             "^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$",

  150.             "^/_matrix/client/(api/v1|r0|v3)/initialSync$",

  151.             "^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$",

  152.         ],

  153.         "shared_extra_conf": {},

  154.         "worker_extra_conf": "",

  155.     },

  156.     "client_reader": {

  157.         "app": "synapse.app.generic_worker",

  158.         "listener_resources": ["client"],

  159.         "endpoint_patterns": [

  160.             "^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",

  161.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$",

  162.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$",

  163.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$",

  164.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$",

  165.             "^/_matrix/client/v1/rooms/.*/hierarchy$",

  166.             "^/_matrix/client/(v1|unstable)/rooms/.*/relations/",

  167.             "^/_matrix/client/v1/rooms/.*/threads$",

  168.             "^/_matrix/client/(api/v1|r0|v3|unstable)/login$",

  169.             "^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$",

  170.             "^/_matrix/client/(api/v1|r0|v3|unstable)/account/whoami$",

  171.             "^/_matrix/client/versions$",

  172.             "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",

  173.             "^/_matrix/client/(r0|v3|unstable)/register$",

  174.             "^/_matrix/client/(r0|v3|unstable)/register/available$",

  175.             "^/_matrix/client/(r0|v3|unstable)/auth/.*/fallback/web$",

  176.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",

  177.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event",

  178.             "^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms",

  179.             "^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases",

  180.             "^/_matrix/client/v1/rooms/.*/timestamp_to_event$",

  181.             "^/_matrix/client/(api/v1|r0|v3|unstable)/search",

  182.             "^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)",

  183.             "^/_matrix/client/(r0|v3|unstable)/password_policy$",

  184.             "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",

  185.             "^/_matrix/client/(r0|v3|unstable)/capabilities$",

  186.             "^/_matrix/client/(r0|v3|unstable)/notifications$",

  187.         ],

  188.         "shared_extra_conf": {},

  189.         "worker_extra_conf": "",

  190.     },

  191.     "federation_reader": {

  192.         "app": "synapse.app.generic_worker",

  193.         "listener_resources": ["federation"],

  194.         "endpoint_patterns": [

  195.             "^/_matrix/federation/(v1|v2)/event/",

  196.             "^/_matrix/federation/(v1|v2)/state/",

  197.             "^/_matrix/federation/(v1|v2)/state_ids/",

  198.             "^/_matrix/federation/(v1|v2)/backfill/",

  199.             "^/_matrix/federation/(v1|v2)/get_missing_events/",

  200.             "^/_matrix/federation/(v1|v2)/publicRooms",

  201.             "^/_matrix/federation/(v1|v2)/query/",

  202.             "^/_matrix/federation/(v1|v2)/make_join/",

  203.             "^/_matrix/federation/(v1|v2)/make_leave/",

  204.             "^/_matrix/federation/(v1|v2)/send_join/",

  205.             "^/_matrix/federation/(v1|v2)/send_leave/",

  206.             "^/_matrix/federation/(v1|v2)/invite/",

  207.             "^/_matrix/federation/(v1|v2)/query_auth/",

  208.             "^/_matrix/federation/(v1|v2)/event_auth/",

  209.             "^/_matrix/federation/v1/timestamp_to_event/",

  210.             "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",

  211.             "^/_matrix/federation/(v1|v2)/user/devices/",

  212.             "^/_matrix/federation/(v1|v2)/get_groups_publicised$",

  213.             "^/_matrix/key/v2/query",

  214.         ],

  215.         "shared_extra_conf": {},

  216.         "worker_extra_conf": "",

  217.     },

  218.     "federation_inbound": {

  219.         "app": "synapse.app.generic_worker",

  220.         "listener_resources": ["federation"],

  221.         "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],

  222.         "shared_extra_conf": {},

  223.         "worker_extra_conf": "",

  224.     },

  225.     "event_persister": {

  226.         "app": "synapse.app.generic_worker",

  227.         "listener_resources": ["replication"],

  228.         "endpoint_patterns": [],

  229.         "shared_extra_conf": {},

  230.         "worker_extra_conf": "",

  231.     },

  232.     "background_worker": {

  233.         "app": "synapse.app.generic_worker",

  234.         "listener_resources": [],

  235.         "endpoint_patterns": [],

  236.         # This worker cannot be sharded. Therefore, there should only ever be one

  237.         # background worker. This is enforced for the safety of your database.

  238.         "shared_extra_conf": {"run_background_tasks_on": WORKER_PLACEHOLDER_NAME},

  239.         "worker_extra_conf": "",

  240.     },

  241.     "event_creator": {

  242.         "app": "synapse.app.generic_worker",

  243.         "listener_resources": ["client"],

  244.         "endpoint_patterns": [

  245.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",

  246.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",

  247.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",

  248.             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",

  249.             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",

  250.             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",

  251.         ],

  252.         "shared_extra_conf": {},

  253.         "worker_extra_conf": "",

  254.     },

  255.     "frontend_proxy": {

  256.         "app": "synapse.app.generic_worker",

  257.         "listener_resources": ["client", "replication"],

  258.         "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"],

  259.         "shared_extra_conf": {},

  260.         "worker_extra_conf": "",

  261.     },

  262.     "account_data": {

  263.         "app": "synapse.app.generic_worker",

  264.         "listener_resources": ["client", "replication"],

  265.         "endpoint_patterns": [

  266.             "^/_matrix/client/(r0|v3|unstable)/.*/tags",

  267.             "^/_matrix/client/(r0|v3|unstable)/.*/account_data",

  268.         ],

  269.         "shared_extra_conf": {},

  270.         "worker_extra_conf": "",

  271.     },

  272.     "presence": {

  273.         "app": "synapse.app.generic_worker",

  274.         "listener_resources": ["client", "replication"],

  275.         "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"],

  276.         "shared_extra_conf": {},

  277.         "worker_extra_conf": "",

  278.     },

  279.     "receipts": {

  280.         "app": "synapse.app.generic_worker",

  281.         "listener_resources": ["client", "replication"],

  282.         "endpoint_patterns": [

  283.             "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",

  284.             "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers",

  285.         ],

  286.         "shared_extra_conf": {},

  287.         "worker_extra_conf": "",

  288.     },

  289.     "to_device": {

  290.         "app": "synapse.app.generic_worker",

  291.         "listener_resources": ["client", "replication"],

  292.         "endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"],

  293.         "shared_extra_conf": {},

  294.         "worker_extra_conf": "",

  295.     },

  296.     "typing": {

  297.         "app": "synapse.app.generic_worker",

  298.         "listener_resources": ["client", "replication"],

  299.         "endpoint_patterns": [

  300.             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing"

  301.         ],

  302.         "shared_extra_conf": {},

  303.         "worker_extra_conf": "",

  304.     },

  305. }

  306. 

  307. # Templates for sections that may be inserted multiple times in config files

  308. NGINX_LOCATION_CONFIG_BLOCK = """

  309.     location ~* {endpoint} {{

  310.         proxy_pass {upstream};

  311.         proxy_set_header X-Forwarded-For $remote_addr;

  312.         proxy_set_header X-Forwarded-Proto $scheme;

  313.         proxy_set_header Host $host;

  314.     }}

  315. """

  316. 

  317. NGINX_UPSTREAM_CONFIG_BLOCK = """

  318. upstream {upstream_worker_base_name} {{

  319. {body}

  320. }}

  321. """

  322. 

  323. 

  324. # Utility functions

  325. def log(txt: str) -> None:

  326.     print(txt)

  327. 

  328. 

  329. def error(txt: str) -> NoReturn:

  330.     print(txt, file=sys.stderr)

  331.     sys.exit(2)

  332. 

  333. 

  334. def flush_buffers() -> None:

  335.     sys.stdout.flush()

  336.     sys.stderr.flush()

  337. 

  338. 

  339. def convert(src: str, dst: str, **template_vars: object) -> None:

  340.     """Generate a file from a template

  341. 

  342.     Args:

  343.         src: Path to the input file.

  344.         dst: Path to write to.

  345.         template_vars: The arguments to replace placeholder variables in the template with.

  346.     """

  347.     # Read the template file

  348.     # We disable autoescape to prevent template variables from being escaped,

  349.     # as we're not using HTML.

  350.     env = Environment(loader=FileSystemLoader(os.path.dirname(src)), autoescape=False)

  351.     template = env.get_template(os.path.basename(src))

  352. 

  353.     # Generate a string from the template.

  354.     rendered = template.render(**template_vars)

  355. 

  356.     # Write the generated contents to a file

  357.     #

  358.     # We use append mode in case the files have already been written to by something else

  359.     # (for instance, as part of the instructions in a dockerfile).

  360.     with open(dst, "a") as outfile:

  361.         # In case the existing file doesn't end with a newline

  362.         outfile.write("\n")

  363. 

  364.         outfile.write(rendered)

  365. 

  366. 

  367. def add_worker_roles_to_shared_config(

  368.     shared_config: dict,

  369.     worker_types_set: Set[str],

  370.     worker_name: str,

  371.     worker_port: int,

  372. ) -> None:

  373.     """Given a dictionary representing a config file shared across all workers,

  374.     append appropriate worker information to it for the current worker_type instance.

  375. 

  376.     Args:

  377.         shared_config: The config dict that all worker instances share (after being

  378.             converted to YAML)

  379.         worker_types_set: The type of worker (one of those defined in WORKERS_CONFIG).

  380.             This list can be a single worker type or multiple.

  381.         worker_name: The name of the worker instance.

  382.         worker_port: The HTTP replication port that the worker instance is listening on.

  383.     """

  384.     # The instance_map config field marks the workers that write to various replication

  385.     # streams

  386.     instance_map = shared_config.setdefault("instance_map", {})

  387. 

  388.     # This is a list of the stream_writers that there can be only one of. Events can be

  389.     # sharded, and therefore doesn't belong here.

  390.     singular_stream_writers = [

  391.         "account_data",

  392.         "presence",

  393.         "receipts",

  394.         "to_device",

  395.         "typing",

  396.     ]

  397. 

  398.     # Worker-type specific sharding config. Now a single worker can fulfill multiple

  399.     # roles, check each.

  400.     if "pusher" in worker_types_set:

  401.         shared_config.setdefault("pusher_instances", []).append(worker_name)

  402. 

  403.     if "federation_sender" in worker_types_set:

  404.         shared_config.setdefault("federation_sender_instances", []).append(worker_name)

  405. 

  406.     if "event_persister" in worker_types_set:

  407.         # Event persisters write to the events stream, so we need to update

  408.         # the list of event stream writers

  409.         shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(

  410.             worker_name

  411.         )

  412. 

  413.         # Map of stream writer instance names to host/ports combos

  414.         if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):

  415.             instance_map[worker_name] = {

  416.                 "path": f"/run/worker.{worker_port}",

  417.             }

  418.         else:

  419.             instance_map[worker_name] = {

  420.                 "host": "localhost",

  421.                 "port": worker_port,

  422.             }

  423.     # Update the list of stream writers. It's convenient that the name of the worker

  424.     # type is the same as the stream to write. Iterate over the whole list in case there

  425.     # is more than one.

  426.     for worker in worker_types_set:

  427.         if worker in singular_stream_writers:

  428.             shared_config.setdefault("stream_writers", {}).setdefault(

  429.                 worker, []

  430.             ).append(worker_name)

  431. 

  432.             # Map of stream writer instance names to host/ports combos

  433.             # For now, all stream writers need http replication ports

  434.             if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):

  435.                 instance_map[worker_name] = {

  436.                     "path": f"/run/worker.{worker_port}",

  437.                 }

  438.             else:

  439.                 instance_map[worker_name] = {

  440.                     "host": "localhost",

  441.                     "port": worker_port,

  442.                 }

  443. 

  444. 

  445. def merge_worker_template_configs(

  446.     existing_dict: Optional[Dict[str, Any]],

  447.     to_be_merged_dict: Dict[str, Any],

  448. ) -> Dict[str, Any]:

  449.     """When given an existing dict of worker template configuration consisting with both

  450.         dicts and lists, merge new template data from WORKERS_CONFIG(or create) and

  451.         return new dict.

  452. 

  453.     Args:

  454.         existing_dict: Either an existing worker template or a fresh blank one.

  455.         to_be_merged_dict: The template from WORKERS_CONFIGS to be merged into

  456.             existing_dict.

  457.     Returns: The newly merged together dict values.

  458.     """

  459.     new_dict: Dict[str, Any] = {}

  460.     if not existing_dict:

  461.         # It doesn't exist yet, just use the new dict(but take a copy not a reference)

  462.         new_dict = to_be_merged_dict.copy()

  463.     else:

  464.         for i in to_be_merged_dict.keys():

  465.             if (i == "endpoint_patterns") or (i == "listener_resources"):

  466.                 # merge the two lists, remove duplicates

  467.                 new_dict[i] = list(set(existing_dict[i] + to_be_merged_dict[i]))

  468.             elif i == "shared_extra_conf":

  469.                 # merge dictionary's, the worker name will be replaced later

  470.                 new_dict[i] = {**existing_dict[i], **to_be_merged_dict[i]}

  471.             elif i == "worker_extra_conf":

  472.                 # There is only one worker type that has a 'worker_extra_conf' and it is

  473.                 # the media_repo. Since duplicate worker types on the same worker don't

  474.                 # work, this is fine.

  475.                 new_dict[i] = existing_dict[i] + to_be_merged_dict[i]

  476.             else:

  477.                 # Everything else should be identical, like "app", which only works

  478.                 # because all apps are now generic_workers.

  479.                 new_dict[i] = to_be_merged_dict[i]

  480.     return new_dict

  481. 

  482. 

  483. def insert_worker_name_for_worker_config(

  484.     existing_dict: Dict[str, Any], worker_name: str

  485. ) -> Dict[str, Any]:

  486.     """Insert a given worker name into the worker's configuration dict.

  487. 

  488.     Args:

  489.         existing_dict: The worker_config dict that is imported into shared_config.

  490.         worker_name: The name of the worker to insert.

  491.     Returns: Copy of the dict with newly inserted worker name

  492.     """

  493.     dict_to_edit = existing_dict.copy()

  494.     for k, v in dict_to_edit["shared_extra_conf"].items():

  495.         # Only proceed if it's the placeholder name string

  496.         if v == WORKER_PLACEHOLDER_NAME:

  497.             dict_to_edit["shared_extra_conf"][k] = worker_name

  498.     return dict_to_edit

  499. 

  500. 

  501. def apply_requested_multiplier_for_worker(worker_types: List[str]) -> List[str]:

  502.     """

  503.     Apply multiplier(if found) by returning a new expanded list with some basic error

  504.     checking.

  505. 

  506.     Args:

  507.         worker_types: The unprocessed List of requested workers

  508.     Returns:

  509.         A new list with all requested workers expanded.

  510.     """

  511.     # Checking performed:

  512.     # 1. if worker:2 or more is declared, it will create additional workers up to number

  513.     # 2. if worker:1, it will create a single copy of this worker as if no number was

  514.     #   given

  515.     # 3. if worker:0 is declared, this worker will be ignored. This is to allow for

  516.     #   scripting and automated expansion and is intended behaviour.

  517.     # 4. if worker:NaN or is a negative number, it will error and log it.

  518.     new_worker_types = []

  519.     for worker_type in worker_types:

  520.         if ":" in worker_type:

  521.             worker_type_components = split_and_strip_string(worker_type, ":", 1)

  522.             worker_count = 0

  523.             # Should only be 2 components, a type of worker(s) and an integer as a

  524.             # string. Cast the number as an int then it can be used as a counter.

  525.             try:

  526.                 worker_count = int(worker_type_components[1])

  527.             except ValueError:

  528.                 error(

  529.                     f"Bad number in worker count for '{worker_type}': "

  530.                     f"'{worker_type_components[1]}' is not an integer"

  531.                 )

  532. 

  533.             # As long as there are more than 0, we add one to the list to make below.

  534.             for _ in range(worker_count):

  535.                 new_worker_types.append(worker_type_components[0])

  536. 

  537.         else:

  538.             # If it's not a real worker_type, it will error out later.

  539.             new_worker_types.append(worker_type)

  540.     return new_worker_types

  541. 

  542. 

  543. def is_sharding_allowed_for_worker_type(worker_type: str) -> bool:

  544.     """Helper to check to make sure worker types that cannot have multiples do not.

  545. 

  546.     Args:

  547.         worker_type: The type of worker to check against.

  548.     Returns: True if allowed, False if not

  549.     """

  550.     return worker_type not in [

  551.         "background_worker",

  552.         "account_data",

  553.         "presence",

  554.         "receipts",

  555.         "typing",

  556.         "to_device",

  557.     ]

  558. 

  559. 

  560. def split_and_strip_string(

  561.     given_string: str, split_char: str, max_split: SupportsIndex = -1

  562. ) -> List[str]:

  563.     """

  564.     Helper to split a string on split_char and strip whitespace from each end of each

  565.         element.

  566.     Args:

  567.         given_string: The string to split

  568.         split_char: The character to split the string on

  569.         max_split: kwarg for split() to limit how many times the split() happens

  570.     Returns:

  571.         A List of strings

  572.     """

  573.     # Removes whitespace from ends of result strings before adding to list. Allow for

  574.     # overriding 'maxsplit' kwarg, default being -1 to signify no maximum.

  575.     return [x.strip() for x in given_string.split(split_char, maxsplit=max_split)]

  576. 

  577. 

  578. def generate_base_homeserver_config() -> None:

  579.     """Starts Synapse and generates a basic homeserver config, which will later be

  580.     modified for worker support.

  581. 

  582.     Raises: CalledProcessError if calling start.py returned a non-zero exit code.

  583.     """

  584.     # start.py already does this for us, so just call that.

  585.     # note that this script is copied in in the official, monolith dockerfile

  586.     os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)

  587.     subprocess.run(["/usr/local/bin/python", "/start.py", "migrate_config"], check=True)

  588. 

  589. 

  590. def parse_worker_types(

  591.     requested_worker_types: List[str],

  592. ) -> Dict[str, Set[str]]:

  593.     """Read the desired list of requested workers and prepare the data for use in

  594.         generating worker config files while also checking for potential gotchas.

  595. 

  596.     Args:

  597.         requested_worker_types: The list formed from the split environment variable

  598.             containing the unprocessed requests for workers.

  599. 

  600.     Returns: A dict of worker names to set of worker types. Format:

  601.         {'worker_name':

  602.             {'worker_type', 'worker_type2'}

  603.         }

  604.     """

  605.     # A counter of worker_base_name -> int. Used for determining the name for a given

  606.     # worker when generating its config file, as each worker's name is just

  607.     # worker_base_name followed by instance number

  608.     worker_base_name_counter: Dict[str, int] = defaultdict(int)

  609. 

  610.     # Similar to above, but more finely grained. This is used to determine we don't have

  611.     # more than a single worker for cases where multiples would be bad(e.g. presence).

  612.     worker_type_shard_counter: Dict[str, int] = defaultdict(int)

  613. 

  614.     # The final result of all this processing

  615.     dict_to_return: Dict[str, Set[str]] = {}

  616. 

  617.     # Handle any multipliers requested for given workers.

  618.     multiple_processed_worker_types = apply_requested_multiplier_for_worker(

  619.         requested_worker_types

  620.     )

  621. 

  622.     # Process each worker_type_string

  623.     # Examples of expected formats:

  624.     #  - requested_name=type1+type2+type3

  625.     #  - synchrotron

  626.     #  - event_creator+event_persister

  627.     for worker_type_string in multiple_processed_worker_types:

  628.         # First, if a name is requested, use that — otherwise generate one.

  629.         worker_base_name: str = ""

  630.         if "=" in worker_type_string:

  631.             # Split on "=", remove extra whitespace from ends then make list

  632.             worker_type_split = split_and_strip_string(worker_type_string, "=")

  633.             if len(worker_type_split) > 2:

  634.                 error(

  635.                     "There should only be one '=' in the worker type string. "

  636.                     f"Please fix: {worker_type_string}"

  637.                 )

  638. 

  639.             # Assign the name

  640.             worker_base_name = worker_type_split[0]

  641. 

  642.             if not re.match(r"^[a-zA-Z0-9_+-]*[a-zA-Z_+-]$", worker_base_name):

  643.                 # Apply a fairly narrow regex to the worker names. Some characters

  644.                 # aren't safe for use in file paths or nginx configurations.

  645.                 # Don't allow to end with a number because we'll add a number

  646.                 # ourselves in a moment.

  647.                 error(

  648.                     "Invalid worker name; please choose a name consisting of "

  649.                     "alphanumeric letters, _ + -, but not ending with a digit: "

  650.                     f"{worker_base_name!r}"

  651.                 )

  652. 

  653.             # Continue processing the remainder of the worker_type string

  654.             # with the name override removed.

  655.             worker_type_string = worker_type_split[1]

  656. 

  657.         # Split the worker_type_string on "+", remove whitespace from ends then make

  658.         # the list a set so it's deduplicated.

  659.         worker_types_set: Set[str] = set(

  660.             split_and_strip_string(worker_type_string, "+")

  661.         )

  662. 

  663.         if not worker_base_name:

  664.             # No base name specified: generate one deterministically from set of

  665.             # types

  666.             worker_base_name = "+".join(sorted(worker_types_set))

  667. 

  668.         # At this point, we have:

  669.         #   worker_base_name which is the name for the worker, without counter.

  670.         #   worker_types_set which is the set of worker types for this worker.

  671. 

  672.         # Validate worker_type and make sure we don't allow sharding for a worker type

  673.         # that doesn't support it. Will error and stop if it is a problem,

  674.         # e.g. 'background_worker'.

  675.         for worker_type in worker_types_set:

  676.             # Verify this is a real defined worker type. If it's not, stop everything so

  677.             # it can be fixed.

  678.             if worker_type not in WORKERS_CONFIG:

  679.                 error(

  680.                     f"{worker_type} is an unknown worker type! Was found in "

  681.                     f"'{worker_type_string}'. Please fix!"

  682.                 )

  683. 

  684.             if worker_type in worker_type_shard_counter:

  685.                 if not is_sharding_allowed_for_worker_type(worker_type):

  686.                     error(

  687.                         f"There can be only a single worker with {worker_type} "

  688.                         "type. Please recount and remove."

  689.                     )

  690.             # Not in shard counter, must not have seen it yet, add it.

  691.             worker_type_shard_counter[worker_type] += 1

  692. 

  693.         # Generate the number for the worker using incrementing counter

  694.         worker_base_name_counter[worker_base_name] += 1

  695.         worker_number = worker_base_name_counter[worker_base_name]

  696.         worker_name = f"{worker_base_name}{worker_number}"

  697. 

  698.         if worker_number > 1:

  699.             # If this isn't the first worker, check that we don't have a confusing

  700.             # mixture of worker types with the same base name.

  701.             first_worker_with_base_name = dict_to_return[f"{worker_base_name}1"]

  702.             if first_worker_with_base_name != worker_types_set:

  703.                 error(

  704.                     f"Can not use worker_name: '{worker_name}' for worker_type(s): "

  705.                     f"{worker_types_set!r}. It is already in use by "

  706.                     f"worker_type(s): {first_worker_with_base_name!r}"

  707.                 )

  708. 

  709.         dict_to_return[worker_name] = worker_types_set

  710. 

  711.     return dict_to_return

  712. 

  713. 

  714. def generate_worker_files(

  715.     environ: Mapping[str, str],

  716.     config_path: str,

  717.     data_dir: str,

  718.     requested_worker_types: Dict[str, Set[str]],

  719. ) -> None:

  720.     """Read the desired workers(if any) that is passed in and generate shared

  721.         homeserver, nginx and supervisord configs.

  722. 

  723.     Args:

  724.         environ: os.environ instance.

  725.         config_path: The location of the generated Synapse main worker config file.

  726.         data_dir: The location of the synapse data directory. Where log and

  727.             user-facing config files live.

  728.         requested_worker_types: A Dict containing requested workers in the format of

  729.             {'worker_name1': {'worker_type', ...}}

  730.     """

  731.     # Note that yaml cares about indentation, so care should be taken to insert lines

  732.     # into files at the correct indentation below.

  733. 

  734.     # Convenience helper for if using unix sockets instead of host:port

  735.     using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)

  736.     # First read the original config file and extract the listeners block. Then we'll

  737.     # add another listener for replication. Later we'll write out the result to the

  738.     # shared config file.

  739.     listeners: List[Any]

  740.     if using_unix_sockets:

  741.         listeners = [

  742.             {

  743.                 "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,

  744.                 "type": "http",

  745.                 "resources": [{"names": ["replication"]}],

  746.             }

  747.         ]

  748.     else:

  749.         listeners = [

  750.             {

  751.                 "port": MAIN_PROCESS_REPLICATION_PORT,

  752.                 "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,

  753.                 "type": "http",

  754.                 "resources": [{"names": ["replication"]}],

  755.             }

  756.         ]

  757.     with open(config_path) as file_stream:

  758.         original_config = yaml.safe_load(file_stream)

  759.         original_listeners = original_config.get("listeners")

  760.         if original_listeners:

  761.             listeners += original_listeners

  762. 

  763.     # The shared homeserver config. The contents of which will be inserted into the

  764.     # base shared worker jinja2 template. This config file will be passed to all

  765.     # workers, included Synapse's main process. It is intended mainly for disabling

  766.     # functionality when certain workers are spun up, and adding a replication listener.

  767.     shared_config: Dict[str, Any] = {"listeners": listeners}

  768. 

  769.     # List of dicts that describe workers.

  770.     # We pass this to the Supervisor template later to generate the appropriate

  771.     # program blocks.

  772.     worker_descriptors: List[Dict[str, Any]] = []

  773. 

  774.     # Upstreams for load-balancing purposes. This dict takes the form of the worker

  775.     # type to the ports of each worker. For example:

  776.     # {

  777.     #   worker_type: {1234, 1235, ...}}

  778.     # }

  779.     # and will be used to construct 'upstream' nginx directives.

  780.     nginx_upstreams: Dict[str, Set[int]] = {}

  781. 

  782.     # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what

  783.     # will be placed after the proxy_pass directive. The main benefit to representing

  784.     # this data as a dict over a str is that we can easily deduplicate endpoints

  785.     # across multiple instances of the same worker. The final rendering will be combined

  786.     # with nginx_upstreams and placed in /etc/nginx/conf.d.

  787.     nginx_locations: Dict[str, str] = {}

  788. 

  789.     # Create the worker configuration directory if it doesn't already exist

  790.     os.makedirs("/conf/workers", exist_ok=True)

  791. 

  792.     # Start worker ports from this arbitrary port

  793.     worker_port = 18009

  794. 

  795.     # A list of internal endpoints to healthcheck, starting with the main process

  796.     # which exists even if no workers do.

  797.     # This list ends up being part of the command line to curl, (curl added support for

  798.     # Unix sockets in version 7.40).

  799.     if using_unix_sockets:

  800.         healthcheck_urls = [

  801.             f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "

  802.             # The scheme and hostname from the following URL are ignored.

  803.             # The only thing that matters is the path `/health`

  804.             "http://localhost/health"

  805.         ]

  806.     else:

  807.         healthcheck_urls = ["http://localhost:8080/health"]

  808. 

  809.     # Get the set of all worker types that we have configured

  810.     all_worker_types_in_use = set(chain(*requested_worker_types.values()))

  811.     # Map locations to upstreams (corresponding to worker types) in Nginx

  812.     # but only if we use the appropriate worker type

  813.     for worker_type in all_worker_types_in_use:

  814.         for endpoint_pattern in WORKERS_CONFIG[worker_type]["endpoint_patterns"]:

  815.             nginx_locations[endpoint_pattern] = f"http://{worker_type}"

  816. 

  817.     # For each worker type specified by the user, create config values and write it's

  818.     # yaml config file

  819.     for worker_name, worker_types_set in requested_worker_types.items():

  820.         # The collected and processed data will live here.

  821.         worker_config: Dict[str, Any] = {}

  822. 

  823.         # Merge all worker config templates for this worker into a single config

  824.         for worker_type in worker_types_set:

  825.             copy_of_template_config = WORKERS_CONFIG[worker_type].copy()

  826. 

  827.             # Merge worker type template configuration data. It's a combination of lists

  828.             # and dicts, so use this helper.

  829.             worker_config = merge_worker_template_configs(

  830.                 worker_config, copy_of_template_config

  831.             )

  832. 

  833.         # Replace placeholder names in the config template with the actual worker name.

  834.         worker_config = insert_worker_name_for_worker_config(worker_config, worker_name)

  835. 

  836.         worker_config.update(

  837.             {"name": worker_name, "port": str(worker_port), "config_path": config_path}

  838.         )

  839. 

  840.         # Update the shared config with any worker_type specific options. The first of a

  841.         # given worker_type needs to stay assigned and not be replaced.

  842.         worker_config["shared_extra_conf"].update(shared_config)

  843.         shared_config = worker_config["shared_extra_conf"]

  844.         if using_unix_sockets:

  845.             healthcheck_urls.append(

  846.                 f"--unix-socket /run/worker.{worker_port} http://localhost/health"

  847.             )

  848.         else:

  849.             healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))

  850. 

  851.         # Update the shared config with sharding-related options if necessary

  852.         add_worker_roles_to_shared_config(

  853.             shared_config, worker_types_set, worker_name, worker_port

  854.         )

  855. 

  856.         # Enable the worker in supervisord

  857.         worker_descriptors.append(worker_config)

  858. 

  859.         # Write out the worker's logging config file

  860.         log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)

  861. 

  862.         # Then a worker config file

  863.         convert(

  864.             "/conf/worker.yaml.j2",

  865.             f"/conf/workers/{worker_name}.yaml",

  866.             **worker_config,

  867.             worker_log_config_filepath=log_config_filepath,

  868.             using_unix_sockets=using_unix_sockets,

  869.         )

  870. 

  871.         # Save this worker's port number to the correct nginx upstreams

  872.         for worker_type in worker_types_set:

  873.             nginx_upstreams.setdefault(worker_type, set()).add(worker_port)

  874. 

  875.         worker_port += 1

  876. 

  877.     # Build the nginx location config blocks

  878.     nginx_location_config = ""

  879.     for endpoint, upstream in nginx_locations.items():

  880.         nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(

  881.             endpoint=endpoint,

  882.             upstream=upstream,

  883.         )

  884. 

  885.     # Determine the load-balancing upstreams to configure

  886.     nginx_upstream_config = ""

  887.     for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():

  888.         body = ""

  889.         if using_unix_sockets:

  890.             for port in upstream_worker_ports:

  891.                 body += f"    server unix:/run/worker.{port};\n"

  892. 

  893.         else:

  894.             for port in upstream_worker_ports:

  895.                 body += f"    server localhost:{port};\n"

  896. 

  897.         # Add to the list of configured upstreams

  898.         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(

  899.             upstream_worker_base_name=upstream_worker_base_name,

  900.             body=body,

  901.         )

  902. 

  903.     # Finally, we'll write out the config files.

  904. 

  905.     # log config for the master process

  906.     master_log_config = generate_worker_log_config(environ, "master", data_dir)

  907.     shared_config["log_config"] = master_log_config

  908. 

  909.     # Find application service registrations

  910.     appservice_registrations = None

  911.     appservice_registration_dir = os.environ.get("SYNAPSE_AS_REGISTRATION_DIR")

  912.     if appservice_registration_dir:

  913.         # Scan for all YAML files that should be application service registrations.

  914.         appservice_registrations = [

  915.             str(reg_path.resolve())

  916.             for reg_path in Path(appservice_registration_dir).iterdir()

  917.             if reg_path.suffix.lower() in (".yaml", ".yml")

  918.         ]

  919. 

  920.     workers_in_use = len(requested_worker_types) > 0

  921. 

  922.     # If there are workers, add the main process to the instance_map too.

  923.     if workers_in_use:

  924.         instance_map = shared_config.setdefault("instance_map", {})

  925.         if using_unix_sockets:

  926.             instance_map[MAIN_PROCESS_INSTANCE_NAME] = {

  927.                 "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,

  928.             }

  929.         else:

  930.             instance_map[MAIN_PROCESS_INSTANCE_NAME] = {

  931.                 "host": MAIN_PROCESS_LOCALHOST_ADDRESS,

  932.                 "port": MAIN_PROCESS_REPLICATION_PORT,

  933.             }

  934. 

  935.     # Shared homeserver config

  936.     convert(

  937.         "/conf/shared.yaml.j2",

  938.         "/conf/workers/shared.yaml",

  939.         shared_worker_config=yaml.dump(shared_config),

  940.         appservice_registrations=appservice_registrations,

  941.         enable_redis=workers_in_use,

  942.         workers_in_use=workers_in_use,

  943.         using_unix_sockets=using_unix_sockets,

  944.     )

  945. 

  946.     # Nginx config

  947.     convert(

  948.         "/conf/nginx.conf.j2",

  949.         "/etc/nginx/conf.d/matrix-synapse.conf",

  950.         worker_locations=nginx_location_config,

  951.         upstream_directives=nginx_upstream_config,

  952.         tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),

  953.         tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),

  954.         using_unix_sockets=using_unix_sockets,

  955.     )

  956. 

  957.     # Supervisord config

  958.     os.makedirs("/etc/supervisor", exist_ok=True)

  959.     convert(

  960.         "/conf/supervisord.conf.j2",

  961.         "/etc/supervisor/supervisord.conf",

  962.         main_config_path=config_path,

  963.         enable_redis=workers_in_use,

  964.         using_unix_sockets=using_unix_sockets,

  965.     )

  966. 

  967.     convert(

  968.         "/conf/synapse.supervisord.conf.j2",

  969.         "/etc/supervisor/conf.d/synapse.conf",

  970.         workers=worker_descriptors,

  971.         main_config_path=config_path,

  972.         use_forking_launcher=environ.get("SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"),

  973.     )

  974. 

  975.     # healthcheck config

  976.     convert(

  977.         "/conf/healthcheck.sh.j2",

  978.         "/healthcheck.sh",

  979.         healthcheck_urls=healthcheck_urls,

  980.     )

  981. 

  982.     # Ensure the logging directory exists

  983.     log_dir = data_dir + "/logs"

  984.     if not os.path.exists(log_dir):

  985.         os.mkdir(log_dir)

  986. 

  987. 

  988. def generate_worker_log_config(

  989.     environ: Mapping[str, str], worker_name: str, data_dir: str

  990. ) -> str:

  991.     """Generate a log.config file for the given worker.

  992. 

  993.     Returns: the path to the generated file

  994.     """

  995.     # Check whether we should write worker logs to disk, in addition to the console

  996.     extra_log_template_args: Dict[str, Optional[str]] = {}

  997.     if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):

  998.         extra_log_template_args["LOG_FILE_PATH"] = f"{data_dir}/logs/{worker_name}.log"

  999. 

  1000.     extra_log_template_args["SYNAPSE_LOG_LEVEL"] = environ.get("SYNAPSE_LOG_LEVEL")

  1001.     extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(

  1002.         "SYNAPSE_LOG_SENSITIVE"

  1003.     )

  1004.     extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")

  1005. 

  1006.     # Render and write the file

  1007.     log_config_filepath = f"/conf/workers/{worker_name}.log.config"

  1008.     convert(

  1009.         "/conf/log.config",

  1010.         log_config_filepath,

  1011.         worker_name=worker_name,

  1012.         **extra_log_template_args,

  1013.         include_worker_name_in_log_line=environ.get(

  1014.             "SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"

  1015.         ),

  1016.     )

  1017.     return log_config_filepath

  1018. 

  1019. 

  1020. def main(args: List[str], environ: MutableMapping[str, str]) -> None:

  1021.     config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")

  1022.     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")

  1023.     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")

  1024. 

  1025.     # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,

  1026.     # this needs to be handled by a frontend proxy

  1027.     environ["SYNAPSE_NO_TLS"] = "yes"

  1028. 

  1029.     # Generate the base homeserver config if one does not yet exist

  1030.     if not os.path.exists(config_path):

  1031.         log("Generating base homeserver config")

  1032.         generate_base_homeserver_config()

  1033.     else:

  1034.         log("Base homeserver config exists—not regenerating")

  1035.     # This script may be run multiple times (mostly by Complement, see note at top of

  1036.     # file). Don't re-configure workers in this instance.

  1037.     mark_filepath = "/conf/workers_have_been_configured"

  1038.     if not os.path.exists(mark_filepath):

  1039.         # Collect and validate worker_type requests

  1040.         # Read the desired worker configuration from the environment

  1041.         worker_types_env = environ.get("SYNAPSE_WORKER_TYPES", "").strip()

  1042.         # Only process worker_types if they exist

  1043.         if not worker_types_env:

  1044.             # No workers, just the main process

  1045.             worker_types = []

  1046.             requested_worker_types: Dict[str, Any] = {}

  1047.         else:

  1048.             # Split type names by comma, ignoring whitespace.

  1049.             worker_types = split_and_strip_string(worker_types_env, ",")

  1050.             requested_worker_types = parse_worker_types(worker_types)

  1051. 

  1052.         # Always regenerate all other config files

  1053.         log("Generating worker config files")

  1054.         generate_worker_files(environ, config_path, data_dir, requested_worker_types)

  1055. 

  1056.         # Mark workers as being configured

  1057.         with open(mark_filepath, "w") as f:

  1058.             f.write("")

  1059.     else:

  1060.         log("Worker config exists—not regenerating")

  1061. 

  1062.     # Lifted right out of start.py

  1063.     jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)

  1064. 

  1065.     if os.path.isfile(jemallocpath):

  1066.         environ["LD_PRELOAD"] = jemallocpath

  1067.     else:

  1068.         log("Could not find %s, will not use" % (jemallocpath,))

  1069. 

  1070.     # Start supervisord, which will start Synapse, all of the configured worker

  1071.     # processes, redis, nginx etc. according to the config we created above.

  1072.     log("Starting supervisord")

  1073.     flush_buffers()

  1074.     os.execle(

  1075.         "/usr/local/bin/supervisord",

  1076.         "supervisord",

  1077.         "-c",

  1078.         "/etc/supervisor/supervisord.conf",

  1079.         environ,

  1080.     )

  1081. 

  1082. 

  1083. if __name__ == "__main__":

  1084.     main(sys.argv, os.environ)