# Copyright 2018 New Vector # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, cast from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, LoggingTransaction, make_in_list_sql_clause, ) from synapse.storage.databases.main.registration import RegistrationWorkerStore from synapse.util.caches.descriptors import cached from synapse.util.threepids import canonicalise_email if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger(__name__) # Number of msec of granularity to store the monthly_active_user timestamp # This means it is not necessary to update the table on every request LAST_SEEN_GRANULARITY = 60 * 60 * 1000 class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore): def __init__( self, database: DatabasePool, db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): super().__init__(database, db_conn, hs) self._clock = hs.get_clock() self.hs = hs if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to # the background worker self._update_on_this_worker = hs.config.worker.run_background_tasks else: # If we're NOT using Redis, this must be handled by the master self._update_on_this_worker = hs.get_instance_name() == "master" self._limit_usage_by_mau = hs.config.server.limit_usage_by_mau self._max_mau_value = hs.config.server.max_mau_value self._mau_stats_only = hs.config.server.mau_stats_only if self._update_on_this_worker: # Do not add more reserved users than the total allowable number self.db_pool.new_transaction( db_conn, "initialise_mau_threepids", [], [], [], self._initialise_reserved_users, hs.config.server.mau_limits_reserved_threepids[: self._max_mau_value], ) @cached(num_args=0) async def get_monthly_active_count(self) -> int: """Generates current count of monthly active users Returns: Number of current monthly active users """ def _count_users(txn: LoggingTransaction) -> int: # Exclude app service users sql = """ SELECT COUNT(*) FROM monthly_active_users LEFT JOIN users ON monthly_active_users.user_id=users.name WHERE (users.appservice_id IS NULL OR users.appservice_id = ''); """ txn.execute(sql) (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction("count_users", _count_users) @cached(num_args=0) async def get_monthly_active_count_by_service(self) -> Mapping[str, int]: """Generates current count of monthly active users broken down by service. A service is typically an appservice but also includes native matrix users. Since the `monthly_active_users` table is populated from the `user_ips` table `config.appservice.track_appservice_user_ips` must be set to `true` for this method to return anything other than native matrix users. Returns: A mapping between app_service_id and the number of occurrences. """ def _count_users_by_service(txn: LoggingTransaction) -> Dict[str, int]: sql = """ SELECT COALESCE(appservice_id, 'native'), COUNT(*) FROM monthly_active_users LEFT JOIN users ON monthly_active_users.user_id=users.name GROUP BY appservice_id; """ txn.execute(sql) result = cast(List[Tuple[str, int]], txn.fetchall()) return dict(result) return await self.db_pool.runInteraction( "count_users_by_service", _count_users_by_service ) async def get_monthly_active_users_by_service( self, start_timestamp: Optional[int] = None, end_timestamp: Optional[int] = None ) -> List[Tuple[str, str]]: """Generates list of monthly active users and their services. Please see "get_monthly_active_count_by_service" docstring for more details about services. Arguments: start_timestamp: If specified, only include users that were first active at or after this point end_timestamp: If specified, only include users that were first active at or before this point Returns: A list of tuples (appservice_id, user_id). "native" is emitted as the appservice for users that don't come from appservices (i.e. native Matrix users). """ if start_timestamp is not None and end_timestamp is not None: where_clause = 'WHERE "timestamp" >= ? and "timestamp" <= ?' query_params = [start_timestamp, end_timestamp] elif start_timestamp is not None: where_clause = 'WHERE "timestamp" >= ?' query_params = [start_timestamp] elif end_timestamp is not None: where_clause = 'WHERE "timestamp" <= ?' query_params = [end_timestamp] else: where_clause = "" query_params = [] def _list_users(txn: LoggingTransaction) -> List[Tuple[str, str]]: sql = f""" SELECT COALESCE(appservice_id, 'native'), user_id FROM monthly_active_users LEFT JOIN users ON monthly_active_users.user_id=users.name {where_clause}; """ txn.execute(sql, query_params) return cast(List[Tuple[str, str]], txn.fetchall()) return await self.db_pool.runInteraction("list_users", _list_users) async def get_registered_reserved_users(self) -> List[str]: """Of the reserved threepids defined in config, retrieve those that are associated with registered users Returns: User IDs of actual users that are reserved """ users = [] for tp in self.hs.config.server.mau_limits_reserved_threepids[ : self.hs.config.server.max_mau_value ]: user_id = await self.hs.get_datastores().main.get_user_id_by_threepid( tp["medium"], canonicalise_email(tp["address"]) ) if user_id: users.append(user_id) return users @cached(num_args=1) async def user_last_seen_monthly_active(self, user_id: str) -> Optional[int]: """ Checks if a given user is part of the monthly active user group Arguments: user_id: user to add/update Return: Timestamp since last seen, None if never seen """ return await self.db_pool.simple_select_one_onecol( table="monthly_active_users", keyvalues={"user_id": user_id}, retcol="timestamp", allow_none=True, desc="user_last_seen_monthly_active", ) @wrap_as_background_process("reap_monthly_active_users") async def reap_monthly_active_users(self) -> None: """Cleans out monthly active user table to ensure that no stale entries exist. """ def _reap_users(txn: LoggingTransaction, reserved_users: List[str]) -> None: """ Args: reserved_users: reserved users to preserve """ thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) in_clause, in_clause_args = make_in_list_sql_clause( self.database_engine, "user_id", reserved_users ) txn.execute( "DELETE FROM monthly_active_users WHERE timestamp < ? AND NOT %s" % (in_clause,), [thirty_days_ago] + in_clause_args, ) if self._limit_usage_by_mau: # If MAU user count still exceeds the MAU threshold, then delete on # a least recently active basis. # Note it is not possible to write this query using OFFSET due to # incompatibilities in how sqlite and postgres support the feature. # Sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present, # while Postgres does not require 'LIMIT', but also does not support # negative LIMIT values. So there is no way to write it that both can # support # Limit must be >= 0 for postgres num_of_non_reserved_users_to_remove = max( self._max_mau_value - len(reserved_users), 0 ) # It is important to filter reserved users twice to guard # against the case where the reserved user is present in the # SELECT, meaning that a legitimate mau is deleted. sql = """ DELETE FROM monthly_active_users WHERE user_id NOT IN ( SELECT user_id FROM monthly_active_users WHERE NOT %s ORDER BY timestamp DESC LIMIT ? ) AND NOT %s """ % ( in_clause, in_clause, ) query_args = ( in_clause_args + [num_of_non_reserved_users_to_remove] + in_clause_args ) txn.execute(sql, query_args) # It seems poor to invalidate the whole cache. Postgres supports # 'Returning' which would allow me to invalidate only the # specific users, but sqlite has no way to do this and instead # I would need to SELECT and the DELETE which without locking # is racy. # Have resolved to invalidate the whole cache for now and do # something about it if and when the perf becomes significant self._invalidate_all_cache_and_stream( txn, self.user_last_seen_monthly_active ) self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) reserved_users = await self.get_registered_reserved_users() await self.db_pool.runInteraction( "reap_monthly_active_users", _reap_users, reserved_users ) def _initialise_reserved_users( self, txn: LoggingTransaction, threepids: List[dict] ) -> None: """Ensures that reserved threepids are accounted for in the MAU table, should be called on start up. Args: txn: threepids: List of threepid dicts to reserve """ assert ( self._update_on_this_worker ), "This worker is not designated to update MAUs" # XXX what is this function trying to achieve? It upserts into # monthly_active_users for each *registered* reserved mau user, but why? # # - shouldn't there already be an entry for each reserved user (at least # if they have been active recently)? # # - if it's important that the timestamp is kept up to date, why do we only # run this at startup? for tp in threepids: user_id = self.get_user_id_by_threepid_txn(txn, tp["medium"], tp["address"]) if user_id: is_support = self.is_support_user_txn(txn, user_id) if not is_support: # We do this manually here to avoid hitting https://github.com/matrix-org/synapse/issues/6791 self.db_pool.simple_upsert_txn( txn, table="monthly_active_users", keyvalues={"user_id": user_id}, values={"timestamp": int(self._clock.time_msec())}, ) else: logger.warning("mau limit reserved threepid %s not found in db" % tp) async def upsert_monthly_active_user(self, user_id: str) -> None: """Updates or inserts the user into the monthly active user table, which is used to track the current MAU usage of the server Args: user_id: user to add/update """ assert ( self._update_on_this_worker ), "This worker is not designated to update MAUs" # Support user never to be included in MAU stats. Note I can't easily call this # from upsert_monthly_active_user_txn because then I need a _txn form of # is_support_user which is complicated because I want to cache the result. # Therefore I call it here and ignore the case where # upsert_monthly_active_user_txn is called directly from # _initialise_reserved_users reasoning that it would be very strange to # include a support user in this context. is_support = await self.is_support_user(user_id) if is_support: return await self.db_pool.runInteraction( "upsert_monthly_active_user", self.upsert_monthly_active_user_txn, user_id ) def upsert_monthly_active_user_txn( self, txn: LoggingTransaction, user_id: str ) -> None: """Updates or inserts monthly active user member We consciously do not call is_support_txn from this method because it is not possible to cache the response. is_support_txn will be false in almost all cases, so it seems reasonable to call it only for upsert_monthly_active_user and to call is_support_txn manually for cases where upsert_monthly_active_user_txn is called directly, like _initialise_reserved_users In short, don't call this method with support users. (Support users should not appear in the MAU stats). Args: txn: user_id: user to add/update """ assert ( self._update_on_this_worker ), "This worker is not designated to update MAUs" # Am consciously deciding to lock the table on the basis that is ought # never be a big table and alternative approaches (batching multiple # upserts into a single txn) introduced a lot of extra complexity. # See https://github.com/matrix-org/synapse/issues/3854 for more self.db_pool.simple_upsert_txn( txn, table="monthly_active_users", keyvalues={"user_id": user_id}, values={"timestamp": int(self._clock.time_msec())}, ) self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) self._invalidate_cache_and_stream( txn, self.get_monthly_active_count_by_service, () ) self._invalidate_cache_and_stream( txn, self.user_last_seen_monthly_active, (user_id,) ) async def populate_monthly_active_users(self, user_id: str) -> None: """Checks on the state of monthly active user limits and optionally add the user to the monthly active tables Args: user_id: the user_id to query """ assert ( self._update_on_this_worker ), "This worker is not designated to update MAUs" if self._limit_usage_by_mau or self._mau_stats_only: # Trial users and guests should not be included as part of MAU group is_guest = await self.is_guest(user_id) if is_guest: return is_trial = await self.is_trial_user(user_id) if is_trial: return last_seen_timestamp = await self.user_last_seen_monthly_active(user_id) now = self.hs.get_clock().time_msec() # We want to reduce to the total number of db writes, and are happy # to trade accuracy of timestamp in order to lighten load. This means # We always insert new users (where MAU threshold has not been reached), # but only update if we have not previously seen the user for # LAST_SEEN_GRANULARITY ms if last_seen_timestamp is None: # In the case where mau_stats_only is True and limit_usage_by_mau is # False, there is no point in checking get_monthly_active_count - it # adds no value and will break the logic if max_mau_value is exceeded. if not self._limit_usage_by_mau: await self.upsert_monthly_active_user(user_id) else: count = await self.get_monthly_active_count() if count < self._max_mau_value: await self.upsert_monthly_active_user(user_id) elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY: await self.upsert_monthly_active_user(user_id)