|
- # Copyright 2014-2016 OpenMarket Ltd
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- import datetime
- import itertools
- import logging
- from queue import Empty, PriorityQueue
- from typing import (
- TYPE_CHECKING,
- Collection,
- Dict,
- Iterable,
- List,
- Optional,
- Sequence,
- Set,
- Tuple,
- cast,
- )
-
- import attr
- from prometheus_client import Counter, Gauge
-
- from synapse.api.constants import MAX_DEPTH, EventTypes
- from synapse.api.errors import StoreError
- from synapse.api.room_versions import EventFormatVersions, RoomVersion
- from synapse.events import EventBase, make_event_from_dict
- from synapse.logging.opentracing import tag_args, trace
- from synapse.metrics.background_process_metrics import wrap_as_background_process
- from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
- from synapse.storage.database import (
- DatabasePool,
- LoggingDatabaseConnection,
- LoggingTransaction,
- )
- from synapse.storage.databases.main.events_worker import EventsWorkerStore
- from synapse.storage.databases.main.signatures import SignatureWorkerStore
- from synapse.storage.engines import PostgresEngine, Sqlite3Engine
- from synapse.types import JsonDict
- from synapse.util import json_encoder
- from synapse.util.caches.descriptors import cached
- from synapse.util.caches.lrucache import LruCache
- from synapse.util.cancellation import cancellable
- from synapse.util.iterutils import batch_iter
-
- if TYPE_CHECKING:
- from synapse.server import HomeServer
-
- oldest_pdu_in_federation_staging = Gauge(
- "synapse_federation_server_oldest_inbound_pdu_in_staging",
- "The age in seconds since we received the oldest pdu in the federation staging area",
- )
-
- number_pdus_in_federation_queue = Gauge(
- "synapse_federation_server_number_inbound_pdu_in_staging",
- "The total number of events in the inbound federation staging",
- )
-
- pdus_pruned_from_federation_queue = Counter(
- "synapse_federation_server_number_inbound_pdu_pruned",
- "The number of events in the inbound federation staging that have been "
- "pruned due to the queue getting too long",
- )
-
- logger = logging.getLogger(__name__)
-
- # Parameters controlling exponential backoff between backfill failures.
- # After the first failure to backfill, we wait 2 hours before trying again. If the
- # second attempt fails, we wait 4 hours before trying again. If the third attempt fails,
- # we wait 8 hours before trying again, ... and so on.
- #
- # Each successive backoff period is twice as long as the last. However we cap this
- # period at a maximum of 2^8 = 256 hours: a little over 10 days. (This is the smallest
- # power of 2 which yields a maximum backoff period of at least 7 days---which was the
- # original maximum backoff period.) Even when we hit this cap, we will continue to
- # make backfill attempts once every 10 days.
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS = 8
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS = int(
- datetime.timedelta(hours=1).total_seconds() * 1000
- )
-
- # We need a cap on the power of 2 or else the backoff period
- # 2^N * (milliseconds per hour)
- # will overflow when calcuated within the database. We ensure overflow does not occur
- # by checking that the largest backoff period fits in a 32-bit signed integer.
- _LONGEST_BACKOFF_PERIOD_MILLISECONDS = (
- 2**BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS
- ) * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS
- assert 0 < _LONGEST_BACKOFF_PERIOD_MILLISECONDS <= ((2**31) - 1)
-
-
- # All the info we need while iterating the DAG while backfilling
- @attr.s(frozen=True, slots=True, auto_attribs=True)
- class BackfillQueueNavigationItem:
- depth: int
- stream_ordering: int
- event_id: str
- type: str
-
-
- class _NoChainCoverIndex(Exception):
- def __init__(self, room_id: str):
- super().__init__("Unexpectedly no chain cover for events in %s" % (room_id,))
-
-
- class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBaseStore):
- def __init__(
- self,
- database: DatabasePool,
- db_conn: LoggingDatabaseConnection,
- hs: "HomeServer",
- ):
- super().__init__(database, db_conn, hs)
-
- self.hs = hs
-
- if hs.config.worker.run_background_tasks:
- hs.get_clock().looping_call(
- self._delete_old_forward_extrem_cache, 60 * 60 * 1000
- )
-
- # Cache of event ID to list of auth event IDs and their depths.
- self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache(
- 500000, "_event_auth_cache", size_callback=len
- )
-
- self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000)
-
- async def get_auth_chain(
- self, room_id: str, event_ids: Collection[str], include_given: bool = False
- ) -> List[EventBase]:
- """Get auth events for given event_ids. The events *must* be state events.
-
- Args:
- room_id: The room the event is in.
- event_ids: state events
- include_given: include the given events in result
-
- Returns:
- list of events
- """
- event_ids = await self.get_auth_chain_ids(
- room_id, event_ids, include_given=include_given
- )
- return await self.get_events_as_list(event_ids)
-
- @trace
- @tag_args
- async def get_auth_chain_ids(
- self,
- room_id: str,
- event_ids: Collection[str],
- include_given: bool = False,
- ) -> Set[str]:
- """Get auth events for given event_ids. The events *must* be state events.
-
- Args:
- room_id: The room the event is in.
- event_ids: state events
- include_given: include the given events in result
-
- Returns:
- set of event_ids
- """
-
- # Check if we have indexed the room so we can use the chain cover
- # algorithm.
- room = await self.get_room(room_id) # type: ignore[attr-defined]
- if room["has_auth_chain_index"]:
- try:
- return await self.db_pool.runInteraction(
- "get_auth_chain_ids_chains",
- self._get_auth_chain_ids_using_cover_index_txn,
- room_id,
- event_ids,
- include_given,
- )
- except _NoChainCoverIndex:
- # For whatever reason we don't actually have a chain cover index
- # for the events in question, so we fall back to the old method.
- pass
-
- return await self.db_pool.runInteraction(
- "get_auth_chain_ids",
- self._get_auth_chain_ids_txn,
- event_ids,
- include_given,
- )
-
- def _get_auth_chain_ids_using_cover_index_txn(
- self,
- txn: LoggingTransaction,
- room_id: str,
- event_ids: Collection[str],
- include_given: bool,
- ) -> Set[str]:
- """Calculates the auth chain IDs using the chain index."""
-
- # First we look up the chain ID/sequence numbers for the given events.
-
- initial_events = set(event_ids)
-
- # All the events that we've found that are reachable from the events.
- seen_events: Set[str] = set()
-
- # A map from chain ID to max sequence number of the given events.
- event_chains: Dict[int, int] = {}
-
- sql = """
- SELECT event_id, chain_id, sequence_number
- FROM event_auth_chains
- WHERE %s
- """
- for batch in batch_iter(initial_events, 1000):
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "event_id", batch
- )
- txn.execute(sql % (clause,), args)
-
- for event_id, chain_id, sequence_number in txn:
- seen_events.add(event_id)
- event_chains[chain_id] = max(
- sequence_number, event_chains.get(chain_id, 0)
- )
-
- # Check that we actually have a chain ID for all the events.
- events_missing_chain_info = initial_events.difference(seen_events)
- if events_missing_chain_info:
- # This can happen due to e.g. downgrade/upgrade of the server. We
- # raise an exception and fall back to the previous algorithm.
- logger.info(
- "Unexpectedly found that events don't have chain IDs in room %s: %s",
- room_id,
- events_missing_chain_info,
- )
- raise _NoChainCoverIndex(room_id)
-
- # Now we look up all links for the chains we have, adding chains that
- # are reachable from any event.
- sql = """
- SELECT
- origin_chain_id, origin_sequence_number,
- target_chain_id, target_sequence_number
- FROM event_auth_chain_links
- WHERE %s
- """
-
- # A map from chain ID to max sequence number *reachable* from any event ID.
- chains: Dict[int, int] = {}
-
- # Add all linked chains reachable from initial set of chains.
- for batch2 in batch_iter(event_chains, 1000):
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "origin_chain_id", batch2
- )
- txn.execute(sql % (clause,), args)
-
- for (
- origin_chain_id,
- origin_sequence_number,
- target_chain_id,
- target_sequence_number,
- ) in txn:
- # chains are only reachable if the origin sequence number of
- # the link is less than the max sequence number in the
- # origin chain.
- if origin_sequence_number <= event_chains.get(origin_chain_id, 0):
- chains[target_chain_id] = max(
- target_sequence_number,
- chains.get(target_chain_id, 0),
- )
-
- # Add the initial set of chains, excluding the sequence corresponding to
- # initial event.
- for chain_id, seq_no in event_chains.items():
- chains[chain_id] = max(seq_no - 1, chains.get(chain_id, 0))
-
- # Now for each chain we figure out the maximum sequence number reachable
- # from *any* event ID. Events with a sequence less than that are in the
- # auth chain.
- if include_given:
- results = initial_events
- else:
- results = set()
-
- if isinstance(self.database_engine, PostgresEngine):
- # We can use `execute_values` to efficiently fetch the gaps when
- # using postgres.
- sql = """
- SELECT event_id
- FROM event_auth_chains AS c, (VALUES ?) AS l(chain_id, max_seq)
- WHERE
- c.chain_id = l.chain_id
- AND sequence_number <= max_seq
- """
-
- rows = txn.execute_values(sql, chains.items())
- results.update(r for r, in rows)
- else:
- # For SQLite we just fall back to doing a noddy for loop.
- sql = """
- SELECT event_id FROM event_auth_chains
- WHERE chain_id = ? AND sequence_number <= ?
- """
- for chain_id, max_no in chains.items():
- txn.execute(sql, (chain_id, max_no))
- results.update(r for r, in txn)
-
- return results
-
- def _get_auth_chain_ids_txn(
- self, txn: LoggingTransaction, event_ids: Collection[str], include_given: bool
- ) -> Set[str]:
- """Calculates the auth chain IDs.
-
- This is used when we don't have a cover index for the room.
- """
- if include_given:
- results = set(event_ids)
- else:
- results = set()
-
- # We pull out the depth simply so that we can populate the
- # `_event_auth_cache` cache.
- base_sql = """
- SELECT a.event_id, auth_id, depth
- FROM event_auth AS a
- INNER JOIN events AS e ON (e.event_id = a.auth_id)
- WHERE
- """
-
- front = set(event_ids)
- while front:
- new_front: Set[str] = set()
- for chunk in batch_iter(front, 100):
- # Pull the auth events either from the cache or DB.
- to_fetch: List[str] = [] # Event IDs to fetch from DB
- for event_id in chunk:
- res = self._event_auth_cache.get(event_id)
- if res is None:
- to_fetch.append(event_id)
- else:
- new_front.update(auth_id for auth_id, depth in res)
-
- if to_fetch:
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "a.event_id", to_fetch
- )
- txn.execute(base_sql + clause, args)
-
- # Note we need to batch up the results by event ID before
- # adding to the cache.
- to_cache: Dict[str, List[Tuple[str, int]]] = {}
- for event_id, auth_event_id, auth_event_depth in txn:
- to_cache.setdefault(event_id, []).append(
- (auth_event_id, auth_event_depth)
- )
- new_front.add(auth_event_id)
-
- for event_id, auth_events in to_cache.items():
- self._event_auth_cache.set(event_id, auth_events)
-
- new_front -= results
-
- front = new_front
- results.update(front)
-
- return results
-
- async def get_auth_chain_difference(
- self, room_id: str, state_sets: List[Set[str]]
- ) -> Set[str]:
- """Given sets of state events figure out the auth chain difference (as
- per state res v2 algorithm).
-
- This equivalent to fetching the full auth chain for each set of state
- and returning the events that don't appear in each and every auth
- chain.
-
- Returns:
- The set of the difference in auth chains.
- """
-
- # Check if we have indexed the room so we can use the chain cover
- # algorithm.
- room = await self.get_room(room_id) # type: ignore[attr-defined]
- if room["has_auth_chain_index"]:
- try:
- return await self.db_pool.runInteraction(
- "get_auth_chain_difference_chains",
- self._get_auth_chain_difference_using_cover_index_txn,
- room_id,
- state_sets,
- )
- except _NoChainCoverIndex:
- # For whatever reason we don't actually have a chain cover index
- # for the events in question, so we fall back to the old method.
- pass
-
- return await self.db_pool.runInteraction(
- "get_auth_chain_difference",
- self._get_auth_chain_difference_txn,
- state_sets,
- )
-
- def _get_auth_chain_difference_using_cover_index_txn(
- self, txn: LoggingTransaction, room_id: str, state_sets: List[Set[str]]
- ) -> Set[str]:
- """Calculates the auth chain difference using the chain index.
-
- See docs/auth_chain_difference_algorithm.md for details
- """
-
- # First we look up the chain ID/sequence numbers for all the events, and
- # work out the chain/sequence numbers reachable from each state set.
-
- initial_events = set(state_sets[0]).union(*state_sets[1:])
-
- # Map from event_id -> (chain ID, seq no)
- chain_info: Dict[str, Tuple[int, int]] = {}
-
- # Map from chain ID -> seq no -> event Id
- chain_to_event: Dict[int, Dict[int, str]] = {}
-
- # All the chains that we've found that are reachable from the state
- # sets.
- seen_chains: Set[int] = set()
-
- sql = """
- SELECT event_id, chain_id, sequence_number
- FROM event_auth_chains
- WHERE %s
- """
- for batch in batch_iter(initial_events, 1000):
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "event_id", batch
- )
- txn.execute(sql % (clause,), args)
-
- for event_id, chain_id, sequence_number in txn:
- chain_info[event_id] = (chain_id, sequence_number)
- seen_chains.add(chain_id)
- chain_to_event.setdefault(chain_id, {})[sequence_number] = event_id
-
- # Check that we actually have a chain ID for all the events.
- events_missing_chain_info = initial_events.difference(chain_info)
- if events_missing_chain_info:
- # This can happen due to e.g. downgrade/upgrade of the server. We
- # raise an exception and fall back to the previous algorithm.
- logger.info(
- "Unexpectedly found that events don't have chain IDs in room %s: %s",
- room_id,
- events_missing_chain_info,
- )
- raise _NoChainCoverIndex(room_id)
-
- # Corresponds to `state_sets`, except as a map from chain ID to max
- # sequence number reachable from the state set.
- set_to_chain: List[Dict[int, int]] = []
- for state_set in state_sets:
- chains: Dict[int, int] = {}
- set_to_chain.append(chains)
-
- for event_id in state_set:
- chain_id, seq_no = chain_info[event_id]
-
- chains[chain_id] = max(seq_no, chains.get(chain_id, 0))
-
- # Now we look up all links for the chains we have, adding chains to
- # set_to_chain that are reachable from each set.
- sql = """
- SELECT
- origin_chain_id, origin_sequence_number,
- target_chain_id, target_sequence_number
- FROM event_auth_chain_links
- WHERE %s
- """
-
- # (We need to take a copy of `seen_chains` as we want to mutate it in
- # the loop)
- for batch2 in batch_iter(set(seen_chains), 1000):
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "origin_chain_id", batch2
- )
- txn.execute(sql % (clause,), args)
-
- for (
- origin_chain_id,
- origin_sequence_number,
- target_chain_id,
- target_sequence_number,
- ) in txn:
- for chains in set_to_chain:
- # chains are only reachable if the origin sequence number of
- # the link is less than the max sequence number in the
- # origin chain.
- if origin_sequence_number <= chains.get(origin_chain_id, 0):
- chains[target_chain_id] = max(
- target_sequence_number,
- chains.get(target_chain_id, 0),
- )
-
- seen_chains.add(target_chain_id)
-
- # Now for each chain we figure out the maximum sequence number reachable
- # from *any* state set and the minimum sequence number reachable from
- # *all* state sets. Events in that range are in the auth chain
- # difference.
- result = set()
-
- # Mapping from chain ID to the range of sequence numbers that should be
- # pulled from the database.
- chain_to_gap: Dict[int, Tuple[int, int]] = {}
-
- for chain_id in seen_chains:
- min_seq_no = min(chains.get(chain_id, 0) for chains in set_to_chain)
- max_seq_no = max(chains.get(chain_id, 0) for chains in set_to_chain)
-
- if min_seq_no < max_seq_no:
- # We have a non empty gap, try and fill it from the events that
- # we have, otherwise add them to the list of gaps to pull out
- # from the DB.
- for seq_no in range(min_seq_no + 1, max_seq_no + 1):
- event_id = chain_to_event.get(chain_id, {}).get(seq_no)
- if event_id:
- result.add(event_id)
- else:
- chain_to_gap[chain_id] = (min_seq_no, max_seq_no)
- break
-
- if not chain_to_gap:
- # If there are no gaps to fetch, we're done!
- return result
-
- if isinstance(self.database_engine, PostgresEngine):
- # We can use `execute_values` to efficiently fetch the gaps when
- # using postgres.
- sql = """
- SELECT event_id
- FROM event_auth_chains AS c, (VALUES ?) AS l(chain_id, min_seq, max_seq)
- WHERE
- c.chain_id = l.chain_id
- AND min_seq < sequence_number AND sequence_number <= max_seq
- """
-
- args = [
- (chain_id, min_no, max_no)
- for chain_id, (min_no, max_no) in chain_to_gap.items()
- ]
-
- rows = txn.execute_values(sql, args)
- result.update(r for r, in rows)
- else:
- # For SQLite we just fall back to doing a noddy for loop.
- sql = """
- SELECT event_id FROM event_auth_chains
- WHERE chain_id = ? AND ? < sequence_number AND sequence_number <= ?
- """
- for chain_id, (min_no, max_no) in chain_to_gap.items():
- txn.execute(sql, (chain_id, min_no, max_no))
- result.update(r for r, in txn)
-
- return result
-
- def _get_auth_chain_difference_txn(
- self, txn: LoggingTransaction, state_sets: List[Set[str]]
- ) -> Set[str]:
- """Calculates the auth chain difference using a breadth first search.
-
- This is used when we don't have a cover index for the room.
- """
-
- # Algorithm Description
- # ~~~~~~~~~~~~~~~~~~~~~
- #
- # The idea here is to basically walk the auth graph of each state set in
- # tandem, keeping track of which auth events are reachable by each state
- # set. If we reach an auth event we've already visited (via a different
- # state set) then we mark that auth event and all ancestors as reachable
- # by the state set. This requires that we keep track of the auth chains
- # in memory.
- #
- # Doing it in a such a way means that we can stop early if all auth
- # events we're currently walking are reachable by all state sets.
- #
- # *Note*: We can't stop walking an event's auth chain if it is reachable
- # by all state sets. This is because other auth chains we're walking
- # might be reachable only via the original auth chain. For example,
- # given the following auth chain:
- #
- # A -> C -> D -> E
- # / /
- # B -´---------´
- #
- # and state sets {A} and {B} then walking the auth chains of A and B
- # would immediately show that C is reachable by both. However, if we
- # stopped at C then we'd only reach E via the auth chain of B and so E
- # would erroneously get included in the returned difference.
- #
- # The other thing that we do is limit the number of auth chains we walk
- # at once, due to practical limits (i.e. we can only query the database
- # with a limited set of parameters). We pick the auth chains we walk
- # each iteration based on their depth, in the hope that events with a
- # lower depth are likely reachable by those with higher depths.
- #
- # We could use any ordering that we believe would give a rough
- # topological ordering, e.g. origin server timestamp. If the ordering
- # chosen is not topological then the algorithm still produces the right
- # result, but perhaps a bit more inefficiently. This is why it is safe
- # to use "depth" here.
-
- initial_events = set(state_sets[0]).union(*state_sets[1:])
-
- # Dict from events in auth chains to which sets *cannot* reach them.
- # I.e. if the set is empty then all sets can reach the event.
- event_to_missing_sets = {
- event_id: {i for i, a in enumerate(state_sets) if event_id not in a}
- for event_id in initial_events
- }
-
- # The sorted list of events whose auth chains we should walk.
- search: List[Tuple[int, str]] = []
-
- # We need to get the depth of the initial events for sorting purposes.
- sql = """
- SELECT depth, event_id FROM events
- WHERE %s
- """
- # the list can be huge, so let's avoid looking them all up in one massive
- # query.
- for batch in batch_iter(initial_events, 1000):
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "event_id", batch
- )
- txn.execute(sql % (clause,), args)
-
- # I think building a temporary list with fetchall is more efficient than
- # just `search.extend(txn)`, but this is unconfirmed
- search.extend(cast(List[Tuple[int, str]], txn.fetchall()))
-
- # sort by depth
- search.sort()
-
- # Map from event to its auth events
- event_to_auth_events: Dict[str, Set[str]] = {}
-
- base_sql = """
- SELECT a.event_id, auth_id, depth
- FROM event_auth AS a
- INNER JOIN events AS e ON (e.event_id = a.auth_id)
- WHERE
- """
-
- while search:
- # Check whether all our current walks are reachable by all state
- # sets. If so we can bail.
- if all(not event_to_missing_sets[eid] for _, eid in search):
- break
-
- # Fetch the auth events and their depths of the N last events we're
- # currently walking, either from cache or DB.
- search, chunk = search[:-100], search[-100:]
-
- found: List[Tuple[str, str, int]] = [] # Results found
- to_fetch: List[str] = [] # Event IDs to fetch from DB
- for _, event_id in chunk:
- res = self._event_auth_cache.get(event_id)
- if res is None:
- to_fetch.append(event_id)
- else:
- found.extend((event_id, auth_id, depth) for auth_id, depth in res)
-
- if to_fetch:
- clause, args = make_in_list_sql_clause(
- txn.database_engine, "a.event_id", to_fetch
- )
- txn.execute(base_sql + clause, args)
-
- # We parse the results and add the to the `found` set and the
- # cache (note we need to batch up the results by event ID before
- # adding to the cache).
- to_cache: Dict[str, List[Tuple[str, int]]] = {}
- for event_id, auth_event_id, auth_event_depth in txn:
- to_cache.setdefault(event_id, []).append(
- (auth_event_id, auth_event_depth)
- )
- found.append((event_id, auth_event_id, auth_event_depth))
-
- for event_id, auth_events in to_cache.items():
- self._event_auth_cache.set(event_id, auth_events)
-
- for event_id, auth_event_id, auth_event_depth in found:
- event_to_auth_events.setdefault(event_id, set()).add(auth_event_id)
-
- sets = event_to_missing_sets.get(auth_event_id)
- if sets is None:
- # First time we're seeing this event, so we add it to the
- # queue of things to fetch.
- search.append((auth_event_depth, auth_event_id))
-
- # Assume that this event is unreachable from any of the
- # state sets until proven otherwise
- sets = event_to_missing_sets[auth_event_id] = set(
- range(len(state_sets))
- )
- else:
- # We've previously seen this event, so look up its auth
- # events and recursively mark all ancestors as reachable
- # by the current event's state set.
- a_ids = event_to_auth_events.get(auth_event_id)
- while a_ids:
- new_aids = set()
- for a_id in a_ids:
- event_to_missing_sets[a_id].intersection_update(
- event_to_missing_sets[event_id]
- )
-
- b = event_to_auth_events.get(a_id)
- if b:
- new_aids.update(b)
-
- a_ids = new_aids
-
- # Mark that the auth event is reachable by the appropriate sets.
- sets.intersection_update(event_to_missing_sets[event_id])
-
- search.sort()
-
- # Return all events where not all sets can reach them.
- return {eid for eid, n in event_to_missing_sets.items() if n}
-
- @trace
- @tag_args
- async def get_backfill_points_in_room(
- self,
- room_id: str,
- current_depth: int,
- limit: int,
- ) -> List[Tuple[str, int]]:
- """
- Get the backward extremities to backfill from in the room along with the
- approximate depth.
-
- Only returns events that are at a depth lower than or
- equal to the `current_depth`. Sorted by depth, highest to lowest (descending)
- so the closest events to the `current_depth` are first in the list.
-
- We ignore extremities that are newer than the user's current scroll position
- (ie, those with depth greater than `current_depth`) as:
- 1. we don't really care about getting events that have happened
- after our current position; and
- 2. by the nature of paginating and scrolling back, we have likely
- previously tried and failed to backfill from that extremity, so
- to avoid getting "stuck" requesting the same backfill repeatedly
- we drop those extremities.
-
- Args:
- room_id: Room where we want to find the oldest events
- current_depth: The depth at the user's current scrollback position
- limit: The max number of backfill points to return
-
- Returns:
- List of (event_id, depth) tuples. Sorted by depth, highest to lowest
- (descending) so the closest events to the `current_depth` are first
- in the list.
- """
-
- def get_backfill_points_in_room_txn(
- txn: LoggingTransaction, room_id: str
- ) -> List[Tuple[str, int]]:
- # Assemble a tuple lookup of event_id -> depth for the oldest events
- # we know of in the room. Backwards extremeties are the oldest
- # events we know of in the room but we only know of them because
- # some other event referenced them by prev_event and aren't
- # persisted in our database yet (meaning we don't know their depth
- # specifically). So we need to look for the approximate depth from
- # the events connected to the current backwards extremeties.
-
- if isinstance(self.database_engine, PostgresEngine):
- least_function = "LEAST"
- elif isinstance(self.database_engine, Sqlite3Engine):
- least_function = "MIN"
- else:
- raise RuntimeError("Unknown database engine")
-
- sql = f"""
- SELECT backward_extrem.event_id, event.depth FROM events AS event
- /**
- * Get the edge connections from the event_edges table
- * so we can see whether this event's prev_events points
- * to a backward extremity in the next join.
- */
- INNER JOIN event_edges AS edge
- ON edge.event_id = event.event_id
- /**
- * We find the "oldest" events in the room by looking for
- * events connected to backwards extremeties (oldest events
- * in the room that we know of so far).
- */
- INNER JOIN event_backward_extremities AS backward_extrem
- ON edge.prev_event_id = backward_extrem.event_id
- /**
- * We use this info to make sure we don't retry to use a backfill point
- * if we've already attempted to backfill from it recently.
- */
- LEFT JOIN event_failed_pull_attempts AS failed_backfill_attempt_info
- ON
- failed_backfill_attempt_info.room_id = backward_extrem.room_id
- AND failed_backfill_attempt_info.event_id = backward_extrem.event_id
- WHERE
- backward_extrem.room_id = ?
- /* We only care about non-state edges because we used to use
- * `event_edges` for two different sorts of "edges" (the current
- * event DAG, but also a link to the previous state, for state
- * events). These legacy state event edges can be distinguished by
- * `is_state` and are removed from the codebase and schema but
- * because the schema change is in a background update, it's not
- * necessarily safe to assume that it will have been completed.
- */
- AND edge.is_state is ? /* False */
- /**
- * We only want backwards extremities that are older than or at
- * the same position of the given `current_depth` (where older
- * means less than the given depth) because we're looking backwards
- * from the `current_depth` when backfilling.
- *
- * current_depth (ignore events that come after this, ignore 2-4)
- * |
- * ▼
- * <oldest-in-time> [0]<--[1]<--[2]<--[3]<--[4] <newest-in-time>
- */
- AND event.depth <= ? /* current_depth */
- /**
- * Exponential back-off (up to the upper bound) so we don't retry the
- * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc.
- *
- * We use `1 << n` as a power of 2 equivalent for compatibility
- * with older SQLites. The left shift equivalent only works with
- * powers of 2 because left shift is a binary operation (base-2).
- * Otherwise, we would use `power(2, n)` or the power operator, `2^n`.
- */
- AND (
- failed_backfill_attempt_info.event_id IS NULL
- OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + (
- (1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */))
- * ? /* step */
- )
- )
- /**
- * Sort from highest (closest to the `current_depth`) to the lowest depth
- * because the closest are most relevant to backfill from first.
- * Then tie-break on alphabetical order of the event_ids so we get a
- * consistent ordering which is nice when asserting things in tests.
- */
- ORDER BY event.depth DESC, backward_extrem.event_id DESC
- LIMIT ?
- """
-
- txn.execute(
- sql,
- (
- room_id,
- False,
- current_depth,
- self._clock.time_msec(),
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS,
- limit,
- ),
- )
-
- return cast(List[Tuple[str, int]], txn.fetchall())
-
- return await self.db_pool.runInteraction(
- "get_backfill_points_in_room",
- get_backfill_points_in_room_txn,
- room_id,
- )
-
- @trace
- async def get_insertion_event_backward_extremities_in_room(
- self,
- room_id: str,
- current_depth: int,
- limit: int,
- ) -> List[Tuple[str, int]]:
- """
- Get the insertion events we know about that we haven't backfilled yet
- along with the approximate depth. Only returns insertion events that are
- at a depth lower than or equal to the `current_depth`. Sorted by depth,
- highest to lowest (descending) so the closest events to the
- `current_depth` are first in the list.
-
- We ignore insertion events that are newer than the user's current scroll
- position (ie, those with depth greater than `current_depth`) as:
- 1. we don't really care about getting events that have happened
- after our current position; and
- 2. by the nature of paginating and scrolling back, we have likely
- previously tried and failed to backfill from that insertion event, so
- to avoid getting "stuck" requesting the same backfill repeatedly
- we drop those insertion event.
-
- Args:
- room_id: Room where we want to find the oldest events
- current_depth: The depth at the user's current scrollback position
- limit: The max number of insertion event extremities to return
-
- Returns:
- List of (event_id, depth) tuples. Sorted by depth, highest to lowest
- (descending) so the closest events to the `current_depth` are first
- in the list.
- """
-
- def get_insertion_event_backward_extremities_in_room_txn(
- txn: LoggingTransaction, room_id: str
- ) -> List[Tuple[str, int]]:
- if isinstance(self.database_engine, PostgresEngine):
- least_function = "LEAST"
- elif isinstance(self.database_engine, Sqlite3Engine):
- least_function = "MIN"
- else:
- raise RuntimeError("Unknown database engine")
-
- sql = f"""
- SELECT
- insertion_event_extremity.event_id, event.depth
- /* We only want insertion events that are also marked as backwards extremities */
- FROM insertion_event_extremities AS insertion_event_extremity
- /* Get the depth of the insertion event from the events table */
- INNER JOIN events AS event USING (event_id)
- /**
- * We use this info to make sure we don't retry to use a backfill point
- * if we've already attempted to backfill from it recently.
- */
- LEFT JOIN event_failed_pull_attempts AS failed_backfill_attempt_info
- ON
- failed_backfill_attempt_info.room_id = insertion_event_extremity.room_id
- AND failed_backfill_attempt_info.event_id = insertion_event_extremity.event_id
- WHERE
- insertion_event_extremity.room_id = ?
- /**
- * We only want extremities that are older than or at
- * the same position of the given `current_depth` (where older
- * means less than the given depth) because we're looking backwards
- * from the `current_depth` when backfilling.
- *
- * current_depth (ignore events that come after this, ignore 2-4)
- * |
- * ▼
- * <oldest-in-time> [0]<--[1]<--[2]<--[3]<--[4] <newest-in-time>
- */
- AND event.depth <= ? /* current_depth */
- /**
- * Exponential back-off (up to the upper bound) so we don't retry the
- * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc
- *
- * We use `1 << n` as a power of 2 equivalent for compatibility
- * with older SQLites. The left shift equivalent only works with
- * powers of 2 because left shift is a binary operation (base-2).
- * Otherwise, we would use `power(2, n)` or the power operator, `2^n`.
- */
- AND (
- failed_backfill_attempt_info.event_id IS NULL
- OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + (
- (1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */))
- * ? /* step */
- )
- )
- /**
- * Sort from highest (closest to the `current_depth`) to the lowest depth
- * because the closest are most relevant to backfill from first.
- * Then tie-break on alphabetical order of the event_ids so we get a
- * consistent ordering which is nice when asserting things in tests.
- */
- ORDER BY event.depth DESC, insertion_event_extremity.event_id DESC
- LIMIT ?
- """
-
- txn.execute(
- sql,
- (
- room_id,
- current_depth,
- self._clock.time_msec(),
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS,
- limit,
- ),
- )
- return cast(List[Tuple[str, int]], txn.fetchall())
-
- return await self.db_pool.runInteraction(
- "get_insertion_event_backward_extremities_in_room",
- get_insertion_event_backward_extremities_in_room_txn,
- room_id,
- )
-
- async def get_max_depth_of(
- self, event_ids: Collection[str]
- ) -> Tuple[Optional[str], int]:
- """Returns the event ID and depth for the event that has the max depth from a set of event IDs
-
- Args:
- event_ids: The event IDs to calculate the max depth of.
- """
- rows = await self.db_pool.simple_select_many_batch(
- table="events",
- column="event_id",
- iterable=event_ids,
- retcols=(
- "event_id",
- "depth",
- ),
- desc="get_max_depth_of",
- )
-
- if not rows:
- return None, 0
- else:
- max_depth_event_id = ""
- current_max_depth = 0
- for row in rows:
- if row["depth"] > current_max_depth:
- max_depth_event_id = row["event_id"]
- current_max_depth = row["depth"]
-
- return max_depth_event_id, current_max_depth
-
- async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[Optional[str], int]:
- """Returns the event ID and depth for the event that has the min depth from a set of event IDs
-
- Args:
- event_ids: The event IDs to calculate the max depth of.
- """
- rows = await self.db_pool.simple_select_many_batch(
- table="events",
- column="event_id",
- iterable=event_ids,
- retcols=(
- "event_id",
- "depth",
- ),
- desc="get_min_depth_of",
- )
-
- if not rows:
- return None, 0
- else:
- min_depth_event_id = ""
- current_min_depth = MAX_DEPTH
- for row in rows:
- if row["depth"] < current_min_depth:
- min_depth_event_id = row["event_id"]
- current_min_depth = row["depth"]
-
- return min_depth_event_id, current_min_depth
-
- async def get_prev_events_for_room(self, room_id: str) -> List[str]:
- """
- Gets a subset of the current forward extremities in the given room.
-
- Limits the result to 10 extremities, so that we can avoid creating
- events which refer to hundreds of prev_events.
-
- Args:
- room_id: room_id
-
- Returns:
- The event ids of the forward extremities.
-
- """
-
- return await self.db_pool.runInteraction(
- "get_prev_events_for_room", self._get_prev_events_for_room_txn, room_id
- )
-
- def _get_prev_events_for_room_txn(
- self, txn: LoggingTransaction, room_id: str
- ) -> List[str]:
- # we just use the 10 newest events. Older events will become
- # prev_events of future events.
-
- sql = """
- SELECT e.event_id FROM event_forward_extremities AS f
- INNER JOIN events AS e USING (event_id)
- WHERE f.room_id = ?
- ORDER BY e.depth DESC
- LIMIT 10
- """
-
- txn.execute(sql, (room_id,))
-
- return [row[0] for row in txn]
-
- async def get_rooms_with_many_extremities(
- self, min_count: int, limit: int, room_id_filter: Iterable[str]
- ) -> List[str]:
- """Get the top rooms with at least N extremities.
-
- Args:
- min_count: The minimum number of extremities
- limit: The maximum number of rooms to return.
- room_id_filter: room_ids to exclude from the results
-
- Returns:
- At most `limit` room IDs that have at least `min_count` extremities,
- sorted by extremity count.
- """
-
- def _get_rooms_with_many_extremities_txn(txn: LoggingTransaction) -> List[str]:
- where_clause = "1=1"
- if room_id_filter:
- where_clause = "room_id NOT IN (%s)" % (
- ",".join("?" for _ in room_id_filter),
- )
-
- sql = """
- SELECT room_id FROM event_forward_extremities
- WHERE %s
- GROUP BY room_id
- HAVING count(*) > ?
- ORDER BY count(*) DESC
- LIMIT ?
- """ % (
- where_clause,
- )
-
- query_args = list(itertools.chain(room_id_filter, [min_count, limit]))
- txn.execute(sql, query_args)
- return [room_id for room_id, in txn]
-
- return await self.db_pool.runInteraction(
- "get_rooms_with_many_extremities", _get_rooms_with_many_extremities_txn
- )
-
- @cached(max_entries=5000, iterable=True)
- async def get_latest_event_ids_in_room(self, room_id: str) -> Sequence[str]:
- return await self.db_pool.simple_select_onecol(
- table="event_forward_extremities",
- keyvalues={"room_id": room_id},
- retcol="event_id",
- desc="get_latest_event_ids_in_room",
- )
-
- async def get_min_depth(self, room_id: str) -> Optional[int]:
- """For the given room, get the minimum depth we have seen for it."""
- return await self.db_pool.runInteraction(
- "get_min_depth", self._get_min_depth_interaction, room_id
- )
-
- def _get_min_depth_interaction(
- self, txn: LoggingTransaction, room_id: str
- ) -> Optional[int]:
- min_depth = self.db_pool.simple_select_one_onecol_txn(
- txn,
- table="room_depth",
- keyvalues={"room_id": room_id},
- retcol="min_depth",
- allow_none=True,
- )
-
- return int(min_depth) if min_depth is not None else None
-
- @cancellable
- async def get_forward_extremities_for_room_at_stream_ordering(
- self, room_id: str, stream_ordering: int
- ) -> Sequence[str]:
- """For a given room_id and stream_ordering, return the forward
- extremeties of the room at that point in "time".
-
- Throws a StoreError if we have since purged the index for
- stream_orderings from that point.
-
- Args:
- room_id:
- stream_ordering:
-
- Returns:
- A list of event_ids
- """
- # We want to make the cache more effective, so we clamp to the last
- # change before the given ordering.
- last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id) # type: ignore[attr-defined]
-
- # We don't always have a full stream_to_exterm_id table, e.g. after
- # the upgrade that introduced it, so we make sure we never ask for a
- # stream_ordering from before a restart
- last_change = max(self._stream_order_on_start, last_change) # type: ignore[attr-defined]
-
- # provided the last_change is recent enough, we now clamp the requested
- # stream_ordering to it.
- if last_change > self.stream_ordering_month_ago: # type: ignore[attr-defined]
- stream_ordering = min(last_change, stream_ordering)
-
- return await self._get_forward_extremeties_for_room(room_id, stream_ordering)
-
- @cached(max_entries=5000, num_args=2)
- async def _get_forward_extremeties_for_room(
- self, room_id: str, stream_ordering: int
- ) -> Sequence[str]:
- """For a given room_id and stream_ordering, return the forward
- extremeties of the room at that point in "time".
-
- Throws a StoreError if we have since purged the index for
- stream_orderings from that point.
- """
-
- if stream_ordering <= self.stream_ordering_month_ago: # type: ignore[attr-defined]
- raise StoreError(400, "stream_ordering too old %s" % (stream_ordering,))
-
- sql = """
- SELECT event_id FROM stream_ordering_to_exterm
- INNER JOIN (
- SELECT room_id, MAX(stream_ordering) AS stream_ordering
- FROM stream_ordering_to_exterm
- WHERE stream_ordering <= ? GROUP BY room_id
- ) AS rms USING (room_id, stream_ordering)
- WHERE room_id = ?
- """
-
- def get_forward_extremeties_for_room_txn(txn: LoggingTransaction) -> List[str]:
- txn.execute(sql, (stream_ordering, room_id))
- return [event_id for event_id, in txn]
-
- return await self.db_pool.runInteraction(
- "get_forward_extremeties_for_room", get_forward_extremeties_for_room_txn
- )
-
- def _get_connected_batch_event_backfill_results_txn(
- self, txn: LoggingTransaction, insertion_event_id: str, limit: int
- ) -> List[BackfillQueueNavigationItem]:
- """
- Find any batch connections of a given insertion event.
- A batch event points at a insertion event via:
- batch_event.content[MSC2716_BATCH_ID] -> insertion_event.content[MSC2716_NEXT_BATCH_ID]
-
- Args:
- txn: The database transaction to use
- insertion_event_id: The event ID to navigate from. We will find
- batch events that point back at this insertion event.
- limit: Max number of event ID's to query for and return
-
- Returns:
- List of batch events that the backfill queue can process
- """
- batch_connection_query = """
- SELECT e.depth, e.stream_ordering, c.event_id, e.type FROM insertion_events AS i
- /* Find the batch that connects to the given insertion event */
- INNER JOIN batch_events AS c
- ON i.next_batch_id = c.batch_id
- /* Get the depth of the batch start event from the events table */
- INNER JOIN events AS e ON c.event_id = e.event_id
- /* Find an insertion event which matches the given event_id */
- WHERE i.event_id = ?
- LIMIT ?
- """
-
- # Find any batch connections for the given insertion event
- txn.execute(
- batch_connection_query,
- (insertion_event_id, limit),
- )
- return [
- BackfillQueueNavigationItem(
- depth=row[0],
- stream_ordering=row[1],
- event_id=row[2],
- type=row[3],
- )
- for row in txn
- ]
-
- def _get_connected_prev_event_backfill_results_txn(
- self, txn: LoggingTransaction, event_id: str, limit: int
- ) -> List[BackfillQueueNavigationItem]:
- """
- Find any events connected by prev_event the specified event_id.
-
- Args:
- txn: The database transaction to use
- event_id: The event ID to navigate from
- limit: Max number of event ID's to query for and return
-
- Returns:
- List of prev events that the backfill queue can process
- """
- # Look for the prev_event_id connected to the given event_id
- connected_prev_event_query = """
- SELECT depth, stream_ordering, prev_event_id, events.type FROM event_edges
- /* Get the depth and stream_ordering of the prev_event_id from the events table */
- INNER JOIN events
- ON prev_event_id = events.event_id
-
- /* exclude outliers from the results (we don't have the state, so cannot
- * verify if the requesting server can see them).
- */
- WHERE NOT events.outlier
-
- /* Look for an edge which matches the given event_id */
- AND event_edges.event_id = ? AND NOT event_edges.is_state
-
- /* Because we can have many events at the same depth,
- * we want to also tie-break and sort on stream_ordering */
- ORDER BY depth DESC, stream_ordering DESC
- LIMIT ?
- """
-
- txn.execute(
- connected_prev_event_query,
- (event_id, limit),
- )
- return [
- BackfillQueueNavigationItem(
- depth=row[0],
- stream_ordering=row[1],
- event_id=row[2],
- type=row[3],
- )
- for row in txn
- ]
-
- async def get_backfill_events(
- self, room_id: str, seed_event_id_list: List[str], limit: int
- ) -> List[EventBase]:
- """Get a list of Events for a given topic that occurred before (and
- including) the events in seed_event_id_list. Return a list of max size `limit`
-
- Args:
- room_id
- seed_event_id_list
- limit
- """
- event_ids = await self.db_pool.runInteraction(
- "get_backfill_events",
- self._get_backfill_events,
- room_id,
- seed_event_id_list,
- limit,
- )
- events = await self.get_events_as_list(event_ids)
- return sorted(
- # type-ignore: mypy doesn't like negating the Optional[int] stream_ordering.
- # But it's never None, because these events were previously persisted to the DB.
- events,
- key=lambda e: (-e.depth, -e.internal_metadata.stream_ordering), # type: ignore[operator]
- )
-
- def _get_backfill_events(
- self,
- txn: LoggingTransaction,
- room_id: str,
- seed_event_id_list: List[str],
- limit: int,
- ) -> Set[str]:
- """
- We want to make sure that we do a breadth-first, "depth" ordered search.
- We also handle navigating historical branches of history connected by
- insertion and batch events.
- """
- logger.debug(
- "_get_backfill_events(room_id=%s): seeding backfill with seed_event_id_list=%s limit=%s",
- room_id,
- seed_event_id_list,
- limit,
- )
-
- event_id_results: Set[str] = set()
-
- # In a PriorityQueue, the lowest valued entries are retrieved first.
- # We're using depth as the priority in the queue and tie-break based on
- # stream_ordering. Depth is lowest at the oldest-in-time message and
- # highest and newest-in-time message. We add events to the queue with a
- # negative depth so that we process the newest-in-time messages first
- # going backwards in time. stream_ordering follows the same pattern.
- queue: "PriorityQueue[Tuple[int, int, str, str]]" = PriorityQueue()
-
- for seed_event_id in seed_event_id_list:
- event_lookup_result = self.db_pool.simple_select_one_txn(
- txn,
- table="events",
- keyvalues={"event_id": seed_event_id, "room_id": room_id},
- retcols=(
- "type",
- "depth",
- "stream_ordering",
- ),
- allow_none=True,
- )
-
- if event_lookup_result is not None:
- logger.debug(
- "_get_backfill_events(room_id=%s): seed_event_id=%s depth=%s stream_ordering=%s type=%s",
- room_id,
- seed_event_id,
- event_lookup_result["depth"],
- event_lookup_result["stream_ordering"],
- event_lookup_result["type"],
- )
-
- if event_lookup_result["depth"]:
- queue.put(
- (
- -event_lookup_result["depth"],
- -event_lookup_result["stream_ordering"],
- seed_event_id,
- event_lookup_result["type"],
- )
- )
-
- while not queue.empty() and len(event_id_results) < limit:
- try:
- _, _, event_id, event_type = queue.get_nowait()
- except Empty:
- break
-
- if event_id in event_id_results:
- continue
-
- event_id_results.add(event_id)
-
- # Try and find any potential historical batches of message history.
- if self.hs.config.experimental.msc2716_enabled:
- # We need to go and try to find any batch events connected
- # to a given insertion event (by batch_id). If we find any, we'll
- # add them to the queue and navigate up the DAG like normal in the
- # next iteration of the loop.
- if event_type == EventTypes.MSC2716_INSERTION:
- # Find any batch connections for the given insertion event
- connected_batch_event_backfill_results = (
- self._get_connected_batch_event_backfill_results_txn(
- txn, event_id, limit - len(event_id_results)
- )
- )
- logger.debug(
- "_get_backfill_events(room_id=%s): connected_batch_event_backfill_results=%s",
- room_id,
- connected_batch_event_backfill_results,
- )
- for (
- connected_batch_event_backfill_item
- ) in connected_batch_event_backfill_results:
- if (
- connected_batch_event_backfill_item.event_id
- not in event_id_results
- ):
- queue.put(
- (
- -connected_batch_event_backfill_item.depth,
- -connected_batch_event_backfill_item.stream_ordering,
- connected_batch_event_backfill_item.event_id,
- connected_batch_event_backfill_item.type,
- )
- )
-
- # Now we just look up the DAG by prev_events as normal
- connected_prev_event_backfill_results = (
- self._get_connected_prev_event_backfill_results_txn(
- txn, event_id, limit - len(event_id_results)
- )
- )
- logger.debug(
- "_get_backfill_events(room_id=%s): connected_prev_event_backfill_results=%s",
- room_id,
- connected_prev_event_backfill_results,
- )
- for (
- connected_prev_event_backfill_item
- ) in connected_prev_event_backfill_results:
- if connected_prev_event_backfill_item.event_id not in event_id_results:
- queue.put(
- (
- -connected_prev_event_backfill_item.depth,
- -connected_prev_event_backfill_item.stream_ordering,
- connected_prev_event_backfill_item.event_id,
- connected_prev_event_backfill_item.type,
- )
- )
-
- return event_id_results
-
- @trace
- async def record_event_failed_pull_attempt(
- self, room_id: str, event_id: str, cause: str
- ) -> None:
- """
- Record when we fail to pull an event over federation.
-
- This information allows us to be more intelligent when we decide to
- retry (we don't need to fail over and over) and we can process that
- event in the background so we don't block on it each time.
-
- Args:
- room_id: The room where the event failed to pull from
- event_id: The event that failed to be fetched or processed
- cause: The error message or reason that we failed to pull the event
- """
- logger.debug(
- "record_event_failed_pull_attempt room_id=%s, event_id=%s, cause=%s",
- room_id,
- event_id,
- cause,
- )
- await self.db_pool.runInteraction(
- "record_event_failed_pull_attempt",
- self._record_event_failed_pull_attempt_upsert_txn,
- room_id,
- event_id,
- cause,
- db_autocommit=True, # Safe as it's a single upsert
- )
-
- def _record_event_failed_pull_attempt_upsert_txn(
- self,
- txn: LoggingTransaction,
- room_id: str,
- event_id: str,
- cause: str,
- ) -> None:
- sql = """
- INSERT INTO event_failed_pull_attempts (
- room_id, event_id, num_attempts, last_attempt_ts, last_cause
- )
- VALUES (?, ?, ?, ?, ?)
- ON CONFLICT (room_id, event_id) DO UPDATE SET
- num_attempts=event_failed_pull_attempts.num_attempts + 1,
- last_attempt_ts=EXCLUDED.last_attempt_ts,
- last_cause=EXCLUDED.last_cause;
- """
-
- txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec(), cause))
-
- @trace
- async def get_event_ids_to_not_pull_from_backoff(
- self,
- room_id: str,
- event_ids: Collection[str],
- ) -> List[str]:
- """
- Filter down the events to ones that we've failed to pull before recently. Uses
- exponential backoff.
-
- Args:
- room_id: The room that the events belong to
- event_ids: A list of events to filter down
-
- Returns:
- List of event_ids that should not be attempted to be pulled
- """
- event_failed_pull_attempts = await self.db_pool.simple_select_many_batch(
- table="event_failed_pull_attempts",
- column="event_id",
- iterable=event_ids,
- keyvalues={},
- retcols=(
- "event_id",
- "last_attempt_ts",
- "num_attempts",
- ),
- desc="get_event_ids_to_not_pull_from_backoff",
- )
-
- current_time = self._clock.time_msec()
- return [
- event_failed_pull_attempt["event_id"]
- for event_failed_pull_attempt in event_failed_pull_attempts
- # Exponential back-off (up to the upper bound) so we don't try to
- # pull the same event over and over. ex. 2hr, 4hr, 8hr, 16hr, etc.
- if current_time
- < event_failed_pull_attempt["last_attempt_ts"]
- + (
- 2
- ** min(
- event_failed_pull_attempt["num_attempts"],
- BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
- )
- )
- * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS
- ]
-
- async def get_missing_events(
- self,
- room_id: str,
- earliest_events: List[str],
- latest_events: List[str],
- limit: int,
- ) -> List[EventBase]:
- ids = await self.db_pool.runInteraction(
- "get_missing_events",
- self._get_missing_events,
- room_id,
- earliest_events,
- latest_events,
- limit,
- )
- return await self.get_events_as_list(ids)
-
- def _get_missing_events(
- self,
- txn: LoggingTransaction,
- room_id: str,
- earliest_events: List[str],
- latest_events: List[str],
- limit: int,
- ) -> List[str]:
- seen_events = set(earliest_events)
- front = set(latest_events) - seen_events
- event_results: List[str] = []
-
- query = (
- "SELECT prev_event_id FROM event_edges "
- "WHERE event_id = ? AND NOT is_state "
- "LIMIT ?"
- )
-
- while front and len(event_results) < limit:
- new_front = set()
- for event_id in front:
- txn.execute(query, (event_id, limit - len(event_results)))
- new_results = {t[0] for t in txn} - seen_events
-
- new_front |= new_results
- seen_events |= new_results
- event_results.extend(new_results)
-
- front = new_front
-
- # we built the list working backwards from latest_events; we now need to
- # reverse it so that the events are approximately chronological.
- event_results.reverse()
- return event_results
-
- @trace
- @tag_args
- async def get_successor_events(self, event_id: str) -> List[str]:
- """Fetch all events that have the given event as a prev event
-
- Args:
- event_id: The event to search for as a prev_event.
- """
- return await self.db_pool.simple_select_onecol(
- table="event_edges",
- keyvalues={"prev_event_id": event_id},
- retcol="event_id",
- desc="get_successor_events",
- )
-
- @wrap_as_background_process("delete_old_forward_extrem_cache")
- async def _delete_old_forward_extrem_cache(self) -> None:
- def _delete_old_forward_extrem_cache_txn(txn: LoggingTransaction) -> None:
- # Delete entries older than a month, while making sure we don't delete
- # the only entries for a room.
- sql = """
- DELETE FROM stream_ordering_to_exterm
- WHERE
- room_id IN (
- SELECT room_id
- FROM stream_ordering_to_exterm
- WHERE stream_ordering > ?
- ) AND stream_ordering < ?
- """
- txn.execute(
- sql, (self.stream_ordering_month_ago, self.stream_ordering_month_ago) # type: ignore[attr-defined]
- )
-
- await self.db_pool.runInteraction(
- "_delete_old_forward_extrem_cache",
- _delete_old_forward_extrem_cache_txn,
- )
-
- @trace
- async def insert_insertion_extremity(self, event_id: str, room_id: str) -> None:
- await self.db_pool.simple_upsert(
- table="insertion_event_extremities",
- keyvalues={"event_id": event_id},
- values={
- "event_id": event_id,
- "room_id": room_id,
- },
- insertion_values={},
- desc="insert_insertion_extremity",
- )
-
- async def insert_received_event_to_staging(
- self, origin: str, event: EventBase
- ) -> None:
- """Insert a newly received event from federation into the staging area."""
-
- # We use an upsert here to handle the case where we see the same event
- # from the same server multiple times.
- await self.db_pool.simple_upsert(
- table="federation_inbound_events_staging",
- keyvalues={
- "origin": origin,
- "event_id": event.event_id,
- },
- values={},
- insertion_values={
- "room_id": event.room_id,
- "received_ts": self._clock.time_msec(),
- "event_json": json_encoder.encode(event.get_dict()),
- "internal_metadata": json_encoder.encode(
- event.internal_metadata.get_dict()
- ),
- },
- desc="insert_received_event_to_staging",
- )
-
- async def remove_received_event_from_staging(
- self,
- origin: str,
- event_id: str,
- ) -> Optional[int]:
- """Remove the given event from the staging area.
-
- Returns:
- The received_ts of the row that was deleted, if any.
- """
- if self.db_pool.engine.supports_returning:
-
- def _remove_received_event_from_staging_txn(
- txn: LoggingTransaction,
- ) -> Optional[int]:
- sql = """
- DELETE FROM federation_inbound_events_staging
- WHERE origin = ? AND event_id = ?
- RETURNING received_ts
- """
-
- txn.execute(sql, (origin, event_id))
- row = cast(Optional[Tuple[int]], txn.fetchone())
-
- if row is None:
- return None
-
- return row[0]
-
- return await self.db_pool.runInteraction(
- "remove_received_event_from_staging",
- _remove_received_event_from_staging_txn,
- db_autocommit=True,
- )
-
- else:
-
- def _remove_received_event_from_staging_txn(
- txn: LoggingTransaction,
- ) -> Optional[int]:
- received_ts = self.db_pool.simple_select_one_onecol_txn(
- txn,
- table="federation_inbound_events_staging",
- keyvalues={
- "origin": origin,
- "event_id": event_id,
- },
- retcol="received_ts",
- allow_none=True,
- )
- self.db_pool.simple_delete_txn(
- txn,
- table="federation_inbound_events_staging",
- keyvalues={
- "origin": origin,
- "event_id": event_id,
- },
- )
-
- return received_ts
-
- return await self.db_pool.runInteraction(
- "remove_received_event_from_staging",
- _remove_received_event_from_staging_txn,
- )
-
- async def get_next_staged_event_id_for_room(
- self,
- room_id: str,
- ) -> Optional[Tuple[str, str]]:
- """
- Get the next event ID in the staging area for the given room.
-
- Returns:
- Tuple of the `origin` and `event_id`
- """
-
- def _get_next_staged_event_id_for_room_txn(
- txn: LoggingTransaction,
- ) -> Optional[Tuple[str, str]]:
- sql = """
- SELECT origin, event_id
- FROM federation_inbound_events_staging
- WHERE room_id = ?
- ORDER BY received_ts ASC
- LIMIT 1
- """
-
- txn.execute(sql, (room_id,))
-
- return cast(Optional[Tuple[str, str]], txn.fetchone())
-
- return await self.db_pool.runInteraction(
- "get_next_staged_event_id_for_room", _get_next_staged_event_id_for_room_txn
- )
-
- async def get_next_staged_event_for_room(
- self,
- room_id: str,
- room_version: RoomVersion,
- ) -> Optional[Tuple[str, EventBase]]:
- """Get the next event in the staging area for the given room."""
-
- def _get_next_staged_event_for_room_txn(
- txn: LoggingTransaction,
- ) -> Optional[Tuple[str, str, str]]:
- sql = """
- SELECT event_json, internal_metadata, origin
- FROM federation_inbound_events_staging
- WHERE room_id = ?
- ORDER BY received_ts ASC
- LIMIT 1
- """
- txn.execute(sql, (room_id,))
-
- return cast(Optional[Tuple[str, str, str]], txn.fetchone())
-
- row = await self.db_pool.runInteraction(
- "get_next_staged_event_for_room", _get_next_staged_event_for_room_txn
- )
-
- if not row:
- return None
-
- event_d = db_to_json(row[0])
- internal_metadata_d = db_to_json(row[1])
- origin = row[2]
-
- event = make_event_from_dict(
- event_dict=event_d,
- room_version=room_version,
- internal_metadata_dict=internal_metadata_d,
- )
-
- return origin, event
-
- async def prune_staged_events_in_room(
- self,
- room_id: str,
- room_version: RoomVersion,
- ) -> bool:
- """Checks if there are lots of staged events for the room, and if so
- prune them down.
-
- Returns:
- Whether any events were pruned
- """
-
- # First check the size of the queue.
- count = await self.db_pool.simple_select_one_onecol(
- table="federation_inbound_events_staging",
- keyvalues={"room_id": room_id},
- retcol="COUNT(*)",
- desc="prune_staged_events_in_room_count",
- )
-
- if count < 100:
- return False
-
- # If the queue is too large, then we want clear the entire queue,
- # keeping only the forward extremities (i.e. the events not referenced
- # by other events in the queue). We do this so that we can always
- # backpaginate in all the events we have dropped.
- rows = await self.db_pool.simple_select_list(
- table="federation_inbound_events_staging",
- keyvalues={"room_id": room_id},
- retcols=("event_id", "event_json"),
- desc="prune_staged_events_in_room_fetch",
- )
-
- # Find the set of events referenced by those in the queue, as well as
- # collecting all the event IDs in the queue.
- referenced_events: Set[str] = set()
- seen_events: Set[str] = set()
- for row in rows:
- event_id = row["event_id"]
- seen_events.add(event_id)
- event_d = db_to_json(row["event_json"])
-
- # We don't bother parsing the dicts into full blown event objects,
- # as that is needlessly expensive.
-
- # We haven't checked that the `prev_events` have the right format
- # yet, so we check as we go.
- prev_events = event_d.get("prev_events", [])
- if not isinstance(prev_events, list):
- logger.info("Invalid prev_events for %s", event_id)
- continue
-
- if room_version.event_format == EventFormatVersions.ROOM_V1_V2:
- for prev_event_tuple in prev_events:
- if (
- not isinstance(prev_event_tuple, list)
- or len(prev_event_tuple) != 2
- ):
- logger.info("Invalid prev_events for %s", event_id)
- break
-
- prev_event_id = prev_event_tuple[0]
- if not isinstance(prev_event_id, str):
- logger.info("Invalid prev_events for %s", event_id)
- break
-
- referenced_events.add(prev_event_id)
- else:
- for prev_event_id in prev_events:
- if not isinstance(prev_event_id, str):
- logger.info("Invalid prev_events for %s", event_id)
- break
-
- referenced_events.add(prev_event_id)
-
- to_delete = referenced_events & seen_events
- if not to_delete:
- return False
-
- pdus_pruned_from_federation_queue.inc(len(to_delete))
- logger.info(
- "Pruning %d events in room %s from federation queue",
- len(to_delete),
- room_id,
- )
-
- await self.db_pool.simple_delete_many(
- table="federation_inbound_events_staging",
- keyvalues={"room_id": room_id},
- iterable=to_delete,
- column="event_id",
- desc="prune_staged_events_in_room_delete",
- )
-
- return True
-
- async def get_all_rooms_with_staged_incoming_events(self) -> List[str]:
- """Get the room IDs of all events currently staged."""
- return await self.db_pool.simple_select_onecol(
- table="federation_inbound_events_staging",
- keyvalues={},
- retcol="DISTINCT room_id",
- desc="get_all_rooms_with_staged_incoming_events",
- )
-
- @wrap_as_background_process("_get_stats_for_federation_staging")
- async def _get_stats_for_federation_staging(self) -> None:
- """Update the prometheus metrics for the inbound federation staging area."""
-
- def _get_stats_for_federation_staging_txn(
- txn: LoggingTransaction,
- ) -> Tuple[int, int]:
- txn.execute("SELECT count(*) FROM federation_inbound_events_staging")
- (count,) = cast(Tuple[int], txn.fetchone())
-
- txn.execute(
- "SELECT min(received_ts) FROM federation_inbound_events_staging"
- )
-
- (received_ts,) = cast(Tuple[Optional[int]], txn.fetchone())
-
- # If there is nothing in the staging area default it to 0.
- age = 0
- if received_ts is not None:
- age = self._clock.time_msec() - received_ts
-
- return count, age
-
- count, age = await self.db_pool.runInteraction(
- "_get_stats_for_federation_staging", _get_stats_for_federation_staging_txn
- )
-
- number_pdus_in_federation_queue.set(count)
- oldest_pdu_in_federation_staging.set(age)
-
-
- class EventFederationStore(EventFederationWorkerStore):
- """Responsible for storing and serving up the various graphs associated
- with an event. Including the main event graph and the auth chains for an
- event.
-
- Also has methods for getting the front (latest) and back (oldest) edges
- of the event graphs. These are used to generate the parents for new events
- and backfilling from another server respectively.
- """
-
- EVENT_AUTH_STATE_ONLY = "event_auth_state_only"
-
- def __init__(
- self,
- database: DatabasePool,
- db_conn: LoggingDatabaseConnection,
- hs: "HomeServer",
- ):
- super().__init__(database, db_conn, hs)
-
- self.db_pool.updates.register_background_update_handler(
- self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth
- )
-
- async def clean_room_for_join(self, room_id: str) -> None:
- await self.db_pool.runInteraction(
- "clean_room_for_join", self._clean_room_for_join_txn, room_id
- )
-
- def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None:
- query = "DELETE FROM event_forward_extremities WHERE room_id = ?"
-
- txn.execute(query, (room_id,))
- txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
-
- async def _background_delete_non_state_event_auth(
- self, progress: JsonDict, batch_size: int
- ) -> int:
- def delete_event_auth(txn: LoggingTransaction) -> bool:
- target_min_stream_id = progress.get("target_min_stream_id_inclusive")
- max_stream_id = progress.get("max_stream_id_exclusive")
-
- if not target_min_stream_id or not max_stream_id:
- txn.execute("SELECT COALESCE(MIN(stream_ordering), 0) FROM events")
- rows = txn.fetchall()
- target_min_stream_id = rows[0][0]
-
- txn.execute("SELECT COALESCE(MAX(stream_ordering), 0) FROM events")
- rows = txn.fetchall()
- max_stream_id = rows[0][0]
-
- min_stream_id = max_stream_id - batch_size
-
- sql = """
- DELETE FROM event_auth
- WHERE event_id IN (
- SELECT event_id FROM events
- LEFT JOIN state_events AS se USING (room_id, event_id)
- WHERE ? <= stream_ordering AND stream_ordering < ?
- AND se.state_key IS null
- )
- """
-
- txn.execute(sql, (min_stream_id, max_stream_id))
-
- new_progress = {
- "target_min_stream_id_inclusive": target_min_stream_id,
- "max_stream_id_exclusive": min_stream_id,
- }
-
- self.db_pool.updates._background_update_progress_txn(
- txn, self.EVENT_AUTH_STATE_ONLY, new_progress
- )
-
- return min_stream_id >= target_min_stream_id
-
- result = await self.db_pool.runInteraction(
- self.EVENT_AUTH_STATE_ONLY, delete_event_auth
- )
-
- if not result:
- await self.db_pool.updates._end_background_update(
- self.EVENT_AUTH_STATE_ONLY
- )
-
- return batch_size
|