You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

1397 lines
48 KiB

  1. #!/usr/bin/env python
  2. # Copyright 2015, 2016 OpenMarket Ltd
  3. # Copyright 2018 New Vector Ltd
  4. # Copyright 2019 The Matrix.org Foundation C.I.C.
  5. #
  6. # Licensed under the Apache License, Version 2.0 (the "License");
  7. # you may not use this file except in compliance with the License.
  8. # You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import argparse
  18. import curses
  19. import logging
  20. import sys
  21. import time
  22. import traceback
  23. from types import TracebackType
  24. from typing import (
  25. Any,
  26. Awaitable,
  27. Callable,
  28. Dict,
  29. Generator,
  30. Iterable,
  31. List,
  32. NoReturn,
  33. Optional,
  34. Set,
  35. Tuple,
  36. Type,
  37. TypeVar,
  38. cast,
  39. )
  40. import yaml
  41. from typing_extensions import TypedDict
  42. from twisted.internet import defer, reactor as reactor_
  43. from synapse.config.database import DatabaseConnectionConfig
  44. from synapse.config.homeserver import HomeServerConfig
  45. from synapse.logging.context import (
  46. LoggingContext,
  47. make_deferred_yieldable,
  48. run_in_background,
  49. )
  50. from synapse.notifier import ReplicationNotifier
  51. from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn
  52. from synapse.storage.databases.main import PushRuleStore
  53. from synapse.storage.databases.main.account_data import AccountDataWorkerStore
  54. from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore
  55. from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
  56. from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
  57. from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
  58. from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
  59. from synapse.storage.databases.main.events_bg_updates import (
  60. EventsBackgroundUpdatesStore,
  61. )
  62. from synapse.storage.databases.main.media_repository import (
  63. MediaRepositoryBackgroundUpdateStore,
  64. )
  65. from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
  66. from synapse.storage.databases.main.pusher import PusherWorkerStore
  67. from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore
  68. from synapse.storage.databases.main.registration import (
  69. RegistrationBackgroundUpdateStore,
  70. find_max_generated_user_id_localpart,
  71. )
  72. from synapse.storage.databases.main.relations import RelationsWorkerStore
  73. from synapse.storage.databases.main.room import RoomBackgroundUpdateStore
  74. from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore
  75. from synapse.storage.databases.main.search import SearchBackgroundUpdateStore
  76. from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore
  77. from synapse.storage.databases.main.stats import StatsStore
  78. from synapse.storage.databases.main.user_directory import (
  79. UserDirectoryBackgroundUpdateStore,
  80. )
  81. from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore
  82. from synapse.storage.engines import create_engine
  83. from synapse.storage.prepare_database import prepare_database
  84. from synapse.types import ISynapseReactor
  85. from synapse.util import SYNAPSE_VERSION, Clock
  86. # Cast safety: Twisted does some naughty magic which replaces the
  87. # twisted.internet.reactor module with a Reactor instance at runtime.
  88. reactor = cast(ISynapseReactor, reactor_)
  89. logger = logging.getLogger("synapse_port_db")
  90. # SQLite doesn't have a dedicated boolean type (it stores True/False as 1/0). This means
  91. # portdb will read sqlite bools as integers, then try to insert them into postgres
  92. # boolean columns---which fails. Lacking some Python-parseable metaschema, we must
  93. # specify which integer columns should be inserted as booleans into postgres.
  94. BOOLEAN_COLUMNS = {
  95. "access_tokens": ["used"],
  96. "account_validity": ["email_sent"],
  97. "device_lists_changes_in_room": ["converted_to_destinations"],
  98. "device_lists_outbound_pokes": ["sent"],
  99. "devices": ["hidden"],
  100. "e2e_fallback_keys_json": ["used"],
  101. "e2e_room_keys": ["is_verified"],
  102. "event_edges": ["is_state"],
  103. "events": ["processed", "outlier", "contains_url"],
  104. "local_media_repository": ["safe_from_quarantine"],
  105. "presence_list": ["accepted"],
  106. "presence_stream": ["currently_active"],
  107. "public_room_list_stream": ["visibility"],
  108. "pushers": ["enabled"],
  109. "redactions": ["have_censored"],
  110. "room_stats_state": ["is_federatable"],
  111. "rooms": ["is_public", "has_auth_chain_index"],
  112. "users": ["shadow_banned", "approved"],
  113. "un_partial_stated_event_stream": ["rejection_status_changed"],
  114. "users_who_share_rooms": ["share_private"],
  115. }
  116. # These tables are never deleted from in normal operation [*], so we can resume porting
  117. # over rows from a previous attempt rather than starting from scratch.
  118. #
  119. # [*]: We do delete from many of these tables when purging a room, and
  120. # presumably when purging old events. So we might e.g.
  121. #
  122. # 1. Run portdb and port half of some table.
  123. # 2. Stop portdb.
  124. # 3. Purge something, deleting some of the rows we've ported over.
  125. # 4. Restart portdb. The rows deleted from sqlite are still present in postgres.
  126. #
  127. # But this isn't the end of the world: we should be able to repeat the purge
  128. # on the postgres DB when porting completes.
  129. APPEND_ONLY_TABLES = [
  130. "cache_invalidation_stream_by_instance",
  131. "event_auth",
  132. "event_edges",
  133. "event_json",
  134. "event_reference_hashes",
  135. "event_search",
  136. "event_to_state_groups",
  137. "events",
  138. "ex_outlier_stream",
  139. "local_media_repository",
  140. "local_media_repository_thumbnails",
  141. "presence_stream",
  142. "public_room_list_stream",
  143. "push_rules_stream",
  144. "received_transactions",
  145. "redactions",
  146. "rejections",
  147. "remote_media_cache",
  148. "remote_media_cache_thumbnails",
  149. "room_memberships",
  150. "room_names",
  151. "rooms",
  152. "sent_transactions",
  153. "state_events",
  154. "state_group_edges",
  155. "state_groups",
  156. "state_groups_state",
  157. "stream_ordering_to_exterm",
  158. "topics",
  159. "transaction_id_to_pdu",
  160. "un_partial_stated_event_stream",
  161. "users",
  162. ]
  163. IGNORED_TABLES = {
  164. # We don't port these tables, as they're a faff and we can regenerate
  165. # them anyway.
  166. "user_directory",
  167. "user_directory_search",
  168. "user_directory_search_content",
  169. "user_directory_search_docsize",
  170. "user_directory_search_segdir",
  171. "user_directory_search_segments",
  172. "user_directory_search_stat",
  173. "user_directory_search_pos",
  174. "users_who_share_private_rooms",
  175. "users_in_public_room",
  176. # UI auth sessions have foreign keys so additional care needs to be taken,
  177. # the sessions are transient anyway, so ignore them.
  178. "ui_auth_sessions",
  179. "ui_auth_sessions_credentials",
  180. "ui_auth_sessions_ips",
  181. }
  182. # Error returned by the run function. Used at the top-level part of the script to
  183. # handle errors and return codes.
  184. end_error: Optional[str] = None
  185. # The exec_info for the error, if any. If error is defined but not exec_info the script
  186. # will show only the error message without the stacktrace, if exec_info is defined but
  187. # not the error then the script will show nothing outside of what's printed in the run
  188. # function. If both are defined, the script will print both the error and the stacktrace.
  189. end_error_exec_info: Optional[
  190. Tuple[Type[BaseException], BaseException, TracebackType]
  191. ] = None
  192. R = TypeVar("R")
  193. class Store(
  194. EventPushActionsStore,
  195. ClientIpBackgroundUpdateStore,
  196. DeviceInboxBackgroundUpdateStore,
  197. DeviceBackgroundUpdateStore,
  198. EventsBackgroundUpdatesStore,
  199. MediaRepositoryBackgroundUpdateStore,
  200. RegistrationBackgroundUpdateStore,
  201. RoomBackgroundUpdateStore,
  202. RoomMemberBackgroundUpdateStore,
  203. SearchBackgroundUpdateStore,
  204. StateBackgroundUpdateStore,
  205. MainStateBackgroundUpdateStore,
  206. UserDirectoryBackgroundUpdateStore,
  207. EndToEndKeyBackgroundStore,
  208. StatsStore,
  209. AccountDataWorkerStore,
  210. PushRuleStore,
  211. PusherWorkerStore,
  212. PresenceBackgroundUpdateStore,
  213. ReceiptsBackgroundUpdateStore,
  214. RelationsWorkerStore,
  215. ):
  216. def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
  217. return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
  218. def execute_sql(self, sql: str, *args: object) -> Awaitable[List[Tuple]]:
  219. def r(txn: LoggingTransaction) -> List[Tuple]:
  220. txn.execute(sql, args)
  221. return txn.fetchall()
  222. return self.db_pool.runInteraction("execute_sql", r)
  223. def insert_many_txn(
  224. self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple]
  225. ) -> None:
  226. sql = "INSERT INTO %s (%s) VALUES (%s)" % (
  227. table,
  228. ", ".join(k for k in headers),
  229. ", ".join("%s" for _ in headers),
  230. )
  231. try:
  232. txn.executemany(sql, rows)
  233. except Exception:
  234. logger.exception("Failed to insert: %s", table)
  235. raise
  236. # Note: the parent method is an `async def`.
  237. def set_room_is_public(self, room_id: str, is_public: bool) -> NoReturn:
  238. raise Exception(
  239. "Attempt to set room_is_public during port_db: database not empty?"
  240. )
  241. class MockHomeserver:
  242. def __init__(self, config: HomeServerConfig):
  243. self.clock = Clock(reactor)
  244. self.config = config
  245. self.hostname = config.server.server_name
  246. self.version_string = SYNAPSE_VERSION
  247. def get_clock(self) -> Clock:
  248. return self.clock
  249. def get_reactor(self) -> ISynapseReactor:
  250. return reactor
  251. def get_instance_name(self) -> str:
  252. return "master"
  253. def should_send_federation(self) -> bool:
  254. return False
  255. def get_replication_notifier(self) -> ReplicationNotifier:
  256. return ReplicationNotifier()
  257. class Porter:
  258. def __init__(
  259. self,
  260. sqlite_config: Dict[str, Any],
  261. progress: "Progress",
  262. batch_size: int,
  263. hs_config: HomeServerConfig,
  264. ):
  265. self.sqlite_config = sqlite_config
  266. self.progress = progress
  267. self.batch_size = batch_size
  268. self.hs_config = hs_config
  269. async def setup_table(self, table: str) -> Tuple[str, int, int, int, int]:
  270. if table in APPEND_ONLY_TABLES:
  271. # It's safe to just carry on inserting.
  272. row = await self.postgres_store.db_pool.simple_select_one(
  273. table="port_from_sqlite3",
  274. keyvalues={"table_name": table},
  275. retcols=("forward_rowid", "backward_rowid"),
  276. allow_none=True,
  277. )
  278. total_to_port = None
  279. if row is None:
  280. if table == "sent_transactions":
  281. (
  282. forward_chunk,
  283. already_ported,
  284. total_to_port,
  285. ) = await self._setup_sent_transactions()
  286. backward_chunk = 0
  287. else:
  288. await self.postgres_store.db_pool.simple_insert(
  289. table="port_from_sqlite3",
  290. values={
  291. "table_name": table,
  292. "forward_rowid": 1,
  293. "backward_rowid": 0,
  294. },
  295. )
  296. forward_chunk = 1
  297. backward_chunk = 0
  298. already_ported = 0
  299. else:
  300. forward_chunk = row["forward_rowid"]
  301. backward_chunk = row["backward_rowid"]
  302. if total_to_port is None:
  303. already_ported, total_to_port = await self._get_total_count_to_port(
  304. table, forward_chunk, backward_chunk
  305. )
  306. else:
  307. def delete_all(txn: LoggingTransaction) -> None:
  308. txn.execute(
  309. "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,)
  310. )
  311. txn.execute("TRUNCATE %s CASCADE" % (table,))
  312. await self.postgres_store.execute(delete_all)
  313. await self.postgres_store.db_pool.simple_insert(
  314. table="port_from_sqlite3",
  315. values={"table_name": table, "forward_rowid": 1, "backward_rowid": 0},
  316. )
  317. forward_chunk = 1
  318. backward_chunk = 0
  319. already_ported, total_to_port = await self._get_total_count_to_port(
  320. table, forward_chunk, backward_chunk
  321. )
  322. return table, already_ported, total_to_port, forward_chunk, backward_chunk
  323. async def get_table_constraints(self) -> Dict[str, Set[str]]:
  324. """Returns a map of tables that have foreign key constraints to tables they depend on."""
  325. def _get_constraints(txn: LoggingTransaction) -> Dict[str, Set[str]]:
  326. # We can pull the information about foreign key constraints out from
  327. # the postgres schema tables.
  328. sql = """
  329. SELECT DISTINCT
  330. tc.table_name,
  331. ccu.table_name AS foreign_table_name
  332. FROM
  333. information_schema.table_constraints AS tc
  334. INNER JOIN information_schema.constraint_column_usage AS ccu
  335. USING (table_schema, constraint_name)
  336. WHERE tc.constraint_type = 'FOREIGN KEY'
  337. AND tc.table_name != ccu.table_name;
  338. """
  339. txn.execute(sql)
  340. results: Dict[str, Set[str]] = {}
  341. for table, foreign_table in txn:
  342. results.setdefault(table, set()).add(foreign_table)
  343. return results
  344. return await self.postgres_store.db_pool.runInteraction(
  345. "get_table_constraints", _get_constraints
  346. )
  347. async def handle_table(
  348. self,
  349. table: str,
  350. postgres_size: int,
  351. table_size: int,
  352. forward_chunk: int,
  353. backward_chunk: int,
  354. ) -> None:
  355. logger.info(
  356. "Table %s: %i/%i (rows %i-%i) already ported",
  357. table,
  358. postgres_size,
  359. table_size,
  360. backward_chunk + 1,
  361. forward_chunk - 1,
  362. )
  363. if not table_size:
  364. return
  365. self.progress.add_table(table, postgres_size, table_size)
  366. if table == "event_search":
  367. await self.handle_search_table(
  368. postgres_size, table_size, forward_chunk, backward_chunk
  369. )
  370. return
  371. if table in IGNORED_TABLES:
  372. self.progress.update(table, table_size) # Mark table as done
  373. return
  374. if table == "user_directory_stream_pos":
  375. # We need to make sure there is a single row, `(X, null), as that is
  376. # what synapse expects to be there.
  377. await self.postgres_store.db_pool.simple_insert(
  378. table=table, values={"stream_id": None}
  379. )
  380. self.progress.update(table, table_size) # Mark table as done
  381. return
  382. # We sweep over rowids in two directions: one forwards (rowids 1, 2, 3, ...)
  383. # and another backwards (rowids 0, -1, -2, ...).
  384. forward_select = (
  385. "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
  386. )
  387. backward_select = (
  388. "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid DESC LIMIT ?"
  389. % (table,)
  390. )
  391. do_forward = [True]
  392. do_backward = [True]
  393. while True:
  394. def r(
  395. txn: LoggingTransaction,
  396. ) -> Tuple[Optional[List[str]], List[Tuple], List[Tuple]]:
  397. forward_rows = []
  398. backward_rows = []
  399. if do_forward[0]:
  400. txn.execute(forward_select, (forward_chunk, self.batch_size))
  401. forward_rows = txn.fetchall()
  402. if not forward_rows:
  403. do_forward[0] = False
  404. if do_backward[0]:
  405. txn.execute(backward_select, (backward_chunk, self.batch_size))
  406. backward_rows = txn.fetchall()
  407. if not backward_rows:
  408. do_backward[0] = False
  409. if forward_rows or backward_rows:
  410. headers = [column[0] for column in txn.description]
  411. else:
  412. headers = None
  413. return headers, forward_rows, backward_rows
  414. headers, frows, brows = await self.sqlite_store.db_pool.runInteraction(
  415. "select", r
  416. )
  417. if frows or brows:
  418. assert headers is not None
  419. if frows:
  420. forward_chunk = max(row[0] for row in frows) + 1
  421. if brows:
  422. backward_chunk = min(row[0] for row in brows) - 1
  423. rows = frows + brows
  424. rows = self._convert_rows(table, headers, rows)
  425. def insert(txn: LoggingTransaction) -> None:
  426. assert headers is not None
  427. self.postgres_store.insert_many_txn(txn, table, headers[1:], rows)
  428. self.postgres_store.db_pool.simple_update_one_txn(
  429. txn,
  430. table="port_from_sqlite3",
  431. keyvalues={"table_name": table},
  432. updatevalues={
  433. "forward_rowid": forward_chunk,
  434. "backward_rowid": backward_chunk,
  435. },
  436. )
  437. await self.postgres_store.execute(insert)
  438. postgres_size += len(rows)
  439. self.progress.update(table, postgres_size)
  440. else:
  441. return
  442. async def handle_search_table(
  443. self,
  444. postgres_size: int,
  445. table_size: int,
  446. forward_chunk: int,
  447. backward_chunk: int,
  448. ) -> None:
  449. select = (
  450. "SELECT es.rowid, es.*, e.origin_server_ts, e.stream_ordering"
  451. " FROM event_search as es"
  452. " INNER JOIN events AS e USING (event_id, room_id)"
  453. " WHERE es.rowid >= ?"
  454. " ORDER BY es.rowid LIMIT ?"
  455. )
  456. while True:
  457. def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]:
  458. txn.execute(select, (forward_chunk, self.batch_size))
  459. rows = txn.fetchall()
  460. headers = [column[0] for column in txn.description]
  461. return headers, rows
  462. headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
  463. if rows:
  464. forward_chunk = rows[-1][0] + 1
  465. # We have to treat event_search differently since it has a
  466. # different structure in the two different databases.
  467. def insert(txn: LoggingTransaction) -> None:
  468. sql = (
  469. "INSERT INTO event_search (event_id, room_id, key,"
  470. " sender, vector, origin_server_ts, stream_ordering)"
  471. " VALUES (?,?,?,?,to_tsvector('english', ?),?,?)"
  472. )
  473. rows_dict = []
  474. for row in rows:
  475. d = dict(zip(headers, row))
  476. if "\0" in d["value"]:
  477. logger.warning("dropping search row %s", d)
  478. else:
  479. rows_dict.append(d)
  480. txn.executemany(
  481. sql,
  482. [
  483. (
  484. row["event_id"],
  485. row["room_id"],
  486. row["key"],
  487. row["sender"],
  488. row["value"],
  489. row["origin_server_ts"],
  490. row["stream_ordering"],
  491. )
  492. for row in rows_dict
  493. ],
  494. )
  495. self.postgres_store.db_pool.simple_update_one_txn(
  496. txn,
  497. table="port_from_sqlite3",
  498. keyvalues={"table_name": "event_search"},
  499. updatevalues={
  500. "forward_rowid": forward_chunk,
  501. "backward_rowid": backward_chunk,
  502. },
  503. )
  504. await self.postgres_store.execute(insert)
  505. postgres_size += len(rows)
  506. self.progress.update("event_search", postgres_size)
  507. else:
  508. return
  509. def build_db_store(
  510. self,
  511. db_config: DatabaseConnectionConfig,
  512. allow_outdated_version: bool = False,
  513. ) -> Store:
  514. """Builds and returns a database store using the provided configuration.
  515. Args:
  516. db_config: The database configuration
  517. allow_outdated_version: True to suppress errors about the database server
  518. version being too old to run a complete synapse
  519. Returns:
  520. The built Store object.
  521. """
  522. self.progress.set_state("Preparing %s" % db_config.config["name"])
  523. engine = create_engine(db_config.config)
  524. hs = MockHomeserver(self.hs_config)
  525. with make_conn(db_config, engine, "portdb") as db_conn:
  526. engine.check_database(
  527. db_conn, allow_outdated_version=allow_outdated_version
  528. )
  529. prepare_database(db_conn, engine, config=self.hs_config)
  530. # Type safety: ignore that we're using Mock homeservers here.
  531. store = Store(DatabasePool(hs, db_config, engine), db_conn, hs) # type: ignore[arg-type]
  532. db_conn.commit()
  533. return store
  534. async def run_background_updates_on_postgres(self) -> None:
  535. # Manually apply all background updates on the PostgreSQL database.
  536. postgres_ready = (
  537. await self.postgres_store.db_pool.updates.has_completed_background_updates()
  538. )
  539. if not postgres_ready:
  540. # Only say that we're running background updates when there are background
  541. # updates to run.
  542. self.progress.set_state("Running background updates on PostgreSQL")
  543. while not postgres_ready:
  544. await self.postgres_store.db_pool.updates.do_next_background_update(True)
  545. postgres_ready = await (
  546. self.postgres_store.db_pool.updates.has_completed_background_updates()
  547. )
  548. @staticmethod
  549. def _is_sqlite_autovacuum_enabled(txn: LoggingTransaction) -> bool:
  550. """
  551. Returns true if auto_vacuum is enabled in SQLite.
  552. https://www.sqlite.org/pragma.html#pragma_auto_vacuum
  553. Vacuuming changes the rowids on rows in the database.
  554. Auto-vacuuming is therefore dangerous when used in conjunction with this script.
  555. Note that the auto_vacuum setting can't be changed without performing
  556. a VACUUM after trying to change the pragma.
  557. """
  558. txn.execute("PRAGMA auto_vacuum")
  559. row = txn.fetchone()
  560. assert row is not None, "`PRAGMA auto_vacuum` did not give a row."
  561. (autovacuum_setting,) = row
  562. # 0 means off. 1 means full. 2 means incremental.
  563. return autovacuum_setting != 0
  564. async def run(self) -> None:
  565. """Ports the SQLite database to a PostgreSQL database.
  566. When a fatal error is met, its message is assigned to the global "end_error"
  567. variable. When this error comes with a stacktrace, its exec_info is assigned to
  568. the global "end_error_exec_info" variable.
  569. """
  570. global end_error
  571. try:
  572. # we allow people to port away from outdated versions of sqlite.
  573. self.sqlite_store = self.build_db_store(
  574. DatabaseConnectionConfig("master-sqlite", self.sqlite_config),
  575. allow_outdated_version=True,
  576. )
  577. # For safety, ensure auto_vacuums are disabled.
  578. if await self.sqlite_store.db_pool.runInteraction(
  579. "is_sqlite_autovacuum_enabled", self._is_sqlite_autovacuum_enabled
  580. ):
  581. end_error = (
  582. "auto_vacuum is enabled in the SQLite database."
  583. " (This is not the default configuration.)\n"
  584. " This script relies on rowids being consistent and must not"
  585. " be used if the database could be vacuumed between re-runs.\n"
  586. " To disable auto_vacuum, you need to stop Synapse and run the following SQL:\n"
  587. " PRAGMA auto_vacuum=off;\n"
  588. " VACUUM;"
  589. )
  590. return
  591. # Check if all background updates are done, abort if not.
  592. updates_complete = (
  593. await self.sqlite_store.db_pool.updates.has_completed_background_updates()
  594. )
  595. if not updates_complete:
  596. end_error = (
  597. "Pending background updates exist in the SQLite3 database."
  598. " Please start Synapse again and wait until every update has finished"
  599. " before running this script.\n"
  600. )
  601. return
  602. self.postgres_store = self.build_db_store(
  603. self.hs_config.database.get_single_database()
  604. )
  605. await self.run_background_updates_on_postgres()
  606. self.progress.set_state("Creating port tables")
  607. def create_port_table(txn: LoggingTransaction) -> None:
  608. txn.execute(
  609. "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
  610. " table_name varchar(100) NOT NULL UNIQUE,"
  611. " forward_rowid bigint NOT NULL,"
  612. " backward_rowid bigint NOT NULL"
  613. ")"
  614. )
  615. # The old port script created a table with just a "rowid" column.
  616. # We want people to be able to rerun this script from an old port
  617. # so that they can pick up any missing events that were not
  618. # ported across.
  619. def alter_table(txn: LoggingTransaction) -> None:
  620. txn.execute(
  621. "ALTER TABLE IF EXISTS port_from_sqlite3"
  622. " RENAME rowid TO forward_rowid"
  623. )
  624. txn.execute(
  625. "ALTER TABLE IF EXISTS port_from_sqlite3"
  626. " ADD backward_rowid bigint NOT NULL DEFAULT 0"
  627. )
  628. try:
  629. await self.postgres_store.db_pool.runInteraction(
  630. "alter_table", alter_table
  631. )
  632. except Exception:
  633. # On Error Resume Next
  634. pass
  635. await self.postgres_store.db_pool.runInteraction(
  636. "create_port_table", create_port_table
  637. )
  638. # Step 2. Set up sequences
  639. #
  640. # We do this before porting the tables so that event if we fail half
  641. # way through the postgres DB always have sequences that are greater
  642. # than their respective tables. If we don't then creating the
  643. # `DataStore` object will fail due to the inconsistency.
  644. self.progress.set_state("Setting up sequence generators")
  645. await self._setup_state_group_id_seq()
  646. await self._setup_user_id_seq()
  647. await self._setup_events_stream_seqs()
  648. await self._setup_sequence(
  649. "device_inbox_sequence", ("device_inbox", "device_federation_outbox")
  650. )
  651. await self._setup_sequence(
  652. "account_data_sequence",
  653. ("room_account_data", "room_tags_revisions", "account_data"),
  654. )
  655. await self._setup_sequence("receipts_sequence", ("receipts_linearized",))
  656. await self._setup_sequence("presence_stream_sequence", ("presence_stream",))
  657. await self._setup_auth_chain_sequence()
  658. # Step 3. Get tables.
  659. self.progress.set_state("Fetching tables")
  660. sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol(
  661. table="sqlite_master", keyvalues={"type": "table"}, retcol="name"
  662. )
  663. postgres_tables = await self.postgres_store.db_pool.simple_select_onecol(
  664. table="information_schema.tables",
  665. keyvalues={},
  666. retcol="distinct table_name",
  667. )
  668. tables = set(sqlite_tables) & set(postgres_tables)
  669. logger.info("Found %d tables", len(tables))
  670. # Step 4. Figure out what still needs copying
  671. self.progress.set_state("Checking on port progress")
  672. setup_res = await make_deferred_yieldable(
  673. defer.gatherResults(
  674. [
  675. run_in_background(self.setup_table, table)
  676. for table in tables
  677. if table not in ["schema_version", "applied_schema_deltas"]
  678. and not table.startswith("sqlite_")
  679. ],
  680. consumeErrors=True,
  681. )
  682. )
  683. # Map from table name to args passed to `handle_table`, i.e. a tuple
  684. # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
  685. tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}
  686. # Step 5. Do the copying.
  687. #
  688. # This is slightly convoluted as we need to ensure tables are ported
  689. # in the correct order due to foreign key constraints.
  690. self.progress.set_state("Copying to postgres")
  691. constraints = await self.get_table_constraints()
  692. tables_ported = set() # type: Set[str]
  693. while tables_to_port_info_map:
  694. # Pulls out all tables that are still to be ported and which
  695. # only depend on tables that are already ported (if any).
  696. tables_to_port = [
  697. table
  698. for table in tables_to_port_info_map
  699. if not constraints.get(table, set()) - tables_ported
  700. ]
  701. await make_deferred_yieldable(
  702. defer.gatherResults(
  703. [
  704. run_in_background(
  705. self.handle_table,
  706. table,
  707. *tables_to_port_info_map.pop(table),
  708. )
  709. for table in tables_to_port
  710. ],
  711. consumeErrors=True,
  712. )
  713. )
  714. tables_ported.update(tables_to_port)
  715. self.progress.done()
  716. except Exception as e:
  717. global end_error_exec_info
  718. end_error = str(e)
  719. # Type safety: we're in an exception handler, so the exc_info() tuple
  720. # will not be (None, None, None).
  721. end_error_exec_info = sys.exc_info() # type: ignore[assignment]
  722. logger.exception("")
  723. finally:
  724. reactor.stop()
  725. def _convert_rows(
  726. self, table: str, headers: List[str], rows: List[Tuple]
  727. ) -> List[Tuple]:
  728. bool_col_names = BOOLEAN_COLUMNS.get(table, [])
  729. bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]
  730. class BadValueException(Exception):
  731. pass
  732. def conv(j: int, col: object) -> object:
  733. if j in bool_cols:
  734. return bool(col)
  735. if isinstance(col, bytes):
  736. return bytearray(col)
  737. elif isinstance(col, str) and "\0" in col:
  738. logger.warning(
  739. "DROPPING ROW: NUL value in table %s col %s: %r",
  740. table,
  741. headers[j],
  742. col,
  743. )
  744. raise BadValueException()
  745. return col
  746. outrows = []
  747. for row in rows:
  748. try:
  749. outrows.append(
  750. tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
  751. )
  752. except BadValueException:
  753. pass
  754. return outrows
  755. async def _setup_sent_transactions(self) -> Tuple[int, int, int]:
  756. # Only save things from the last day
  757. yesterday = int(time.time() * 1000) - 86400000
  758. # And save the max transaction id from each destination
  759. select = (
  760. "SELECT rowid, * FROM sent_transactions WHERE rowid IN ("
  761. "SELECT max(rowid) FROM sent_transactions"
  762. " GROUP BY destination"
  763. ")"
  764. )
  765. def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]:
  766. txn.execute(select)
  767. rows = txn.fetchall()
  768. headers: List[str] = [column[0] for column in txn.description]
  769. ts_ind = headers.index("ts")
  770. return headers, [r for r in rows if r[ts_ind] < yesterday]
  771. headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
  772. rows = self._convert_rows("sent_transactions", headers, rows)
  773. inserted_rows = len(rows)
  774. if inserted_rows:
  775. max_inserted_rowid = max(r[0] for r in rows)
  776. def insert(txn: LoggingTransaction) -> None:
  777. self.postgres_store.insert_many_txn(
  778. txn, "sent_transactions", headers[1:], rows
  779. )
  780. await self.postgres_store.execute(insert)
  781. else:
  782. max_inserted_rowid = 0
  783. def get_start_id(txn: LoggingTransaction) -> int:
  784. txn.execute(
  785. "SELECT rowid FROM sent_transactions WHERE ts >= ?"
  786. " ORDER BY rowid ASC LIMIT 1",
  787. (yesterday,),
  788. )
  789. rows = txn.fetchall()
  790. if rows:
  791. return rows[0][0]
  792. else:
  793. return 1
  794. next_chunk = await self.sqlite_store.execute(get_start_id)
  795. next_chunk = max(max_inserted_rowid + 1, next_chunk)
  796. await self.postgres_store.db_pool.simple_insert(
  797. table="port_from_sqlite3",
  798. values={
  799. "table_name": "sent_transactions",
  800. "forward_rowid": next_chunk,
  801. "backward_rowid": 0,
  802. },
  803. )
  804. def get_sent_table_size(txn: LoggingTransaction) -> int:
  805. txn.execute(
  806. "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,)
  807. )
  808. result = txn.fetchone()
  809. assert result is not None
  810. return int(result[0])
  811. remaining_count = await self.sqlite_store.execute(get_sent_table_size)
  812. total_count = remaining_count + inserted_rows
  813. return next_chunk, inserted_rows, total_count
  814. async def _get_remaining_count_to_port(
  815. self, table: str, forward_chunk: int, backward_chunk: int
  816. ) -> int:
  817. frows = cast(
  818. List[Tuple[int]],
  819. await self.sqlite_store.execute_sql(
  820. "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), forward_chunk
  821. ),
  822. )
  823. brows = cast(
  824. List[Tuple[int]],
  825. await self.sqlite_store.execute_sql(
  826. "SELECT count(*) FROM %s WHERE rowid <= ?" % (table,), backward_chunk
  827. ),
  828. )
  829. return frows[0][0] + brows[0][0]
  830. async def _get_already_ported_count(self, table: str) -> int:
  831. rows = await self.postgres_store.execute_sql(
  832. "SELECT count(*) FROM %s" % (table,)
  833. )
  834. return rows[0][0]
  835. async def _get_total_count_to_port(
  836. self, table: str, forward_chunk: int, backward_chunk: int
  837. ) -> Tuple[int, int]:
  838. remaining, done = await make_deferred_yieldable(
  839. defer.gatherResults(
  840. [
  841. run_in_background(
  842. self._get_remaining_count_to_port,
  843. table,
  844. forward_chunk,
  845. backward_chunk,
  846. ),
  847. run_in_background(self._get_already_ported_count, table),
  848. ],
  849. )
  850. )
  851. remaining = int(remaining) if remaining else 0
  852. done = int(done) if done else 0
  853. return done, remaining + done
  854. async def _setup_state_group_id_seq(self) -> None:
  855. curr_id: Optional[
  856. int
  857. ] = await self.sqlite_store.db_pool.simple_select_one_onecol(
  858. table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True
  859. )
  860. if not curr_id:
  861. return
  862. def r(txn: LoggingTransaction) -> None:
  863. assert curr_id is not None
  864. next_id = curr_id + 1
  865. txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,))
  866. await self.postgres_store.db_pool.runInteraction("setup_state_group_id_seq", r)
  867. async def _setup_user_id_seq(self) -> None:
  868. curr_id = await self.sqlite_store.db_pool.runInteraction(
  869. "setup_user_id_seq", find_max_generated_user_id_localpart
  870. )
  871. def r(txn: LoggingTransaction) -> None:
  872. next_id = curr_id + 1
  873. txn.execute("ALTER SEQUENCE user_id_seq RESTART WITH %s", (next_id,))
  874. await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r)
  875. async def _setup_events_stream_seqs(self) -> None:
  876. """Set the event stream sequences to the correct values."""
  877. # We get called before we've ported the events table, so we need to
  878. # fetch the current positions from the SQLite store.
  879. curr_forward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
  880. table="events", keyvalues={}, retcol="MAX(stream_ordering)", allow_none=True
  881. )
  882. curr_backward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
  883. table="events",
  884. keyvalues={},
  885. retcol="MAX(-MIN(stream_ordering), 1)",
  886. allow_none=True,
  887. )
  888. def _setup_events_stream_seqs_set_pos(txn: LoggingTransaction) -> None:
  889. if curr_forward_id:
  890. txn.execute(
  891. "ALTER SEQUENCE events_stream_seq RESTART WITH %s",
  892. (curr_forward_id + 1,),
  893. )
  894. if curr_backward_id:
  895. txn.execute(
  896. "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
  897. (curr_backward_id + 1,),
  898. )
  899. await self.postgres_store.db_pool.runInteraction(
  900. "_setup_events_stream_seqs",
  901. _setup_events_stream_seqs_set_pos,
  902. )
  903. async def _setup_sequence(
  904. self, sequence_name: str, stream_id_tables: Iterable[str]
  905. ) -> None:
  906. """Set a sequence to the correct value."""
  907. current_stream_ids = []
  908. for stream_id_table in stream_id_tables:
  909. max_stream_id = cast(
  910. int,
  911. await self.sqlite_store.db_pool.simple_select_one_onecol(
  912. table=stream_id_table,
  913. keyvalues={},
  914. retcol="COALESCE(MAX(stream_id), 1)",
  915. allow_none=True,
  916. ),
  917. )
  918. current_stream_ids.append(max_stream_id)
  919. next_id = max(current_stream_ids) + 1
  920. def r(txn: LoggingTransaction) -> None:
  921. sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,)
  922. txn.execute(sql + " %s", (next_id,))
  923. await self.postgres_store.db_pool.runInteraction(
  924. "_setup_%s" % (sequence_name,), r
  925. )
  926. async def _setup_auth_chain_sequence(self) -> None:
  927. curr_chain_id: Optional[
  928. int
  929. ] = await self.sqlite_store.db_pool.simple_select_one_onecol(
  930. table="event_auth_chains",
  931. keyvalues={},
  932. retcol="MAX(chain_id)",
  933. allow_none=True,
  934. )
  935. def r(txn: LoggingTransaction) -> None:
  936. # Presumably there is at least one row in event_auth_chains.
  937. assert curr_chain_id is not None
  938. txn.execute(
  939. "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s",
  940. (curr_chain_id + 1,),
  941. )
  942. if curr_chain_id is not None:
  943. await self.postgres_store.db_pool.runInteraction(
  944. "_setup_event_auth_chain_id",
  945. r,
  946. )
  947. ##############################################
  948. # The following is simply UI stuff
  949. ##############################################
  950. class TableProgress(TypedDict):
  951. start: int
  952. num_done: int
  953. total: int
  954. perc: int
  955. class Progress:
  956. """Used to report progress of the port"""
  957. def __init__(self) -> None:
  958. self.tables: Dict[str, TableProgress] = {}
  959. self.start_time = int(time.time())
  960. def add_table(self, table: str, cur: int, size: int) -> None:
  961. self.tables[table] = {
  962. "start": cur,
  963. "num_done": cur,
  964. "total": size,
  965. "perc": int(cur * 100 / size),
  966. }
  967. def update(self, table: str, num_done: int) -> None:
  968. data = self.tables[table]
  969. data["num_done"] = num_done
  970. data["perc"] = int(num_done * 100 / data["total"])
  971. def done(self) -> None:
  972. pass
  973. def set_state(self, state: str) -> None:
  974. pass
  975. class CursesProgress(Progress):
  976. """Reports progress to a curses window"""
  977. def __init__(self, stdscr: "curses.window"):
  978. self.stdscr = stdscr
  979. curses.use_default_colors()
  980. curses.curs_set(0)
  981. curses.init_pair(1, curses.COLOR_RED, -1)
  982. curses.init_pair(2, curses.COLOR_GREEN, -1)
  983. self.last_update = 0.0
  984. self.finished = False
  985. self.total_processed = 0
  986. self.total_remaining = 0
  987. super(CursesProgress, self).__init__()
  988. def update(self, table: str, num_done: int) -> None:
  989. super(CursesProgress, self).update(table, num_done)
  990. self.total_processed = 0
  991. self.total_remaining = 0
  992. for data in self.tables.values():
  993. self.total_processed += data["num_done"] - data["start"]
  994. self.total_remaining += data["total"] - data["num_done"]
  995. self.render()
  996. def render(self, force: bool = False) -> None:
  997. now = time.time()
  998. if not force and now - self.last_update < 0.2:
  999. # reactor.callLater(1, self.render)
  1000. return
  1001. self.stdscr.clear()
  1002. rows, cols = self.stdscr.getmaxyx()
  1003. duration = int(now) - int(self.start_time)
  1004. minutes, seconds = divmod(duration, 60)
  1005. duration_str = "%02dm %02ds" % (minutes, seconds)
  1006. if self.finished:
  1007. status = "Time spent: %s (Done!)" % (duration_str,)
  1008. else:
  1009. if self.total_processed > 0:
  1010. left = float(self.total_remaining) / self.total_processed
  1011. est_remaining = (int(now) - self.start_time) * left
  1012. est_remaining_str = "%02dm %02ds remaining" % divmod(est_remaining, 60)
  1013. else:
  1014. est_remaining_str = "Unknown"
  1015. status = "Time spent: %s (est. remaining: %s)" % (
  1016. duration_str,
  1017. est_remaining_str,
  1018. )
  1019. self.stdscr.addstr(0, 0, status, curses.A_BOLD)
  1020. max_len = max(len(t) for t in self.tables.keys())
  1021. left_margin = 5
  1022. middle_space = 1
  1023. items = sorted(self.tables.items(), key=lambda i: (i[1]["perc"], i[0]))
  1024. for i, (table, data) in enumerate(items):
  1025. if i + 2 >= rows:
  1026. break
  1027. perc = data["perc"]
  1028. color = curses.color_pair(2) if perc == 100 else curses.color_pair(1)
  1029. self.stdscr.addstr(
  1030. i + 2, left_margin + max_len - len(table), table, curses.A_BOLD | color
  1031. )
  1032. size = 20
  1033. progress = "[%s%s]" % (
  1034. "#" * int(perc * size / 100),
  1035. " " * (size - int(perc * size / 100)),
  1036. )
  1037. self.stdscr.addstr(
  1038. i + 2,
  1039. left_margin + max_len + middle_space,
  1040. "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]),
  1041. )
  1042. if self.finished:
  1043. self.stdscr.addstr(rows - 1, 0, "Press any key to exit...")
  1044. self.stdscr.refresh()
  1045. self.last_update = time.time()
  1046. def done(self) -> None:
  1047. self.finished = True
  1048. self.render(True)
  1049. self.stdscr.getch()
  1050. def set_state(self, state: str) -> None:
  1051. self.stdscr.clear()
  1052. self.stdscr.addstr(0, 0, state + "...", curses.A_BOLD)
  1053. self.stdscr.refresh()
  1054. class TerminalProgress(Progress):
  1055. """Just prints progress to the terminal"""
  1056. def update(self, table: str, num_done: int) -> None:
  1057. super(TerminalProgress, self).update(table, num_done)
  1058. data = self.tables[table]
  1059. print(
  1060. "%s: %d%% (%d/%d)" % (table, data["perc"], data["num_done"], data["total"])
  1061. )
  1062. def set_state(self, state: str) -> None:
  1063. print(state + "...")
  1064. ##############################################
  1065. ##############################################
  1066. def main() -> None:
  1067. parser = argparse.ArgumentParser(
  1068. description="A script to port an existing synapse SQLite database to"
  1069. " a new PostgreSQL database."
  1070. )
  1071. parser.add_argument("-v", action="store_true")
  1072. parser.add_argument(
  1073. "--sqlite-database",
  1074. required=True,
  1075. help="The snapshot of the SQLite database file. This must not be"
  1076. " currently used by a running synapse server",
  1077. )
  1078. parser.add_argument(
  1079. "--postgres-config",
  1080. type=argparse.FileType("r"),
  1081. required=True,
  1082. help="The database config file for the PostgreSQL database",
  1083. )
  1084. parser.add_argument(
  1085. "--curses", action="store_true", help="display a curses based progress UI"
  1086. )
  1087. parser.add_argument(
  1088. "--batch-size",
  1089. type=int,
  1090. default=1000,
  1091. help="The number of rows to select from the SQLite table each"
  1092. " iteration [default=1000]",
  1093. )
  1094. args = parser.parse_args()
  1095. logging.basicConfig(
  1096. level=logging.DEBUG if args.v else logging.INFO,
  1097. format="%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
  1098. filename="port-synapse.log" if args.curses else None,
  1099. )
  1100. sqlite_config = {
  1101. "name": "sqlite3",
  1102. "args": {
  1103. "database": "file:{}?mode=rw".format(args.sqlite_database),
  1104. "cp_min": 1,
  1105. "cp_max": 1,
  1106. "check_same_thread": False,
  1107. },
  1108. }
  1109. hs_config = yaml.safe_load(args.postgres_config)
  1110. if "database" not in hs_config:
  1111. sys.stderr.write("The configuration file must have a 'database' section.\n")
  1112. sys.exit(4)
  1113. postgres_config = hs_config["database"]
  1114. if "name" not in postgres_config:
  1115. sys.stderr.write("Malformed database config: no 'name'\n")
  1116. sys.exit(2)
  1117. if postgres_config["name"] != "psycopg2":
  1118. sys.stderr.write("Database must use the 'psycopg2' connector.\n")
  1119. sys.exit(3)
  1120. config = HomeServerConfig()
  1121. config.parse_config_dict(hs_config, "", "")
  1122. def start(stdscr: Optional["curses.window"] = None) -> None:
  1123. progress: Progress
  1124. if stdscr:
  1125. progress = CursesProgress(stdscr)
  1126. else:
  1127. progress = TerminalProgress()
  1128. porter = Porter(
  1129. sqlite_config=sqlite_config,
  1130. progress=progress,
  1131. batch_size=args.batch_size,
  1132. hs_config=config,
  1133. )
  1134. @defer.inlineCallbacks
  1135. def run() -> Generator["defer.Deferred[Any]", Any, None]:
  1136. with LoggingContext("synapse_port_db_run"):
  1137. yield defer.ensureDeferred(porter.run())
  1138. reactor.callWhenRunning(run)
  1139. reactor.run()
  1140. if args.curses:
  1141. curses.wrapper(start)
  1142. else:
  1143. start()
  1144. if end_error:
  1145. if end_error_exec_info:
  1146. exc_type, exc_value, exc_traceback = end_error_exec_info
  1147. traceback.print_exception(exc_type, exc_value, exc_traceback)
  1148. sys.stderr.write(end_error)
  1149. sys.exit(5)
  1150. if __name__ == "__main__":
  1151. main()