@@ -16,6 +16,7 @@ _trial_temp*/ | |||
/*.log | |||
/*.log.config | |||
/*.pid | |||
/.python-version | |||
/*.signing.key | |||
/env/ | |||
/homeserver*.yaml | |||
@@ -0,0 +1 @@ | |||
Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. |
@@ -565,6 +565,13 @@ log_config: "CONFDIR/SERVERNAME.log.config" | |||
## Media Store ## | |||
# Enable the media store service in the Synapse master. Uncomment the | |||
# following if you are using a separate media store worker. | |||
# | |||
#enable_media_repo: false | |||
# Directory where uploaded images and attachments are stored. | |||
# | |||
media_store_path: "DATADIR/media_store" | |||
@@ -206,6 +206,13 @@ Handles the media repository. It can handle all endpoints starting with:: | |||
/_matrix/media/ | |||
And the following regular expressions matching media-specific administration | |||
APIs:: | |||
^/_synapse/admin/v1/purge_media_cache$ | |||
^/_synapse/admin/v1/room/.*/media$ | |||
^/_synapse/admin/v1/quarantine_media/.*$ | |||
You should also set ``enable_media_repo: False`` in the shared configuration | |||
file to stop the main synapse running background jobs related to managing the | |||
media repository. | |||
@@ -26,6 +26,7 @@ from synapse.app import _base | |||
from synapse.config._base import ConfigError | |||
from synapse.config.homeserver import HomeServerConfig | |||
from synapse.config.logger import setup_logging | |||
from synapse.http.server import JsonResource | |||
from synapse.http.site import SynapseSite | |||
from synapse.logging.context import LoggingContext | |||
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy | |||
@@ -35,6 +36,7 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore | |||
from synapse.replication.slave.storage.registration import SlavedRegistrationStore | |||
from synapse.replication.slave.storage.transactions import SlavedTransactionStore | |||
from synapse.replication.tcp.client import ReplicationClientHandler | |||
from synapse.rest.admin import register_servlets_for_media_repo | |||
from synapse.rest.media.v0.content_repository import ContentRepoResource | |||
from synapse.server import HomeServer | |||
from synapse.storage.engines import create_engine | |||
@@ -71,6 +73,12 @@ class MediaRepositoryServer(HomeServer): | |||
resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) | |||
elif name == "media": | |||
media_repo = self.get_media_repository_resource() | |||
# We need to serve the admin servlets for media on the | |||
# worker. | |||
admin_resource = JsonResource(self, canonical_json=False) | |||
register_servlets_for_media_repo(self, admin_resource) | |||
resources.update( | |||
{ | |||
MEDIA_PREFIX: media_repo, | |||
@@ -78,6 +86,7 @@ class MediaRepositoryServer(HomeServer): | |||
CONTENT_REPO_PREFIX: ContentRepoResource( | |||
self, self.config.uploads_path | |||
), | |||
"/_synapse/admin": admin_resource, | |||
} | |||
) | |||
@@ -12,6 +12,7 @@ | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
import os | |||
from collections import namedtuple | |||
@@ -87,6 +88,18 @@ def parse_thumbnail_requirements(thumbnail_sizes): | |||
class ContentRepositoryConfig(Config): | |||
def read_config(self, config, **kwargs): | |||
# Only enable the media repo if either the media repo is enabled or the | |||
# current worker app is the media repo. | |||
if ( | |||
self.enable_media_repo is False | |||
and config.worker_app != "synapse.app.media_repository" | |||
): | |||
self.can_load_media_repo = False | |||
return | |||
else: | |||
self.can_load_media_repo = True | |||
self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M")) | |||
self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M")) | |||
self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M")) | |||
@@ -202,6 +215,13 @@ class ContentRepositoryConfig(Config): | |||
return ( | |||
r""" | |||
## Media Store ## | |||
# Enable the media store service in the Synapse master. Uncomment the | |||
# following if you are using a separate media store worker. | |||
# | |||
#enable_media_repo: false | |||
# Directory where uploaded images and attachments are stored. | |||
# | |||
media_store_path: "%(media_store)s" | |||
@@ -27,7 +27,7 @@ from twisted.internet import defer | |||
import synapse | |||
from synapse.api.constants import Membership, UserTypes | |||
from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError | |||
from synapse.api.errors import Codes, NotFoundError, SynapseError | |||
from synapse.http.server import JsonResource | |||
from synapse.http.servlet import ( | |||
RestServlet, | |||
@@ -36,7 +36,12 @@ from synapse.http.servlet import ( | |||
parse_json_object_from_request, | |||
parse_string, | |||
) | |||
from synapse.rest.admin._base import assert_requester_is_admin, assert_user_is_admin | |||
from synapse.rest.admin._base import ( | |||
assert_requester_is_admin, | |||
assert_user_is_admin, | |||
historical_admin_path_patterns, | |||
) | |||
from synapse.rest.admin.media import register_servlets_for_media_repo | |||
from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet | |||
from synapse.types import UserID, create_requester | |||
from synapse.util.versionstring import get_version_string | |||
@@ -44,28 +49,6 @@ from synapse.util.versionstring import get_version_string | |||
logger = logging.getLogger(__name__) | |||
def historical_admin_path_patterns(path_regex): | |||
"""Returns the list of patterns for an admin endpoint, including historical ones | |||
This is a backwards-compatibility hack. Previously, the Admin API was exposed at | |||
various paths under /_matrix/client. This function returns a list of patterns | |||
matching those paths (as well as the new one), so that existing scripts which rely | |||
on the endpoints being available there are not broken. | |||
Note that this should only be used for existing endpoints: new ones should just | |||
register for the /_synapse/admin path. | |||
""" | |||
return list( | |||
re.compile(prefix + path_regex) | |||
for prefix in ( | |||
"^/_synapse/admin/v1", | |||
"^/_matrix/client/api/v1/admin", | |||
"^/_matrix/client/unstable/admin", | |||
"^/_matrix/client/r0/admin", | |||
) | |||
) | |||
class UsersRestServlet(RestServlet): | |||
PATTERNS = historical_admin_path_patterns("/users/(?P<user_id>[^/]*)") | |||
@@ -255,25 +238,6 @@ class WhoisRestServlet(RestServlet): | |||
return (200, ret) | |||
class PurgeMediaCacheRestServlet(RestServlet): | |||
PATTERNS = historical_admin_path_patterns("/purge_media_cache") | |||
def __init__(self, hs): | |||
self.media_repository = hs.get_media_repository() | |||
self.auth = hs.get_auth() | |||
@defer.inlineCallbacks | |||
def on_POST(self, request): | |||
yield assert_requester_is_admin(self.auth, request) | |||
before_ts = parse_integer(request, "before_ts", required=True) | |||
logger.info("before_ts: %r", before_ts) | |||
ret = yield self.media_repository.delete_old_remote_media(before_ts) | |||
return (200, ret) | |||
class PurgeHistoryRestServlet(RestServlet): | |||
PATTERNS = historical_admin_path_patterns( | |||
"/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?" | |||
@@ -542,50 +506,6 @@ class ShutdownRoomRestServlet(RestServlet): | |||
) | |||
class QuarantineMediaInRoom(RestServlet): | |||
"""Quarantines all media in a room so that no one can download it via | |||
this server. | |||
""" | |||
PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P<room_id>[^/]+)") | |||
def __init__(self, hs): | |||
self.store = hs.get_datastore() | |||
self.auth = hs.get_auth() | |||
@defer.inlineCallbacks | |||
def on_POST(self, request, room_id): | |||
requester = yield self.auth.get_user_by_req(request) | |||
yield assert_user_is_admin(self.auth, requester.user) | |||
num_quarantined = yield self.store.quarantine_media_ids_in_room( | |||
room_id, requester.user.to_string() | |||
) | |||
return (200, {"num_quarantined": num_quarantined}) | |||
class ListMediaInRoom(RestServlet): | |||
"""Lists all of the media in a given room. | |||
""" | |||
PATTERNS = historical_admin_path_patterns("/room/(?P<room_id>[^/]+)/media") | |||
def __init__(self, hs): | |||
self.store = hs.get_datastore() | |||
@defer.inlineCallbacks | |||
def on_GET(self, request, room_id): | |||
requester = yield self.auth.get_user_by_req(request) | |||
is_admin = yield self.auth.is_server_admin(requester.user) | |||
if not is_admin: | |||
raise AuthError(403, "You are not a server admin") | |||
local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id) | |||
return (200, {"local": local_mxcs, "remote": remote_mxcs}) | |||
class ResetPasswordRestServlet(RestServlet): | |||
"""Post request to allow an administrator reset password for a user. | |||
This needs user to have administrator access in Synapse. | |||
@@ -825,7 +745,6 @@ def register_servlets(hs, http_server): | |||
def register_servlets_for_client_rest_resource(hs, http_server): | |||
"""Register only the servlets which need to be exposed on /_matrix/client/xxx""" | |||
WhoisRestServlet(hs).register(http_server) | |||
PurgeMediaCacheRestServlet(hs).register(http_server) | |||
PurgeHistoryStatusRestServlet(hs).register(http_server) | |||
DeactivateAccountRestServlet(hs).register(http_server) | |||
PurgeHistoryRestServlet(hs).register(http_server) | |||
@@ -834,10 +753,13 @@ def register_servlets_for_client_rest_resource(hs, http_server): | |||
GetUsersPaginatedRestServlet(hs).register(http_server) | |||
SearchUsersRestServlet(hs).register(http_server) | |||
ShutdownRoomRestServlet(hs).register(http_server) | |||
QuarantineMediaInRoom(hs).register(http_server) | |||
ListMediaInRoom(hs).register(http_server) | |||
UserRegisterServlet(hs).register(http_server) | |||
DeleteGroupAdminRestServlet(hs).register(http_server) | |||
AccountValidityRenewServlet(hs).register(http_server) | |||
# Load the media repo ones if we're using them. | |||
if hs.config.can_load_media_repo: | |||
register_servlets_for_media_repo(hs, http_server) | |||
# don't add more things here: new servlets should only be exposed on | |||
# /_synapse/admin so should not go here. Instead register them in AdminRestResource. |
@@ -12,11 +12,36 @@ | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
import re | |||
from twisted.internet import defer | |||
from synapse.api.errors import AuthError | |||
def historical_admin_path_patterns(path_regex): | |||
"""Returns the list of patterns for an admin endpoint, including historical ones | |||
This is a backwards-compatibility hack. Previously, the Admin API was exposed at | |||
various paths under /_matrix/client. This function returns a list of patterns | |||
matching those paths (as well as the new one), so that existing scripts which rely | |||
on the endpoints being available there are not broken. | |||
Note that this should only be used for existing endpoints: new ones should just | |||
register for the /_synapse/admin path. | |||
""" | |||
return list( | |||
re.compile(prefix + path_regex) | |||
for prefix in ( | |||
"^/_synapse/admin/v1", | |||
"^/_matrix/client/api/v1/admin", | |||
"^/_matrix/client/unstable/admin", | |||
"^/_matrix/client/r0/admin", | |||
) | |||
) | |||
@defer.inlineCallbacks | |||
def assert_requester_is_admin(auth, request): | |||
"""Verify that the requester is an admin user | |||
@@ -0,0 +1,101 @@ | |||
# -*- coding: utf-8 -*- | |||
# Copyright 2014-2016 OpenMarket Ltd | |||
# Copyright 2018-2019 New Vector Ltd | |||
# | |||
# Licensed under the Apache License, Version 2.0 (the "License"); | |||
# you may not use this file except in compliance with the License. | |||
# You may obtain a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
import logging | |||
from twisted.internet import defer | |||
from synapse.api.errors import AuthError | |||
from synapse.http.servlet import RestServlet, parse_integer | |||
from synapse.rest.admin._base import ( | |||
assert_requester_is_admin, | |||
assert_user_is_admin, | |||
historical_admin_path_patterns, | |||
) | |||
logger = logging.getLogger(__name__) | |||
class QuarantineMediaInRoom(RestServlet): | |||
"""Quarantines all media in a room so that no one can download it via | |||
this server. | |||
""" | |||
PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P<room_id>[^/]+)") | |||
def __init__(self, hs): | |||
self.store = hs.get_datastore() | |||
self.auth = hs.get_auth() | |||
@defer.inlineCallbacks | |||
def on_POST(self, request, room_id): | |||
requester = yield self.auth.get_user_by_req(request) | |||
yield assert_user_is_admin(self.auth, requester.user) | |||
num_quarantined = yield self.store.quarantine_media_ids_in_room( | |||
room_id, requester.user.to_string() | |||
) | |||
return (200, {"num_quarantined": num_quarantined}) | |||
class ListMediaInRoom(RestServlet): | |||
"""Lists all of the media in a given room. | |||
""" | |||
PATTERNS = historical_admin_path_patterns("/room/(?P<room_id>[^/]+)/media") | |||
def __init__(self, hs): | |||
self.store = hs.get_datastore() | |||
@defer.inlineCallbacks | |||
def on_GET(self, request, room_id): | |||
requester = yield self.auth.get_user_by_req(request) | |||
is_admin = yield self.auth.is_server_admin(requester.user) | |||
if not is_admin: | |||
raise AuthError(403, "You are not a server admin") | |||
local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id) | |||
return (200, {"local": local_mxcs, "remote": remote_mxcs}) | |||
class PurgeMediaCacheRestServlet(RestServlet): | |||
PATTERNS = historical_admin_path_patterns("/purge_media_cache") | |||
def __init__(self, hs): | |||
self.media_repository = hs.get_media_repository() | |||
self.auth = hs.get_auth() | |||
@defer.inlineCallbacks | |||
def on_POST(self, request): | |||
yield assert_requester_is_admin(self.auth, request) | |||
before_ts = parse_integer(request, "before_ts", required=True) | |||
logger.info("before_ts: %r", before_ts) | |||
ret = yield self.media_repository.delete_old_remote_media(before_ts) | |||
return (200, ret) | |||
def register_servlets_for_media_repo(hs, http_server): | |||
""" | |||
Media repo specific APIs. | |||
""" | |||
PurgeMediaCacheRestServlet(hs).register(http_server) | |||
QuarantineMediaInRoom(hs).register(http_server) | |||
ListMediaInRoom(hs).register(http_server) |
@@ -33,6 +33,7 @@ from synapse.api.errors import ( | |||
RequestSendFailed, | |||
SynapseError, | |||
) | |||
from synapse.config._base import ConfigError | |||
from synapse.logging.context import defer_to_thread | |||
from synapse.metrics.background_process_metrics import run_as_background_process | |||
from synapse.util.async_helpers import Linearizer | |||
@@ -753,8 +754,11 @@ class MediaRepositoryResource(Resource): | |||
""" | |||
def __init__(self, hs): | |||
Resource.__init__(self) | |||
# If we're not configured to use it, raise if we somehow got here. | |||
if not hs.config.can_load_media_repo: | |||
raise ConfigError("Synapse is not configured to use a media repo.") | |||
super().__init__() | |||
media_repo = hs.get_media_repository() | |||
self.putChild(b"upload", UploadResource(hs, media_repo)) | |||