Avoid renaming configuration settings for now and rename internal code to use blocklist and allowlist instead.tags/v1.85.0rc1
@@ -1 +1 @@ | |||
Update internal terminology for workers. | |||
Update internal terminology. |
@@ -0,0 +1 @@ | |||
Update internal terminology. |
@@ -224,20 +224,20 @@ class ContentRepositoryConfig(Config): | |||
if "http" in proxy_env or "https" in proxy_env: | |||
logger.warning("".join(HTTP_PROXY_SET_WARNING)) | |||
# we always blacklist '0.0.0.0' and '::', which are supposed to be | |||
# we always block '0.0.0.0' and '::', which are supposed to be | |||
# unroutable addresses. | |||
self.url_preview_ip_range_blacklist = generate_ip_set( | |||
self.url_preview_ip_range_blocklist = generate_ip_set( | |||
config["url_preview_ip_range_blacklist"], | |||
["0.0.0.0", "::"], | |||
config_path=("url_preview_ip_range_blacklist",), | |||
) | |||
self.url_preview_ip_range_whitelist = generate_ip_set( | |||
self.url_preview_ip_range_allowlist = generate_ip_set( | |||
config.get("url_preview_ip_range_whitelist", ()), | |||
config_path=("url_preview_ip_range_whitelist",), | |||
) | |||
self.url_preview_url_blacklist = config.get("url_preview_url_blacklist", ()) | |||
self.url_preview_url_blocklist = config.get("url_preview_url_blacklist", ()) | |||
self.url_preview_accept_language = config.get( | |||
"url_preview_accept_language" | |||
@@ -115,7 +115,7 @@ def generate_ip_set( | |||
# IP ranges that are considered private / unroutable / don't make sense. | |||
DEFAULT_IP_RANGE_BLACKLIST = [ | |||
DEFAULT_IP_RANGE_BLOCKLIST = [ | |||
# Localhost | |||
"127.0.0.0/8", | |||
# Private networks. | |||
@@ -501,36 +501,36 @@ class ServerConfig(Config): | |||
# due to resource constraints | |||
self.admin_contact = config.get("admin_contact", None) | |||
ip_range_blacklist = config.get( | |||
"ip_range_blacklist", DEFAULT_IP_RANGE_BLACKLIST | |||
ip_range_blocklist = config.get( | |||
"ip_range_blacklist", DEFAULT_IP_RANGE_BLOCKLIST | |||
) | |||
# Attempt to create an IPSet from the given ranges | |||
# Always blacklist 0.0.0.0, :: | |||
self.ip_range_blacklist = generate_ip_set( | |||
ip_range_blacklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",) | |||
# Always block 0.0.0.0, :: | |||
self.ip_range_blocklist = generate_ip_set( | |||
ip_range_blocklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",) | |||
) | |||
self.ip_range_whitelist = generate_ip_set( | |||
self.ip_range_allowlist = generate_ip_set( | |||
config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",) | |||
) | |||
# The federation_ip_range_blacklist is used for backwards-compatibility | |||
# and only applies to federation and identity servers. | |||
if "federation_ip_range_blacklist" in config: | |||
# Always blacklist 0.0.0.0, :: | |||
self.federation_ip_range_blacklist = generate_ip_set( | |||
# Always block 0.0.0.0, :: | |||
self.federation_ip_range_blocklist = generate_ip_set( | |||
config["federation_ip_range_blacklist"], | |||
["0.0.0.0", "::"], | |||
config_path=("federation_ip_range_blacklist",), | |||
) | |||
# 'federation_ip_range_whitelist' was never a supported configuration option. | |||
self.federation_ip_range_whitelist = None | |||
self.federation_ip_range_allowlist = None | |||
else: | |||
# No backwards-compatiblity requrired, as federation_ip_range_blacklist | |||
# is not given. Default to ip_range_blacklist and ip_range_whitelist. | |||
self.federation_ip_range_blacklist = self.ip_range_blacklist | |||
self.federation_ip_range_whitelist = self.ip_range_whitelist | |||
self.federation_ip_range_blocklist = self.ip_range_blocklist | |||
self.federation_ip_range_allowlist = self.ip_range_allowlist | |||
# (undocumented) option for torturing the worker-mode replication a bit, | |||
# for testing. The value defines the number of milliseconds to pause before | |||
@@ -148,7 +148,7 @@ class FederationHandler: | |||
self._event_auth_handler = hs.get_event_auth_handler() | |||
self._server_notices_mxid = hs.config.servernotices.server_notices_mxid | |||
self.config = hs.config | |||
self.http_client = hs.get_proxied_blacklisted_http_client() | |||
self.http_client = hs.get_proxied_blocklisted_http_client() | |||
self._replication = hs.get_replication_data_handler() | |||
self._federation_event_handler = hs.get_federation_event_handler() | |||
self._device_handler = hs.get_device_handler() | |||
@@ -52,10 +52,10 @@ class IdentityHandler: | |||
# An HTTP client for contacting trusted URLs. | |||
self.http_client = SimpleHttpClient(hs) | |||
# An HTTP client for contacting identity servers specified by clients. | |||
self.blacklisting_http_client = SimpleHttpClient( | |||
self._http_client = SimpleHttpClient( | |||
hs, | |||
ip_blacklist=hs.config.server.federation_ip_range_blacklist, | |||
ip_whitelist=hs.config.server.federation_ip_range_whitelist, | |||
ip_blocklist=hs.config.server.federation_ip_range_blocklist, | |||
ip_allowlist=hs.config.server.federation_ip_range_allowlist, | |||
) | |||
self.federation_http_client = hs.get_federation_http_client() | |||
self.hs = hs | |||
@@ -197,7 +197,7 @@ class IdentityHandler: | |||
try: | |||
# Use the blacklisting http client as this call is only to identity servers | |||
# provided by a client | |||
data = await self.blacklisting_http_client.post_json_get_json( | |||
data = await self._http_client.post_json_get_json( | |||
bind_url, bind_data, headers=headers | |||
) | |||
@@ -308,9 +308,7 @@ class IdentityHandler: | |||
try: | |||
# Use the blacklisting http client as this call is only to identity servers | |||
# provided by a client | |||
await self.blacklisting_http_client.post_json_get_json( | |||
url, content, headers | |||
) | |||
await self._http_client.post_json_get_json(url, content, headers) | |||
changed = True | |||
except HttpResponseException as e: | |||
changed = False | |||
@@ -579,7 +577,7 @@ class IdentityHandler: | |||
""" | |||
# Check what hashing details are supported by this identity server | |||
try: | |||
hash_details = await self.blacklisting_http_client.get_json( | |||
hash_details = await self._http_client.get_json( | |||
"%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server), | |||
{"access_token": id_access_token}, | |||
) | |||
@@ -646,7 +644,7 @@ class IdentityHandler: | |||
headers = {"Authorization": create_id_access_token_header(id_access_token)} | |||
try: | |||
lookup_results = await self.blacklisting_http_client.post_json_get_json( | |||
lookup_results = await self._http_client.post_json_get_json( | |||
"%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server), | |||
{ | |||
"addresses": [lookup_value], | |||
@@ -752,7 +750,7 @@ class IdentityHandler: | |||
url = "%s%s/_matrix/identity/v2/store-invite" % (id_server_scheme, id_server) | |||
try: | |||
data = await self.blacklisting_http_client.post_json_get_json( | |||
data = await self._http_client.post_json_get_json( | |||
url, | |||
invite_config, | |||
{"Authorization": create_id_access_token_header(id_access_token)}, | |||
@@ -204,7 +204,7 @@ class SsoHandler: | |||
self._media_repo = ( | |||
hs.get_media_repository() if hs.config.media.can_load_media_repo else None | |||
) | |||
self._http_client = hs.get_proxied_blacklisted_http_client() | |||
self._http_client = hs.get_proxied_blocklisted_http_client() | |||
# The following template is shown after a successful user interactive | |||
# authentication session. It tells the user they can close the window. | |||
@@ -117,22 +117,22 @@ RawHeaderValue = Union[ | |||
] | |||
def check_against_blacklist( | |||
ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet | |||
def _is_ip_blocked( | |||
ip_address: IPAddress, allowlist: Optional[IPSet], blocklist: IPSet | |||
) -> bool: | |||
""" | |||
Compares an IP address to allowed and disallowed IP sets. | |||
Args: | |||
ip_address: The IP address to check | |||
ip_whitelist: Allowed IP addresses. | |||
ip_blacklist: Disallowed IP addresses. | |||
allowlist: Allowed IP addresses. | |||
blocklist: Disallowed IP addresses. | |||
Returns: | |||
True if the IP address is in the blacklist and not in the whitelist. | |||
True if the IP address is in the blocklist and not in the allowlist. | |||
""" | |||
if ip_address in ip_blacklist: | |||
if ip_whitelist is None or ip_address not in ip_whitelist: | |||
if ip_address in blocklist: | |||
if allowlist is None or ip_address not in allowlist: | |||
return True | |||
return False | |||
@@ -154,27 +154,27 @@ def _make_scheduler( | |||
return _scheduler | |||
class _IPBlacklistingResolver: | |||
class _IPBlockingResolver: | |||
""" | |||
A proxy for reactor.nameResolver which only produces non-blacklisted IP | |||
addresses, preventing DNS rebinding attacks on URL preview. | |||
A proxy for reactor.nameResolver which only produces non-blocklisted IP | |||
addresses, preventing DNS rebinding attacks. | |||
""" | |||
def __init__( | |||
self, | |||
reactor: IReactorPluggableNameResolver, | |||
ip_whitelist: Optional[IPSet], | |||
ip_blacklist: IPSet, | |||
ip_allowlist: Optional[IPSet], | |||
ip_blocklist: IPSet, | |||
): | |||
""" | |||
Args: | |||
reactor: The twisted reactor. | |||
ip_whitelist: IP addresses to allow. | |||
ip_blacklist: IP addresses to disallow. | |||
ip_allowlist: IP addresses to allow. | |||
ip_blocklist: IP addresses to disallow. | |||
""" | |||
self._reactor = reactor | |||
self._ip_whitelist = ip_whitelist | |||
self._ip_blacklist = ip_blacklist | |||
self._ip_allowlist = ip_allowlist | |||
self._ip_blocklist = ip_blocklist | |||
def resolveHostName( | |||
self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0 | |||
@@ -191,16 +191,13 @@ class _IPBlacklistingResolver: | |||
ip_address = IPAddress(address.host) | |||
if check_against_blacklist( | |||
ip_address, self._ip_whitelist, self._ip_blacklist | |||
): | |||
if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): | |||
logger.info( | |||
"Dropped %s from DNS resolution to %s due to blacklist" | |||
% (ip_address, hostname) | |||
"Blocked %s from DNS resolution to %s" % (ip_address, hostname) | |||
) | |||
has_bad_ip = True | |||
# if we have a blacklisted IP, we'd like to raise an error to block the | |||
# if we have a blocked IP, we'd like to raise an error to block the | |||
# request, but all we can really do from here is claim that there were no | |||
# valid results. | |||
if not has_bad_ip: | |||
@@ -232,24 +229,24 @@ class _IPBlacklistingResolver: | |||
# ISynapseReactor implies IReactorCore, but explicitly marking it this as an implementer | |||
# of IReactorCore seems to keep mypy-zope happier. | |||
@implementer(IReactorCore, ISynapseReactor) | |||
class BlacklistingReactorWrapper: | |||
class BlocklistingReactorWrapper: | |||
""" | |||
A Reactor wrapper which will prevent DNS resolution to blacklisted IP | |||
A Reactor wrapper which will prevent DNS resolution to blocked IP | |||
addresses, to prevent DNS rebinding. | |||
""" | |||
def __init__( | |||
self, | |||
reactor: IReactorPluggableNameResolver, | |||
ip_whitelist: Optional[IPSet], | |||
ip_blacklist: IPSet, | |||
ip_allowlist: Optional[IPSet], | |||
ip_blocklist: IPSet, | |||
): | |||
self._reactor = reactor | |||
# We need to use a DNS resolver which filters out blacklisted IP | |||
# We need to use a DNS resolver which filters out blocked IP | |||
# addresses, to prevent DNS rebinding. | |||
self._nameResolver = _IPBlacklistingResolver( | |||
self._reactor, ip_whitelist, ip_blacklist | |||
self._nameResolver = _IPBlockingResolver( | |||
self._reactor, ip_allowlist, ip_blocklist | |||
) | |||
def __getattr__(self, attr: str) -> Any: | |||
@@ -260,7 +257,7 @@ class BlacklistingReactorWrapper: | |||
return getattr(self._reactor, attr) | |||
class BlacklistingAgentWrapper(Agent): | |||
class BlocklistingAgentWrapper(Agent): | |||
""" | |||
An Agent wrapper which will prevent access to IP addresses being accessed | |||
directly (without an IP address lookup). | |||
@@ -269,18 +266,18 @@ class BlacklistingAgentWrapper(Agent): | |||
def __init__( | |||
self, | |||
agent: IAgent, | |||
ip_blacklist: IPSet, | |||
ip_whitelist: Optional[IPSet] = None, | |||
ip_blocklist: IPSet, | |||
ip_allowlist: Optional[IPSet] = None, | |||
): | |||
""" | |||
Args: | |||
agent: The Agent to wrap. | |||
ip_whitelist: IP addresses to allow. | |||
ip_blacklist: IP addresses to disallow. | |||
ip_allowlist: IP addresses to allow. | |||
ip_blocklist: IP addresses to disallow. | |||
""" | |||
self._agent = agent | |||
self._ip_whitelist = ip_whitelist | |||
self._ip_blacklist = ip_blacklist | |||
self._ip_allowlist = ip_allowlist | |||
self._ip_blocklist = ip_blocklist | |||
def request( | |||
self, | |||
@@ -299,13 +296,9 @@ class BlacklistingAgentWrapper(Agent): | |||
# Not an IP | |||
pass | |||
else: | |||
if check_against_blacklist( | |||
ip_address, self._ip_whitelist, self._ip_blacklist | |||
): | |||
logger.info("Blocking access to %s due to blacklist" % (ip_address,)) | |||
e = SynapseError( | |||
HTTPStatus.FORBIDDEN, "IP address blocked by IP blacklist entry" | |||
) | |||
if _is_ip_blocked(ip_address, self._ip_allowlist, self._ip_blocklist): | |||
logger.info("Blocking access to %s" % (ip_address,)) | |||
e = SynapseError(HTTPStatus.FORBIDDEN, "IP address blocked") | |||
return defer.fail(Failure(e)) | |||
return self._agent.request( | |||
@@ -763,10 +756,9 @@ class SimpleHttpClient(BaseHttpClient): | |||
Args: | |||
hs: The HomeServer instance to pass in | |||
treq_args: Extra keyword arguments to be given to treq.request. | |||
ip_blacklist: The IP addresses that are blacklisted that | |||
we may not request. | |||
ip_whitelist: The whitelisted IP addresses, that we can | |||
request if it were otherwise caught in a blacklist. | |||
ip_blocklist: The IP addresses that we may not request. | |||
ip_allowlist: The allowed IP addresses, that we can | |||
request if it were otherwise caught in a blocklist. | |||
use_proxy: Whether proxy settings should be discovered and used | |||
from conventional environment variables. | |||
""" | |||
@@ -775,19 +767,19 @@ class SimpleHttpClient(BaseHttpClient): | |||
self, | |||
hs: "HomeServer", | |||
treq_args: Optional[Dict[str, Any]] = None, | |||
ip_whitelist: Optional[IPSet] = None, | |||
ip_blacklist: Optional[IPSet] = None, | |||
ip_allowlist: Optional[IPSet] = None, | |||
ip_blocklist: Optional[IPSet] = None, | |||
use_proxy: bool = False, | |||
): | |||
super().__init__(hs, treq_args=treq_args) | |||
self._ip_whitelist = ip_whitelist | |||
self._ip_blacklist = ip_blacklist | |||
if self._ip_blacklist: | |||
# If we have an IP blacklist, we need to use a DNS resolver which | |||
# filters out blacklisted IP addresses, to prevent DNS rebinding. | |||
self.reactor: ISynapseReactor = BlacklistingReactorWrapper( | |||
self.reactor, self._ip_whitelist, self._ip_blacklist | |||
self._ip_allowlist = ip_allowlist | |||
self._ip_blocklist = ip_blocklist | |||
if self._ip_blocklist: | |||
# If we have an IP blocklist, we need to use a DNS resolver which | |||
# filters out blocked IP addresses, to prevent DNS rebinding. | |||
self.reactor: ISynapseReactor = BlocklistingReactorWrapper( | |||
self.reactor, self._ip_allowlist, self._ip_blocklist | |||
) | |||
# the pusher makes lots of concurrent SSL connections to Sygnal, and tends to | |||
@@ -809,14 +801,13 @@ class SimpleHttpClient(BaseHttpClient): | |||
use_proxy=use_proxy, | |||
) | |||
if self._ip_blacklist: | |||
# If we have an IP blacklist, we then install the blacklisting Agent | |||
# which prevents direct access to IP addresses, that are not caught | |||
# by the DNS resolution. | |||
self.agent = BlacklistingAgentWrapper( | |||
if self._ip_blocklist: | |||
# If we have an IP blocklist, we then install the Agent which prevents | |||
# direct access to IP addresses, that are not caught by the DNS resolution. | |||
self.agent = BlocklistingAgentWrapper( | |||
self.agent, | |||
ip_blacklist=self._ip_blacklist, | |||
ip_whitelist=self._ip_whitelist, | |||
ip_blocklist=self._ip_blocklist, | |||
ip_allowlist=self._ip_allowlist, | |||
) | |||
@@ -36,7 +36,7 @@ from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResp | |||
from synapse.crypto.context_factory import FederationPolicyForHTTPS | |||
from synapse.http import proxyagent | |||
from synapse.http.client import BlacklistingAgentWrapper, BlacklistingReactorWrapper | |||
from synapse.http.client import BlocklistingAgentWrapper, BlocklistingReactorWrapper | |||
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint | |||
from synapse.http.federation.srv_resolver import Server, SrvResolver | |||
from synapse.http.federation.well_known_resolver import WellKnownResolver | |||
@@ -65,12 +65,12 @@ class MatrixFederationAgent: | |||
user_agent: | |||
The user agent header to use for federation requests. | |||
ip_whitelist: Allowed IP addresses. | |||
ip_allowlist: Allowed IP addresses. | |||
ip_blacklist: Disallowed IP addresses. | |||
ip_blocklist: Disallowed IP addresses. | |||
proxy_reactor: twisted reactor to use for connections to the proxy server | |||
reactor might have some blacklisting applied (i.e. for DNS queries), | |||
reactor might have some blocking applied (i.e. for DNS queries), | |||
but we need unblocked access to the proxy. | |||
_srv_resolver: | |||
@@ -87,17 +87,17 @@ class MatrixFederationAgent: | |||
reactor: ISynapseReactor, | |||
tls_client_options_factory: Optional[FederationPolicyForHTTPS], | |||
user_agent: bytes, | |||
ip_whitelist: Optional[IPSet], | |||
ip_blacklist: IPSet, | |||
ip_allowlist: Optional[IPSet], | |||
ip_blocklist: IPSet, | |||
_srv_resolver: Optional[SrvResolver] = None, | |||
_well_known_resolver: Optional[WellKnownResolver] = None, | |||
): | |||
# proxy_reactor is not blacklisted | |||
# proxy_reactor is not blocklisting reactor | |||
proxy_reactor = reactor | |||
# We need to use a DNS resolver which filters out blacklisted IP | |||
# We need to use a DNS resolver which filters out blocked IP | |||
# addresses, to prevent DNS rebinding. | |||
reactor = BlacklistingReactorWrapper(reactor, ip_whitelist, ip_blacklist) | |||
reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist) | |||
self._clock = Clock(reactor) | |||
self._pool = HTTPConnectionPool(reactor) | |||
@@ -120,7 +120,7 @@ class MatrixFederationAgent: | |||
if _well_known_resolver is None: | |||
_well_known_resolver = WellKnownResolver( | |||
reactor, | |||
agent=BlacklistingAgentWrapper( | |||
agent=BlocklistingAgentWrapper( | |||
ProxyAgent( | |||
reactor, | |||
proxy_reactor, | |||
@@ -128,7 +128,7 @@ class MatrixFederationAgent: | |||
contextFactory=tls_client_options_factory, | |||
use_proxy=True, | |||
), | |||
ip_blacklist=ip_blacklist, | |||
ip_blocklist=ip_blocklist, | |||
), | |||
user_agent=self.user_agent, | |||
) | |||
@@ -256,7 +256,7 @@ class MatrixHostnameEndpoint: | |||
Args: | |||
reactor: twisted reactor to use for underlying requests | |||
proxy_reactor: twisted reactor to use for connections to the proxy server. | |||
'reactor' might have some blacklisting applied (i.e. for DNS queries), | |||
'reactor' might have some blocking applied (i.e. for DNS queries), | |||
but we need unblocked access to the proxy. | |||
tls_client_options_factory: | |||
factory to use for fetching client tls options, or none to disable TLS. | |||
@@ -64,7 +64,7 @@ from synapse.api.errors import ( | |||
from synapse.crypto.context_factory import FederationPolicyForHTTPS | |||
from synapse.http import QuieterFileBodyProducer | |||
from synapse.http.client import ( | |||
BlacklistingAgentWrapper, | |||
BlocklistingAgentWrapper, | |||
BodyExceededMaxSize, | |||
ByteWriteable, | |||
_make_scheduler, | |||
@@ -392,15 +392,15 @@ class MatrixFederationHttpClient: | |||
self.reactor, | |||
tls_client_options_factory, | |||
user_agent.encode("ascii"), | |||
hs.config.server.federation_ip_range_whitelist, | |||
hs.config.server.federation_ip_range_blacklist, | |||
hs.config.server.federation_ip_range_allowlist, | |||
hs.config.server.federation_ip_range_blocklist, | |||
) | |||
# Use a BlacklistingAgentWrapper to prevent circumventing the IP | |||
# blacklist via IP literals in server names | |||
self.agent = BlacklistingAgentWrapper( | |||
# Use a BlocklistingAgentWrapper to prevent circumventing the IP | |||
# blocking via IP literals in server names | |||
self.agent = BlocklistingAgentWrapper( | |||
federation_agent, | |||
ip_blacklist=hs.config.server.federation_ip_range_blacklist, | |||
ip_blocklist=hs.config.server.federation_ip_range_blocklist, | |||
) | |||
self.clock = hs.get_clock() | |||
@@ -53,7 +53,7 @@ class ProxyAgent(_AgentBase): | |||
connections. | |||
proxy_reactor: twisted reactor to use for connections to the proxy server | |||
reactor might have some blacklisting applied (i.e. for DNS queries), | |||
reactor might have some blocking applied (i.e. for DNS queries), | |||
but we need unblocked access to the proxy. | |||
contextFactory: A factory for TLS contexts, to control the | |||
@@ -105,7 +105,7 @@ class UrlPreviewer: | |||
When Synapse is asked to preview a URL it does the following: | |||
1. Checks against a URL blacklist (defined as `url_preview_url_blacklist` in the | |||
1. Checks against a URL blocklist (defined as `url_preview_url_blacklist` in the | |||
config). | |||
2. Checks the URL against an in-memory cache and returns the result if it exists. (This | |||
is also used to de-duplicate processing of multiple in-flight requests at once.) | |||
@@ -167,8 +167,8 @@ class UrlPreviewer: | |||
self.client = SimpleHttpClient( | |||
hs, | |||
treq_args={"browser_like_redirects": True}, | |||
ip_whitelist=hs.config.media.url_preview_ip_range_whitelist, | |||
ip_blacklist=hs.config.media.url_preview_ip_range_blacklist, | |||
ip_allowlist=hs.config.media.url_preview_ip_range_allowlist, | |||
ip_blocklist=hs.config.media.url_preview_ip_range_blocklist, | |||
use_proxy=True, | |||
) | |||
self.media_repo = media_repo | |||
@@ -186,7 +186,7 @@ class UrlPreviewer: | |||
or instance_running_jobs == hs.get_instance_name() | |||
) | |||
self.url_preview_url_blacklist = hs.config.media.url_preview_url_blacklist | |||
self.url_preview_url_blocklist = hs.config.media.url_preview_url_blocklist | |||
self.url_preview_accept_language = hs.config.media.url_preview_accept_language | |||
# memory cache mapping urls to an ObservableDeferred returning | |||
@@ -391,7 +391,7 @@ class UrlPreviewer: | |||
True if the URL is blocked, False if it is allowed. | |||
""" | |||
url_tuple = urlsplit(url) | |||
for entry in self.url_preview_url_blacklist: | |||
for entry in self.url_preview_url_blocklist: | |||
match = True | |||
# Iterate over each entry. If *all* attributes of that entry match | |||
# the current URL, then reject it. | |||
@@ -426,7 +426,7 @@ class UrlPreviewer: | |||
# All fields matched, return true (the URL is blocked). | |||
if match: | |||
logger.warning("URL %s blocked by url_blacklist entry %s", url, entry) | |||
logger.warning("URL %s blocked by entry %s", url, entry) | |||
return match | |||
# No matches were found, the URL is allowed. | |||
@@ -472,7 +472,7 @@ class UrlPreviewer: | |||
except DNSLookupError: | |||
# DNS lookup returned no results | |||
# Note: This will also be the case if one of the resolved IP | |||
# addresses is blacklisted | |||
# addresses is blocked. | |||
raise SynapseError( | |||
502, | |||
"DNS resolution failure during URL preview generation", | |||
@@ -575,7 +575,7 @@ class UrlPreviewer: | |||
if self._is_url_blocked(url): | |||
raise SynapseError( | |||
403, "URL blocked by url pattern blacklist entry", Codes.UNKNOWN | |||
403, "URL blocked by url pattern blocklist entry", Codes.UNKNOWN | |||
) | |||
# TODO: we should probably honour robots.txt... except in practice | |||
@@ -143,7 +143,7 @@ class HttpPusher(Pusher): | |||
) | |||
self.url = url | |||
self.http_client = hs.get_proxied_blacklisted_http_client() | |||
self.http_client = hs.get_proxied_blocklisted_http_client() | |||
self.data_minus_url = {} | |||
self.data_minus_url.update(self.data) | |||
del self.data_minus_url["url"] | |||
@@ -454,15 +454,15 @@ class HomeServer(metaclass=abc.ABCMeta): | |||
return SimpleHttpClient(self, use_proxy=True) | |||
@cache_in_self | |||
def get_proxied_blacklisted_http_client(self) -> SimpleHttpClient: | |||
def get_proxied_blocklisted_http_client(self) -> SimpleHttpClient: | |||
""" | |||
An HTTP client that uses configured HTTP(S) proxies and blacklists IPs | |||
based on the IP range blacklist/whitelist. | |||
An HTTP client that uses configured HTTP(S) proxies and blocks IPs | |||
based on the configured IP ranges. | |||
""" | |||
return SimpleHttpClient( | |||
self, | |||
ip_whitelist=self.config.server.ip_range_whitelist, | |||
ip_blacklist=self.config.server.ip_range_blacklist, | |||
ip_allowlist=self.config.server.ip_range_allowlist, | |||
ip_blocklist=self.config.server.ip_range_blocklist, | |||
use_proxy=True, | |||
) | |||
@@ -565,9 +565,8 @@ class DatabasePool: | |||
# A set of tables that are not safe to use native upserts in. | |||
self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys()) | |||
# We add the user_directory_search table to the blacklist on SQLite | |||
# because the existing search table does not have an index, making it | |||
# unsafe to use native upserts. | |||
# The user_directory_search table is unsafe to use native upserts | |||
# on SQLite because the existing search table does not have an index. | |||
if isinstance(self.engine, Sqlite3Engine): | |||
self._unsafe_to_upsert_tables.add("user_directory_search") | |||
@@ -63,7 +63,7 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): | |||
class ServerACLsTestCase(unittest.TestCase): | |||
def test_blacklisted_server(self) -> None: | |||
def test_blocked_server(self) -> None: | |||
e = _create_acl_event({"allow": ["*"], "deny": ["evil.com"]}) | |||
logging.info("ACL event: %s", e.content) | |||
@@ -31,7 +31,7 @@ class TestSSOHandler(unittest.HomeserverTestCase): | |||
self.http_client.get_file.side_effect = mock_get_file | |||
self.http_client.user_agent = b"Synapse Test" | |||
hs = self.setup_test_homeserver( | |||
proxied_blacklisted_http_client=self.http_client | |||
proxied_blocklisted_http_client=self.http_client | |||
) | |||
return hs | |||
@@ -269,8 +269,8 @@ class MatrixFederationAgentTests(unittest.TestCase): | |||
reactor=cast(ISynapseReactor, self.reactor), | |||
tls_client_options_factory=self.tls_factory, | |||
user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. | |||
ip_whitelist=IPSet(), | |||
ip_blacklist=IPSet(), | |||
ip_allowlist=IPSet(), | |||
ip_blocklist=IPSet(), | |||
_srv_resolver=self.mock_resolver, | |||
_well_known_resolver=self.well_known_resolver, | |||
) | |||
@@ -997,8 +997,8 @@ class MatrixFederationAgentTests(unittest.TestCase): | |||
reactor=self.reactor, | |||
tls_client_options_factory=tls_factory, | |||
user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. | |||
ip_whitelist=IPSet(), | |||
ip_blacklist=IPSet(), | |||
ip_allowlist=IPSet(), | |||
ip_blocklist=IPSet(), | |||
_srv_resolver=self.mock_resolver, | |||
_well_known_resolver=WellKnownResolver( | |||
cast(ISynapseReactor, self.reactor), | |||
@@ -27,8 +27,8 @@ from twisted.web.iweb import UNKNOWN_LENGTH | |||
from synapse.api.errors import SynapseError | |||
from synapse.http.client import ( | |||
BlacklistingAgentWrapper, | |||
BlacklistingReactorWrapper, | |||
BlocklistingAgentWrapper, | |||
BlocklistingReactorWrapper, | |||
BodyExceededMaxSize, | |||
_DiscardBodyWithMaxSizeProtocol, | |||
read_body_with_max_size, | |||
@@ -140,7 +140,7 @@ class ReadBodyWithMaxSizeTests(TestCase): | |||
self.assertEqual(result.getvalue(), b"") | |||
class BlacklistingAgentTest(TestCase): | |||
class BlocklistingAgentTest(TestCase): | |||
def setUp(self) -> None: | |||
self.reactor, self.clock = get_clock() | |||
@@ -157,16 +157,16 @@ class BlacklistingAgentTest(TestCase): | |||
self.reactor.lookups[domain.decode()] = ip.decode() | |||
self.reactor.lookups[ip.decode()] = ip.decode() | |||
self.ip_whitelist = IPSet([self.allowed_ip.decode()]) | |||
self.ip_blacklist = IPSet(["5.0.0.0/8"]) | |||
self.ip_allowlist = IPSet([self.allowed_ip.decode()]) | |||
self.ip_blocklist = IPSet(["5.0.0.0/8"]) | |||
def test_reactor(self) -> None: | |||
"""Apply the blacklisting reactor and ensure it properly blocks connections to particular domains and IPs.""" | |||
"""Apply the blocklisting reactor and ensure it properly blocks connections to particular domains and IPs.""" | |||
agent = Agent( | |||
BlacklistingReactorWrapper( | |||
BlocklistingReactorWrapper( | |||
self.reactor, | |||
ip_whitelist=self.ip_whitelist, | |||
ip_blacklist=self.ip_blacklist, | |||
ip_allowlist=self.ip_allowlist, | |||
ip_blocklist=self.ip_blocklist, | |||
), | |||
) | |||
@@ -207,11 +207,11 @@ class BlacklistingAgentTest(TestCase): | |||
self.assertEqual(response.code, 200) | |||
def test_agent(self) -> None: | |||
"""Apply the blacklisting agent and ensure it properly blocks connections to particular IPs.""" | |||
agent = BlacklistingAgentWrapper( | |||
"""Apply the blocklisting agent and ensure it properly blocks connections to particular IPs.""" | |||
agent = BlocklistingAgentWrapper( | |||
Agent(self.reactor), | |||
ip_blacklist=self.ip_blacklist, | |||
ip_whitelist=self.ip_whitelist, | |||
ip_blocklist=self.ip_blocklist, | |||
ip_allowlist=self.ip_allowlist, | |||
) | |||
# The unsafe IPs should be rejected. | |||
@@ -231,11 +231,11 @@ class FederationClientTests(HomeserverTestCase): | |||
self.assertIsInstance(f.value, RequestSendFailed) | |||
self.assertIsInstance(f.value.inner_exception, ResponseNeverReceived) | |||
def test_client_ip_range_blacklist(self) -> None: | |||
"""Ensure that Synapse does not try to connect to blacklisted IPs""" | |||
def test_client_ip_range_blocklist(self) -> None: | |||
"""Ensure that Synapse does not try to connect to blocked IPs""" | |||
# Set up the ip_range blacklist | |||
self.hs.config.server.federation_ip_range_blacklist = IPSet( | |||
# Set up the ip_range blocklist | |||
self.hs.config.server.federation_ip_range_blocklist = IPSet( | |||
["127.0.0.0/8", "fe80::/64"] | |||
) | |||
self.reactor.lookups["internal"] = "127.0.0.1" | |||
@@ -243,7 +243,7 @@ class FederationClientTests(HomeserverTestCase): | |||
self.reactor.lookups["fine"] = "10.20.30.40" | |||
cl = MatrixFederationHttpClient(self.hs, None) | |||
# Try making a GET request to a blacklisted IPv4 address | |||
# Try making a GET request to a blocked IPv4 address | |||
# ------------------------------------------------------ | |||
# Make the request | |||
d = defer.ensureDeferred(cl.get_json("internal:8008", "foo/bar", timeout=10000)) | |||
@@ -261,7 +261,7 @@ class FederationClientTests(HomeserverTestCase): | |||
self.assertIsInstance(f.value, RequestSendFailed) | |||
self.assertIsInstance(f.value.inner_exception, DNSLookupError) | |||
# Try making a POST request to a blacklisted IPv6 address | |||
# Try making a POST request to a blocked IPv6 address | |||
# ------------------------------------------------------- | |||
# Make the request | |||
d = defer.ensureDeferred( | |||
@@ -278,11 +278,11 @@ class FederationClientTests(HomeserverTestCase): | |||
clients = self.reactor.tcpClients | |||
self.assertEqual(len(clients), 0) | |||
# Check that it was due to a blacklisted DNS lookup | |||
# Check that it was due to a blocked DNS lookup | |||
f = self.failureResultOf(d, RequestSendFailed) | |||
self.assertIsInstance(f.value.inner_exception, DNSLookupError) | |||
# Try making a GET request to a non-blacklisted IPv4 address | |||
# Try making a GET request to an allowed IPv4 address | |||
# ---------------------------------------------------------- | |||
# Make the request | |||
d = defer.ensureDeferred(cl.post_json("fine:8008", "foo/bar", timeout=10000)) | |||
@@ -32,7 +32,7 @@ from twisted.internet.protocol import Factory, Protocol | |||
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol | |||
from twisted.web.http import HTTPChannel | |||
from synapse.http.client import BlacklistingReactorWrapper | |||
from synapse.http.client import BlocklistingReactorWrapper | |||
from synapse.http.connectproxyclient import ProxyCredentials | |||
from synapse.http.proxyagent import ProxyAgent, parse_proxy | |||
@@ -684,11 +684,11 @@ class MatrixFederationAgentTests(TestCase): | |||
self.assertEqual(body, b"result") | |||
@patch.dict(os.environ, {"http_proxy": "proxy.com:8888"}) | |||
def test_http_request_via_proxy_with_blacklist(self) -> None: | |||
# The blacklist includes the configured proxy IP. | |||
def test_http_request_via_proxy_with_blocklist(self) -> None: | |||
# The blocklist includes the configured proxy IP. | |||
agent = ProxyAgent( | |||
BlacklistingReactorWrapper( | |||
self.reactor, ip_whitelist=None, ip_blacklist=IPSet(["1.0.0.0/8"]) | |||
BlocklistingReactorWrapper( | |||
self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"]) | |||
), | |||
self.reactor, | |||
use_proxy=True, | |||
@@ -730,11 +730,11 @@ class MatrixFederationAgentTests(TestCase): | |||
self.assertEqual(body, b"result") | |||
@patch.dict(os.environ, {"HTTPS_PROXY": "proxy.com"}) | |||
def test_https_request_via_uppercase_proxy_with_blacklist(self) -> None: | |||
# The blacklist includes the configured proxy IP. | |||
def test_https_request_via_uppercase_proxy_with_blocklist(self) -> None: | |||
# The blocklist includes the configured proxy IP. | |||
agent = ProxyAgent( | |||
BlacklistingReactorWrapper( | |||
self.reactor, ip_whitelist=None, ip_blacklist=IPSet(["1.0.0.0/8"]) | |||
BlocklistingReactorWrapper( | |||
self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"]) | |||
), | |||
self.reactor, | |||
contextFactory=get_test_https_policy(), | |||
@@ -123,17 +123,17 @@ class SimpleHttpClientTests(HomeserverTestCase): | |||
self.assertIsInstance(f.value, RequestTimedOutError) | |||
def test_client_ip_range_blacklist(self) -> None: | |||
"""Ensure that Synapse does not try to connect to blacklisted IPs""" | |||
def test_client_ip_range_blocklist(self) -> None: | |||
"""Ensure that Synapse does not try to connect to blocked IPs""" | |||
# Add some DNS entries we'll blacklist | |||
# Add some DNS entries we'll block | |||
self.reactor.lookups["internal"] = "127.0.0.1" | |||
self.reactor.lookups["internalv6"] = "fe80:0:0:0:0:8a2e:370:7337" | |||
ip_blacklist = IPSet(["127.0.0.0/8", "fe80::/64"]) | |||
ip_blocklist = IPSet(["127.0.0.0/8", "fe80::/64"]) | |||
cl = SimpleHttpClient(self.hs, ip_blacklist=ip_blacklist) | |||
cl = SimpleHttpClient(self.hs, ip_blocklist=ip_blocklist) | |||
# Try making a GET request to a blacklisted IPv4 address | |||
# Try making a GET request to a blocked IPv4 address | |||
# ------------------------------------------------------ | |||
# Make the request | |||
d = defer.ensureDeferred(cl.get_json("http://internal:8008/foo/bar")) | |||
@@ -145,7 +145,7 @@ class SimpleHttpClientTests(HomeserverTestCase): | |||
self.failureResultOf(d, DNSLookupError) | |||
# Try making a POST request to a blacklisted IPv6 address | |||
# Try making a POST request to a blocked IPv6 address | |||
# ------------------------------------------------------- | |||
# Make the request | |||
d = defer.ensureDeferred( | |||
@@ -159,10 +159,10 @@ class SimpleHttpClientTests(HomeserverTestCase): | |||
clients = self.reactor.tcpClients | |||
self.assertEqual(len(clients), 0) | |||
# Check that it was due to a blacklisted DNS lookup | |||
# Check that it was due to a blocked DNS lookup | |||
self.failureResultOf(d, DNSLookupError) | |||
# Try making a GET request to a non-blacklisted IPv4 address | |||
# Try making a GET request to a non-blocked IPv4 address | |||
# ---------------------------------------------------------- | |||
# Make the request | |||
d = defer.ensureDeferred(cl.get_json("http://testserv:8008/foo/bar")) | |||
@@ -52,7 +52,7 @@ class HTTPPusherTests(HomeserverTestCase): | |||
m.post_json_get_json = post_json_get_json | |||
hs = self.setup_test_homeserver(proxied_blacklisted_http_client=m) | |||
hs = self.setup_test_homeserver(proxied_blocklisted_http_client=m) | |||
return hs | |||
@@ -93,7 +93,7 @@ class PusherShardTestCase(BaseMultiWorkerStreamTestCase): | |||
self.make_worker_hs( | |||
"synapse.app.generic_worker", | |||
{"worker_name": "pusher1", "pusher_instances": ["pusher1"]}, | |||
proxied_blacklisted_http_client=http_client_mock, | |||
proxied_blocklisted_http_client=http_client_mock, | |||
) | |||
event_id = self._create_pusher_and_send_msg("user") | |||
@@ -126,7 +126,7 @@ class PusherShardTestCase(BaseMultiWorkerStreamTestCase): | |||
"worker_name": "pusher1", | |||
"pusher_instances": ["pusher1", "pusher2"], | |||
}, | |||
proxied_blacklisted_http_client=http_client_mock1, | |||
proxied_blocklisted_http_client=http_client_mock1, | |||
) | |||
http_client_mock2 = Mock(spec_set=["post_json_get_json"]) | |||
@@ -140,7 +140,7 @@ class PusherShardTestCase(BaseMultiWorkerStreamTestCase): | |||
"worker_name": "pusher2", | |||
"pusher_instances": ["pusher1", "pusher2"], | |||
}, | |||
proxied_blacklisted_http_client=http_client_mock2, | |||
proxied_blocklisted_http_client=http_client_mock2, | |||
) | |||
# We choose a user name that we know should go to pusher1. | |||
@@ -418,9 +418,9 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
channel.json_body, {"og:title": "~matrix~", "og:description": "hi"} | |||
) | |||
def test_blacklisted_ip_specific(self) -> None: | |||
def test_blocked_ip_specific(self) -> None: | |||
""" | |||
Blacklisted IP addresses, found via DNS, are not spidered. | |||
Blocked IP addresses, found via DNS, are not spidered. | |||
""" | |||
self.lookups["example.com"] = [(IPv4Address, "192.168.1.1")] | |||
@@ -439,9 +439,9 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
}, | |||
) | |||
def test_blacklisted_ip_range(self) -> None: | |||
def test_blocked_ip_range(self) -> None: | |||
""" | |||
Blacklisted IP ranges, IPs found over DNS, are not spidered. | |||
Blocked IP ranges, IPs found over DNS, are not spidered. | |||
""" | |||
self.lookups["example.com"] = [(IPv4Address, "1.1.1.2")] | |||
@@ -458,9 +458,9 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
}, | |||
) | |||
def test_blacklisted_ip_specific_direct(self) -> None: | |||
def test_blocked_ip_specific_direct(self) -> None: | |||
""" | |||
Blacklisted IP addresses, accessed directly, are not spidered. | |||
Blocked IP addresses, accessed directly, are not spidered. | |||
""" | |||
channel = self.make_request( | |||
"GET", "preview_url?url=http://192.168.1.1", shorthand=False | |||
@@ -470,16 +470,13 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
self.assertEqual(len(self.reactor.tcpClients), 0) | |||
self.assertEqual( | |||
channel.json_body, | |||
{ | |||
"errcode": "M_UNKNOWN", | |||
"error": "IP address blocked by IP blacklist entry", | |||
}, | |||
{"errcode": "M_UNKNOWN", "error": "IP address blocked"}, | |||
) | |||
self.assertEqual(channel.code, 403) | |||
def test_blacklisted_ip_range_direct(self) -> None: | |||
def test_blocked_ip_range_direct(self) -> None: | |||
""" | |||
Blacklisted IP ranges, accessed directly, are not spidered. | |||
Blocked IP ranges, accessed directly, are not spidered. | |||
""" | |||
channel = self.make_request( | |||
"GET", "preview_url?url=http://1.1.1.2", shorthand=False | |||
@@ -488,15 +485,12 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
self.assertEqual(channel.code, 403) | |||
self.assertEqual( | |||
channel.json_body, | |||
{ | |||
"errcode": "M_UNKNOWN", | |||
"error": "IP address blocked by IP blacklist entry", | |||
}, | |||
{"errcode": "M_UNKNOWN", "error": "IP address blocked"}, | |||
) | |||
def test_blacklisted_ip_range_whitelisted_ip(self) -> None: | |||
def test_blocked_ip_range_whitelisted_ip(self) -> None: | |||
""" | |||
Blacklisted but then subsequently whitelisted IP addresses can be | |||
Blocked but then subsequently whitelisted IP addresses can be | |||
spidered. | |||
""" | |||
self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")] | |||
@@ -527,10 +521,10 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
channel.json_body, {"og:title": "~matrix~", "og:description": "hi"} | |||
) | |||
def test_blacklisted_ip_with_external_ip(self) -> None: | |||
def test_blocked_ip_with_external_ip(self) -> None: | |||
""" | |||
If a hostname resolves a blacklisted IP, even if there's a | |||
non-blacklisted one, it will be rejected. | |||
If a hostname resolves a blocked IP, even if there's a non-blocked one, | |||
it will be rejected. | |||
""" | |||
# Hardcode the URL resolving to the IP we want. | |||
self.lookups["example.com"] = [ | |||
@@ -550,9 +544,9 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
}, | |||
) | |||
def test_blacklisted_ipv6_specific(self) -> None: | |||
def test_blocked_ipv6_specific(self) -> None: | |||
""" | |||
Blacklisted IP addresses, found via DNS, are not spidered. | |||
Blocked IP addresses, found via DNS, are not spidered. | |||
""" | |||
self.lookups["example.com"] = [ | |||
(IPv6Address, "3fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") | |||
@@ -573,9 +567,9 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
}, | |||
) | |||
def test_blacklisted_ipv6_range(self) -> None: | |||
def test_blocked_ipv6_range(self) -> None: | |||
""" | |||
Blacklisted IP ranges, IPs found over DNS, are not spidered. | |||
Blocked IP ranges, IPs found over DNS, are not spidered. | |||
""" | |||
self.lookups["example.com"] = [(IPv6Address, "2001:800::1")] | |||
@@ -1359,7 +1353,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
@unittest.override_config({"url_preview_url_blacklist": [{"port": "*"}]}) | |||
def test_blocked_port(self) -> None: | |||
"""Tests that blacklisting URLs with a port makes previewing such URLs | |||
"""Tests that blocking URLs with a port makes previewing such URLs | |||
fail with a 403 error and doesn't impact other previews. | |||
""" | |||
self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] | |||
@@ -1401,7 +1395,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): | |||
{"url_preview_url_blacklist": [{"netloc": "example.com"}]} | |||
) | |||
def test_blocked_url(self) -> None: | |||
"""Tests that blacklisting URLs with a host makes previewing such URLs | |||
"""Tests that blocking URLs with a host makes previewing such URLs | |||
fail with a 403 error. | |||
""" | |||
self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")] | |||