@@ -4,8 +4,12 @@ | |||
# things to include | |||
!docker | |||
!synapse | |||
!rust | |||
!README.rst | |||
!pyproject.toml | |||
!poetry.lock | |||
!build_rust.py | |||
rust/target | |||
**/__pycache__ |
@@ -15,7 +15,7 @@ on: | |||
concurrency: | |||
group: ${{ github.workflow }}-${{ github.ref }} | |||
cancel-in-progress: true | |||
permissions: | |||
contents: write | |||
@@ -89,9 +89,67 @@ jobs: | |||
name: debs | |||
path: debs/* | |||
build-wheels: | |||
name: Build wheels on ${{ matrix.os }} | |||
runs-on: ${{ matrix.os }} | |||
strategy: | |||
matrix: | |||
os: [ubuntu-20.04, macos-10.15] | |||
is_pr: | |||
- ${{ startsWith(github.ref, 'refs/pull/') }} | |||
exclude: | |||
# Don't build macos wheels on PR CI. | |||
- is_pr: true | |||
os: "macos-10.15" | |||
steps: | |||
- uses: actions/checkout@v3 | |||
- uses: actions/setup-python@v3 | |||
- name: Install cibuildwheel | |||
run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0 | |||
# Only build a single wheel in CI. | |||
- name: Set env vars. | |||
run: | | |||
echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV | |||
if: startsWith(github.ref, 'refs/pull/') | |||
- name: Build wheels | |||
run: python -m cibuildwheel --output-dir wheelhouse | |||
env: | |||
# Skip testing for platforms which various libraries don't have wheels | |||
# for, and so need extra build deps. | |||
CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx* | |||
- uses: actions/upload-artifact@v3 | |||
with: | |||
name: Wheel | |||
path: ./wheelhouse/*.whl | |||
build-sdist: | |||
name: "Build pypi distribution files" | |||
uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1" | |||
name: Build sdist | |||
runs-on: ubuntu-latest | |||
if: ${{ !startsWith(github.ref, 'refs/pull/') }} | |||
steps: | |||
- uses: actions/checkout@v3 | |||
- uses: actions/setup-python@v4 | |||
with: | |||
python-version: '3.10' | |||
- run: pip install build | |||
- name: Build sdist | |||
run: python -m build --sdist | |||
- uses: actions/upload-artifact@v2 | |||
with: | |||
name: Sdist | |||
path: dist/*.tar.gz | |||
# if it's a tag, create a release and attach the artifacts to it | |||
attach-assets: | |||
@@ -99,6 +157,7 @@ jobs: | |||
if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} | |||
needs: | |||
- build-debs | |||
- build-wheels | |||
- build-sdist | |||
runs-on: ubuntu-latest | |||
steps: | |||
@@ -139,6 +139,12 @@ jobs: | |||
steps: | |||
- uses: actions/checkout@v2 | |||
- name: Install Rust | |||
uses: actions-rs/toolchain@v1 | |||
with: | |||
toolchain: 1.61.0 | |||
override: true | |||
# There aren't wheels for some of the older deps, so we need to install | |||
# their build dependencies | |||
- run: | | |||
@@ -175,7 +181,7 @@ jobs: | |||
python-version: '3.7' | |||
extras: "all test" | |||
- run: poetry run trial -j 2 tests | |||
- run: poetry run trial -j2 tests | |||
- name: Dump logs | |||
# Logs are most useful when the command fails, always include them. | |||
if: ${{ always() }} | |||
@@ -247,6 +253,11 @@ jobs: | |||
- uses: actions/checkout@v2 | |||
- name: Prepare test blacklist | |||
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers | |||
- name: Install Rust | |||
uses: actions-rs/toolchain@v1 | |||
with: | |||
toolchain: 1.61.0 | |||
override: true | |||
- name: Run SyTest | |||
run: /bootstrap.sh synapse | |||
working-directory: /src | |||
@@ -353,6 +364,12 @@ jobs: | |||
with: | |||
path: synapse | |||
- name: Install Rust | |||
uses: actions-rs/toolchain@v1 | |||
with: | |||
toolchain: 1.61.0 | |||
override: true | |||
- name: Prepare Complement's Prerequisites | |||
run: synapse/.ci/scripts/setup_complement_prerequisites.sh | |||
@@ -60,3 +60,10 @@ book/ | |||
# complement | |||
/complement-* | |||
/master.tar.gz | |||
# rust | |||
/target/ | |||
/synapse/*.so | |||
# Poetry will create a setup.py, which we don't want to include. | |||
/setup.py |
@@ -0,0 +1,5 @@ | |||
# We make the whole Synapse folder a workspace so that we can run `cargo` | |||
# commands from the root (rather than having to cd into rust/). | |||
[workspace] | |||
members = ["rust"] |
@@ -0,0 +1,20 @@ | |||
# A build script for poetry that adds the rust extension. | |||
import os | |||
from typing import Any, Dict | |||
from setuptools_rust import Binding, RustExtension | |||
def build(setup_kwargs: Dict[str, Any]) -> None: | |||
original_project_dir = os.path.dirname(os.path.realpath(__file__)) | |||
cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml") | |||
extension = RustExtension( | |||
target="synapse.synapse_rust", | |||
path=cargo_toml_path, | |||
binding=Binding.PyO3, | |||
py_limited_api=True, | |||
) | |||
setup_kwargs.setdefault("rust_extensions", []).append(extension) | |||
setup_kwargs["zip_safe"] = False |
@@ -0,0 +1 @@ | |||
Add a stub Rust crate. |
@@ -61,7 +61,7 @@ dh_virtualenv \ | |||
--extras="all,systemd,test" \ | |||
--requirements="exported_requirements.txt" | |||
PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" | |||
PACKAGE_BUILD_DIR="$(pwd)/debian/matrix-synapse-py3" | |||
VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" | |||
TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python" | |||
@@ -78,9 +78,14 @@ case "$DEB_BUILD_OPTIONS" in | |||
cp -r tests "$tmpdir" | |||
# To avoid pulling in the unbuilt Synapse in the local directory | |||
pushd / | |||
PYTHONPATH="$tmpdir" \ | |||
"${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests | |||
popd | |||
;; | |||
esac | |||
@@ -12,11 +12,15 @@ matrix-synapse-py3 (1.66.0) stable; urgency=medium | |||
matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium | |||
[ Jörg Behrmann ] | |||
* Update debhelper to compatibility level 12. | |||
* Drop the preinst script stopping synapse. | |||
* Allocate a group for the system user. | |||
* Change dpkg-statoverride to --force-statoverride-add. | |||
[ Erik Johnston ] | |||
* Disable `dh_auto_configure` as it broke during Rust build. | |||
-- Jörg Behrmann <behrmann@physik.fu-berlin.de> Tue, 23 Aug 2022 17:17:00 +0100 | |||
matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium | |||
@@ -12,6 +12,8 @@ override_dh_installsystemd: | |||
# we don't really want to strip the symbols from our object files. | |||
override_dh_strip: | |||
override_dh_auto_configure: | |||
# many libraries pulled from PyPI have allocatable sections after | |||
# non-allocatable ones on which dwz errors out. For those without the issue the | |||
# gains are only marginal | |||
@@ -92,11 +92,20 @@ RUN \ | |||
libxml++2.6-dev \ | |||
libxslt1-dev \ | |||
openssl \ | |||
rustc \ | |||
zlib1g-dev \ | |||
git \ | |||
curl \ | |||
&& rm -rf /var/lib/apt/lists/* | |||
# Install rust and ensure its in the PATH | |||
ENV RUSTUP_HOME=/rust | |||
ENV CARGO_HOME=/cargo | |||
ENV PATH=/cargo/bin:/rust/bin:$PATH | |||
RUN mkdir /rust /cargo | |||
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable | |||
# To speed up rebuilds, install all of the dependencies before we copy over | |||
# the whole synapse project, so that this layer in the Docker cache can be | |||
# used while you develop on the source | |||
@@ -108,8 +117,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ | |||
# Copy over the rest of the synapse source code. | |||
COPY synapse /synapse/synapse/ | |||
COPY rust /synapse/rust/ | |||
# ... and what we need to `pip install`. | |||
COPY pyproject.toml README.rst /synapse/ | |||
COPY pyproject.toml README.rst build_rust.py /synapse/ | |||
# Repeat of earlier build argument declaration, as this is a new build stage. | |||
ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE | |||
@@ -72,6 +72,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \ | |||
&& env DEBIAN_FRONTEND=noninteractive apt-get install \ | |||
-yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ | |||
build-essential \ | |||
curl \ | |||
debhelper \ | |||
devscripts \ | |||
libsystemd-dev \ | |||
@@ -85,6 +86,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \ | |||
libpq-dev \ | |||
xmlsec1 | |||
# Install rust and ensure it's in the PATH | |||
ENV RUSTUP_HOME=/rust | |||
ENV CARGO_HOME=/cargo | |||
ENV PATH=/cargo/bin:/rust/bin:$PATH | |||
RUN mkdir /rust /cargo | |||
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable | |||
COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb / | |||
# install dhvirtualenv. Update the apt cache again first, in case we got a | |||
@@ -18,6 +18,12 @@ documented at [https://endoflife.date/python](https://endoflife.date/python) and | |||
[https://endoflife.date/postgresql](https://endoflife.date/postgresql). | |||
A Rust compiler is required to build Synapse from source. For any given release | |||
the minimum required version may be bumped up to a recent Rust version, and so | |||
people building from source should ensure they can fetch recent versions of Rust | |||
(e.g. by using [rustup](https://rustup.rs/)). | |||
Context | |||
------- | |||
@@ -31,3 +37,10 @@ long process. | |||
By following the upstream support life cycles Synapse can ensure that its | |||
dependencies continue to get security patches, while not requiring system admins | |||
to constantly update their platform dependencies to the latest versions. | |||
For Rust, the situation is a bit different given that a) the Rust foundation | |||
does not generally support older Rust versions, and b) the library ecosystem | |||
generally bump their minimum support Rust versions frequently. In general, the | |||
Synapse team will try to avoid updating the dependency on Rust to the absolute | |||
latest version, but introducing a formal policy is hard given the constraints of | |||
the ecosystem. |
@@ -28,6 +28,9 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver | |||
For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/). | |||
A recent version of the Rust compiler is needed to build the native modules. The | |||
easiest way of installing the latest version is to use [rustup](https://rustup.rs/). | |||
# 3. Get the source. | |||
@@ -114,6 +117,11 @@ Some documentation also exists in [Synapse's GitHub | |||
Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily | |||
contributed to by community authors. | |||
When changes are made to any Rust code then you must call either `poetry install` | |||
or `maturin develop` (if installed) to rebuild the Rust code. Using [`maturin`](https://github.com/PyO3/maturin) | |||
is quicker than `poetry install`, so is recommended when making frequent | |||
changes to the Rust code. | |||
# 8. Test, test, test! | |||
<a name="test-test-test"></a> | |||
@@ -195,7 +203,7 @@ The database file can then be inspected with: | |||
sqlite3 _trial_temp/test.db | |||
``` | |||
Note that the database file is cleared at the beginning of each test run. Thus it | |||
Note that the database file is cleared at the beginning of each test run. Thus it | |||
will always only contain the data generated by the *last run test*. Though generally | |||
when debugging, one is only running a single test anyway. | |||
@@ -196,6 +196,10 @@ System requirements: | |||
- Python 3.7 or later, up to Python 3.10. | |||
- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org | |||
If building on an uncommon architecture for which pre-built wheels are | |||
unavailable, you will need to have a recent Rust compiler installed. The easiest | |||
way of installing the latest version is to use [rustup](https://rustup.rs/). | |||
To install the Synapse homeserver run: | |||
```sh | |||
@@ -16,7 +16,8 @@ files = | |||
docker/, | |||
scripts-dev/, | |||
synapse/, | |||
tests/ | |||
tests/, | |||
build_rust.py | |||
# Note: Better exclusion syntax coming in mypy > 0.910 | |||
# https://github.com/python/mypy/pull/11329 | |||
@@ -181,3 +182,6 @@ ignore_missing_imports = True | |||
[mypy-incremental.*] | |||
ignore_missing_imports = True | |||
[mypy-setuptools_rust.*] | |||
ignore_missing_imports = True |
@@ -1035,6 +1035,18 @@ python-versions = ">=3.6" | |||
cryptography = ">=2.0" | |||
jeepney = ">=0.6" | |||
[[package]] | |||
name = "semantic-version" | |||
version = "2.10.0" | |||
description = "A library implementing the 'SemVer' scheme." | |||
category = "main" | |||
optional = false | |||
python-versions = ">=2.7" | |||
[package.extras] | |||
dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1)", "coverage", "flake8", "nose2", "readme-renderer (<25.0)", "tox", "wheel", "zest.releaser[recommended]"] | |||
doc = ["Sphinx", "sphinx-rtd-theme"] | |||
[[package]] | |||
name = "sentry-sdk" | |||
version = "1.5.11" | |||
@@ -1099,6 +1111,19 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g | |||
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] | |||
testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] | |||
[[package]] | |||
name = "setuptools-rust" | |||
version = "1.5.1" | |||
description = "Setuptools Rust extension plugin" | |||
category = "main" | |||
optional = false | |||
python-versions = ">=3.7" | |||
[package.dependencies] | |||
semantic-version = ">=2.8.2,<3" | |||
setuptools = ">=62.4" | |||
typing-extensions = ">=3.7.4.3" | |||
[[package]] | |||
name = "signedjson" | |||
version = "1.1.4" | |||
@@ -1600,7 +1625,7 @@ url_preview = ["lxml"] | |||
[metadata] | |||
lock-version = "1.1" | |||
python-versions = "^3.7.1" | |||
content-hash = "0df36bf75561fef340a7af704ed379b235f07a7d4a231aaccec5e7afb87159ca" | |||
content-hash = "79cfa09d59f9f8b5ef24318fb860df1915f54328692aa56d04331ecbdd92a8cb" | |||
[metadata.files] | |||
attrs = [ | |||
@@ -2472,6 +2497,10 @@ secretstorage = [ | |||
{file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, | |||
{file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, | |||
] | |||
semantic-version = [ | |||
{file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, | |||
{file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, | |||
] | |||
sentry-sdk = [ | |||
{file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, | |||
{file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, | |||
@@ -2484,6 +2513,10 @@ setuptools = [ | |||
{file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, | |||
{file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, | |||
] | |||
setuptools-rust = [ | |||
{file = "setuptools-rust-1.5.1.tar.gz", hash = "sha256:0e05e456645d59429cb1021370aede73c0760e9360bbfdaaefb5bced530eb9d7"}, | |||
{file = "setuptools_rust-1.5.1-py3-none-any.whl", hash = "sha256:306b236ff3aa5229180e58292610d0c2c51bb488191122d2fc559ae4caeb7d5e"}, | |||
] | |||
signedjson = [ | |||
{file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, | |||
{file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, | |||
@@ -52,6 +52,9 @@ include_trailing_comma = true | |||
combine_as_imports = true | |||
skip_gitignore = true | |||
[tool.maturin] | |||
manifest-path = "rust/Cargo.toml" | |||
[tool.poetry] | |||
name = "matrix-synapse" | |||
version = "1.66.0" | |||
@@ -82,8 +85,17 @@ include = [ | |||
{ path = "sytest-blacklist", format = "sdist" }, | |||
{ path = "tests", format = "sdist" }, | |||
{ path = "UPGRADE.rst", format = "sdist" }, | |||
{ path = "Cargo.toml", format = "sdist" }, | |||
{ path = "rust/Cargo.toml", format = "sdist" }, | |||
{ path = "rust/Cargo.lock", format = "sdist" }, | |||
{ path = "rust/src/**", format = "sdist" }, | |||
] | |||
exclude = [ | |||
{ path = "synapse/*.so", format = "sdist"} | |||
] | |||
build = "build_rust.py" | |||
[tool.poetry.scripts] | |||
synapse_homeserver = "synapse.app.homeserver:main" | |||
synapse_worker = "synapse.app.generic_worker:main" | |||
@@ -161,6 +173,15 @@ importlib_metadata = { version = ">=1.4", python = "<3.8" } | |||
# This is the most recent version of Pydantic with available on common distros. | |||
pydantic = ">=1.7.4" | |||
# This is for building the rust components during "poetry install", which | |||
# currently ignores the `build-system.requires` directive (c.f. | |||
# https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and | |||
# `poetry build` do the right thing without this explicit dependency. | |||
# | |||
# This isn't really a dev-dependency, as `poetry install --no-dev` will fail, | |||
# but the alternative is to add it to the main list of deps where it isn't | |||
# needed. | |||
setuptools_rust = ">=1.3" | |||
# Optional Dependencies | |||
@@ -285,5 +306,21 @@ twine = "*" | |||
towncrier = ">=18.6.0rc1" | |||
[build-system] | |||
requires = ["poetry-core>=1.0.0"] | |||
requires = ["poetry-core>=1.0.0", "setuptools_rust>=1.3"] | |||
build-backend = "poetry.core.masonry.api" | |||
[tool.cibuildwheel] | |||
# Skip unsupported platforms (by us or by Rust). | |||
skip = "cp36* *-musllinux_i686" | |||
# We need a rust compiler | |||
before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y" | |||
environment= { PATH = "$PATH:$HOME/.cargo/bin" } | |||
# For some reason if we don't manually clean the build directory we | |||
# can end up polluting the next build with a .so that is for the wrong | |||
# Python version. | |||
before-build = "rm -rf {project}/build" | |||
build-frontend = "build" | |||
test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print(sum_as_string(1, 2))'" |
@@ -0,0 +1,21 @@ | |||
[package] | |||
# We name the package `synapse` so that things like logging have the right | |||
# logging target. | |||
name = "synapse" | |||
# dummy version. See pyproject.toml for the Synapse's version number. | |||
version = "0.1.0" | |||
edition = "2021" | |||
rust-version = "1.61.0" | |||
[lib] | |||
name = "synapse" | |||
crate-type = ["cdylib"] | |||
[package.metadata.maturin] | |||
# This is where we tell maturin where to place the built library. | |||
name = "synapse.synapse_rust" | |||
[dependencies] | |||
pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } |
@@ -0,0 +1,15 @@ | |||
use pyo3::prelude::*; | |||
/// Formats the sum of two numbers as string. | |||
#[pyfunction] | |||
#[pyo3(text_signature = "(a, b, /)")] | |||
fn sum_as_string(a: usize, b: usize) -> PyResult<String> { | |||
Ok((a + b).to_string()) | |||
} | |||
/// The entry point for defining the Python module. | |||
#[pymodule] | |||
fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | |||
m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; | |||
Ok(()) | |||
} |
@@ -0,0 +1 @@ | |||
def sum_as_string(a: int, b: int) -> str: ... |
@@ -0,0 +1,11 @@ | |||
from synapse.synapse_rust import sum_as_string | |||
from tests import unittest | |||
class RustTestCase(unittest.TestCase): | |||
"""Basic tests to ensure that we can call into Rust code.""" | |||
def test_basic(self): | |||
result = sum_as_string(1, 2) | |||
self.assertEqual("3", result) |