The `exact_event_property_contains` condition can be used to search for a value inside of an array.tags/v1.78.0rc1
@@ -0,0 +1 @@ | |||
Experimental support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): the `exact_event_property_contains` push rule condition. |
@@ -15,8 +15,8 @@ | |||
#![feature(test)] | |||
use std::collections::BTreeSet; | |||
use synapse::push::{ | |||
evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, PushRules, | |||
SimpleJsonValue, | |||
evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue, | |||
PushRules, SimpleJsonValue, | |||
}; | |||
use test::Bencher; | |||
@@ -27,15 +27,15 @@ fn bench_match_exact(b: &mut Bencher) { | |||
let flattened_keys = [ | |||
( | |||
"type".to_string(), | |||
SimpleJsonValue::Str("m.text".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())), | |||
), | |||
( | |||
"room_id".to_string(), | |||
SimpleJsonValue::Str("!room:server".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())), | |||
), | |||
( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("test message".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())), | |||
), | |||
] | |||
.into_iter() | |||
@@ -54,6 +54,7 @@ fn bench_match_exact(b: &mut Bencher) { | |||
vec![], | |||
false, | |||
false, | |||
false, | |||
) | |||
.unwrap(); | |||
@@ -76,15 +77,15 @@ fn bench_match_word(b: &mut Bencher) { | |||
let flattened_keys = [ | |||
( | |||
"type".to_string(), | |||
SimpleJsonValue::Str("m.text".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())), | |||
), | |||
( | |||
"room_id".to_string(), | |||
SimpleJsonValue::Str("!room:server".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())), | |||
), | |||
( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("test message".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())), | |||
), | |||
] | |||
.into_iter() | |||
@@ -103,6 +104,7 @@ fn bench_match_word(b: &mut Bencher) { | |||
vec![], | |||
false, | |||
false, | |||
false, | |||
) | |||
.unwrap(); | |||
@@ -125,15 +127,15 @@ fn bench_match_word_miss(b: &mut Bencher) { | |||
let flattened_keys = [ | |||
( | |||
"type".to_string(), | |||
SimpleJsonValue::Str("m.text".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())), | |||
), | |||
( | |||
"room_id".to_string(), | |||
SimpleJsonValue::Str("!room:server".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())), | |||
), | |||
( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("test message".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())), | |||
), | |||
] | |||
.into_iter() | |||
@@ -152,6 +154,7 @@ fn bench_match_word_miss(b: &mut Bencher) { | |||
vec![], | |||
false, | |||
false, | |||
false, | |||
) | |||
.unwrap(); | |||
@@ -174,15 +177,15 @@ fn bench_eval_message(b: &mut Bencher) { | |||
let flattened_keys = [ | |||
( | |||
"type".to_string(), | |||
SimpleJsonValue::Str("m.text".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())), | |||
), | |||
( | |||
"room_id".to_string(), | |||
SimpleJsonValue::Str("!room:server".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())), | |||
), | |||
( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("test message".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())), | |||
), | |||
] | |||
.into_iter() | |||
@@ -201,6 +204,7 @@ fn bench_eval_message(b: &mut Bencher) { | |||
vec![], | |||
false, | |||
false, | |||
false, | |||
) | |||
.unwrap(); | |||
@@ -14,6 +14,7 @@ | |||
use std::collections::{BTreeMap, BTreeSet}; | |||
use crate::push::JsonValue; | |||
use anyhow::{Context, Error}; | |||
use lazy_static::lazy_static; | |||
use log::warn; | |||
@@ -63,7 +64,7 @@ impl RoomVersionFeatures { | |||
pub struct PushRuleEvaluator { | |||
/// A mapping of "flattened" keys to simple JSON values in the event, e.g. | |||
/// includes things like "type" and "content.msgtype". | |||
flattened_keys: BTreeMap<String, SimpleJsonValue>, | |||
flattened_keys: BTreeMap<String, JsonValue>, | |||
/// The "content.body", if any. | |||
body: String, | |||
@@ -87,7 +88,7 @@ pub struct PushRuleEvaluator { | |||
/// The related events, indexed by relation type. Flattened in the same manner as | |||
/// `flattened_keys`. | |||
related_events_flattened: BTreeMap<String, BTreeMap<String, SimpleJsonValue>>, | |||
related_events_flattened: BTreeMap<String, BTreeMap<String, JsonValue>>, | |||
/// If msc3664, push rules for related events, is enabled. | |||
related_event_match_enabled: bool, | |||
@@ -101,6 +102,9 @@ pub struct PushRuleEvaluator { | |||
/// If MSC3758 (exact_event_match push rule condition) is enabled. | |||
msc3758_exact_event_match: bool, | |||
/// If MSC3966 (exact_event_property_contains push rule condition) is enabled. | |||
msc3966_exact_event_property_contains: bool, | |||
} | |||
#[pymethods] | |||
@@ -109,21 +113,22 @@ impl PushRuleEvaluator { | |||
#[allow(clippy::too_many_arguments)] | |||
#[new] | |||
pub fn py_new( | |||
flattened_keys: BTreeMap<String, SimpleJsonValue>, | |||
flattened_keys: BTreeMap<String, JsonValue>, | |||
has_mentions: bool, | |||
user_mentions: BTreeSet<String>, | |||
room_mention: bool, | |||
room_member_count: u64, | |||
sender_power_level: Option<i64>, | |||
notification_power_levels: BTreeMap<String, i64>, | |||
related_events_flattened: BTreeMap<String, BTreeMap<String, SimpleJsonValue>>, | |||
related_events_flattened: BTreeMap<String, BTreeMap<String, JsonValue>>, | |||
related_event_match_enabled: bool, | |||
room_version_feature_flags: Vec<String>, | |||
msc3931_enabled: bool, | |||
msc3758_exact_event_match: bool, | |||
msc3966_exact_event_property_contains: bool, | |||
) -> Result<Self, Error> { | |||
let body = match flattened_keys.get("content.body") { | |||
Some(SimpleJsonValue::Str(s)) => s.clone(), | |||
Some(JsonValue::Value(SimpleJsonValue::Str(s))) => s.clone(), | |||
_ => String::new(), | |||
}; | |||
@@ -141,6 +146,7 @@ impl PushRuleEvaluator { | |||
room_version_feature_flags, | |||
msc3931_enabled, | |||
msc3758_exact_event_match, | |||
msc3966_exact_event_property_contains, | |||
}) | |||
} | |||
@@ -263,6 +269,9 @@ impl PushRuleEvaluator { | |||
KnownCondition::RelatedEventMatch(event_match) => { | |||
self.match_related_event_match(event_match, user_id)? | |||
} | |||
KnownCondition::ExactEventPropertyContains(exact_event_match) => { | |||
self.match_exact_event_property_contains(exact_event_match)? | |||
} | |||
KnownCondition::IsUserMention => { | |||
if let Some(uid) = user_id { | |||
self.user_mentions.contains(uid) | |||
@@ -345,7 +354,7 @@ impl PushRuleEvaluator { | |||
return Ok(false); | |||
}; | |||
let haystack = if let Some(SimpleJsonValue::Str(haystack)) = | |||
let haystack = if let Some(JsonValue::Value(SimpleJsonValue::Str(haystack))) = | |||
self.flattened_keys.get(&*event_match.key) | |||
{ | |||
haystack | |||
@@ -377,7 +386,9 @@ impl PushRuleEvaluator { | |||
let value = &exact_event_match.value; | |||
let haystack = if let Some(haystack) = self.flattened_keys.get(&*exact_event_match.key) { | |||
let haystack = if let Some(JsonValue::Value(haystack)) = | |||
self.flattened_keys.get(&*exact_event_match.key) | |||
{ | |||
haystack | |||
} else { | |||
return Ok(false); | |||
@@ -441,11 +452,12 @@ impl PushRuleEvaluator { | |||
return Ok(false); | |||
}; | |||
let haystack = if let Some(SimpleJsonValue::Str(haystack)) = event.get(&**key) { | |||
haystack | |||
} else { | |||
return Ok(false); | |||
}; | |||
let haystack = | |||
if let Some(JsonValue::Value(SimpleJsonValue::Str(haystack))) = event.get(&**key) { | |||
haystack | |||
} else { | |||
return Ok(false); | |||
}; | |||
// For the content.body we match against "words", but for everything | |||
// else we match against the entire value. | |||
@@ -459,6 +471,29 @@ impl PushRuleEvaluator { | |||
compiled_pattern.is_match(haystack) | |||
} | |||
/// Evaluates a `exact_event_property_contains` condition. (MSC3758) | |||
fn match_exact_event_property_contains( | |||
&self, | |||
exact_event_match: &ExactEventMatchCondition, | |||
) -> Result<bool, Error> { | |||
// First check if the feature is enabled. | |||
if !self.msc3966_exact_event_property_contains { | |||
return Ok(false); | |||
} | |||
let value = &exact_event_match.value; | |||
let haystack = if let Some(JsonValue::Array(haystack)) = | |||
self.flattened_keys.get(&*exact_event_match.key) | |||
{ | |||
haystack | |||
} else { | |||
return Ok(false); | |||
}; | |||
Ok(haystack.contains(&**value)) | |||
} | |||
/// Match the member count against an 'is' condition | |||
/// The `is` condition can be things like '>2', '==3' or even just '4'. | |||
fn match_member_count(&self, is: &str) -> Result<bool, Error> { | |||
@@ -488,7 +523,7 @@ fn push_rule_evaluator() { | |||
let mut flattened_keys = BTreeMap::new(); | |||
flattened_keys.insert( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("foo bar bob hello".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())), | |||
); | |||
let evaluator = PushRuleEvaluator::py_new( | |||
flattened_keys, | |||
@@ -503,6 +538,7 @@ fn push_rule_evaluator() { | |||
vec![], | |||
true, | |||
true, | |||
true, | |||
) | |||
.unwrap(); | |||
@@ -519,7 +555,7 @@ fn test_requires_room_version_supports_condition() { | |||
let mut flattened_keys = BTreeMap::new(); | |||
flattened_keys.insert( | |||
"content.body".to_string(), | |||
SimpleJsonValue::Str("foo bar bob hello".to_string()), | |||
JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())), | |||
); | |||
let flags = vec![RoomVersionFeatures::ExtensibleEvents.as_str().to_string()]; | |||
let evaluator = PushRuleEvaluator::py_new( | |||
@@ -535,6 +571,7 @@ fn test_requires_room_version_supports_condition() { | |||
flags, | |||
true, | |||
true, | |||
true, | |||
) | |||
.unwrap(); | |||
@@ -58,7 +58,7 @@ use anyhow::{Context, Error}; | |||
use log::warn; | |||
use pyo3::exceptions::PyTypeError; | |||
use pyo3::prelude::*; | |||
use pyo3::types::{PyBool, PyLong, PyString}; | |||
use pyo3::types::{PyBool, PyList, PyLong, PyString}; | |||
use pythonize::{depythonize, pythonize}; | |||
use serde::de::Error as _; | |||
use serde::{Deserialize, Serialize}; | |||
@@ -280,6 +280,35 @@ impl<'source> FromPyObject<'source> for SimpleJsonValue { | |||
} | |||
} | |||
/// A JSON values (list, string, int, boolean, or null). | |||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] | |||
#[serde(untagged)] | |||
pub enum JsonValue { | |||
Array(Vec<SimpleJsonValue>), | |||
Value(SimpleJsonValue), | |||
} | |||
impl<'source> FromPyObject<'source> for JsonValue { | |||
fn extract(ob: &'source PyAny) -> PyResult<Self> { | |||
if let Ok(l) = <PyList as pyo3::PyTryFrom>::try_from(ob) { | |||
match l.iter().map(SimpleJsonValue::extract).collect() { | |||
Ok(a) => Ok(JsonValue::Array(a)), | |||
Err(e) => Err(PyTypeError::new_err(format!( | |||
"Can't convert to JsonValue::Array: {}", | |||
e | |||
))), | |||
} | |||
} else if let Ok(v) = SimpleJsonValue::extract(ob) { | |||
Ok(JsonValue::Value(v)) | |||
} else { | |||
Err(PyTypeError::new_err(format!( | |||
"Can't convert from {} to JsonValue", | |||
ob.get_type().name()? | |||
))) | |||
} | |||
} | |||
} | |||
/// A condition used in push rules to match against an event. | |||
/// | |||
/// We need this split as `serde` doesn't give us the ability to have a | |||
@@ -303,6 +332,8 @@ pub enum KnownCondition { | |||
ExactEventMatch(ExactEventMatchCondition), | |||
#[serde(rename = "im.nheko.msc3664.related_event_match")] | |||
RelatedEventMatch(RelatedEventMatchCondition), | |||
#[serde(rename = "org.matrix.msc3966.exact_event_property_contains")] | |||
ExactEventPropertyContains(ExactEventMatchCondition), | |||
#[serde(rename = "org.matrix.msc3952.is_user_mention")] | |||
IsUserMention, | |||
#[serde(rename = "org.matrix.msc3952.is_room_mention")] | |||
@@ -14,7 +14,7 @@ | |||
from typing import Any, Collection, Dict, Mapping, Optional, Sequence, Set, Tuple, Union | |||
from synapse.types import JsonDict, SimpleJsonValue | |||
from synapse.types import JsonDict, JsonValue | |||
class PushRule: | |||
@property | |||
@@ -56,18 +56,19 @@ def get_base_rule_ids() -> Collection[str]: ... | |||
class PushRuleEvaluator: | |||
def __init__( | |||
self, | |||
flattened_keys: Mapping[str, SimpleJsonValue], | |||
flattened_keys: Mapping[str, JsonValue], | |||
has_mentions: bool, | |||
user_mentions: Set[str], | |||
room_mention: bool, | |||
room_member_count: int, | |||
sender_power_level: Optional[int], | |||
notification_power_levels: Mapping[str, int], | |||
related_events_flattened: Mapping[str, Mapping[str, SimpleJsonValue]], | |||
related_events_flattened: Mapping[str, Mapping[str, JsonValue]], | |||
related_event_match_enabled: bool, | |||
room_version_feature_flags: Tuple[str, ...], | |||
msc3931_enabled: bool, | |||
msc3758_exact_event_match: bool, | |||
msc3966_exact_event_property_contains: bool, | |||
): ... | |||
def run( | |||
self, | |||
@@ -188,3 +188,8 @@ class ExperimentalConfig(Config): | |||
self.msc3958_supress_edit_notifs = experimental.get( | |||
"msc3958_supress_edit_notifs", False | |||
) | |||
# MSC3966: exact_event_property_contains push rule condition. | |||
self.msc3966_exact_event_property_contains = experimental.get( | |||
"msc3966_exact_event_property_contains", False | |||
) |
@@ -44,7 +44,7 @@ from synapse.events.snapshot import EventContext | |||
from synapse.state import POWER_KEY | |||
from synapse.storage.databases.main.roommember import EventIdMembership | |||
from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator | |||
from synapse.types import SimpleJsonValue | |||
from synapse.types import JsonValue | |||
from synapse.types.state import StateFilter | |||
from synapse.util.caches import register_cache | |||
from synapse.util.metrics import measure_func | |||
@@ -259,13 +259,13 @@ class BulkPushRuleEvaluator: | |||
async def _related_events( | |||
self, event: EventBase | |||
) -> Dict[str, Dict[str, SimpleJsonValue]]: | |||
) -> Dict[str, Dict[str, JsonValue]]: | |||
"""Fetches the related events for 'event'. Sets the im.vector.is_falling_back key if the event is from a fallback relation | |||
Returns: | |||
Mapping of relation type to flattened events. | |||
""" | |||
related_events: Dict[str, Dict[str, SimpleJsonValue]] = {} | |||
related_events: Dict[str, Dict[str, JsonValue]] = {} | |||
if self._related_event_match_enabled: | |||
related_event_id = event.content.get("m.relates_to", {}).get("event_id") | |||
relation_type = event.content.get("m.relates_to", {}).get("rel_type") | |||
@@ -429,6 +429,7 @@ class BulkPushRuleEvaluator: | |||
event.room_version.msc3931_push_features, | |||
self.hs.config.experimental.msc1767_enabled, # MSC3931 flag | |||
self.hs.config.experimental.msc3758_exact_event_match, | |||
self.hs.config.experimental.msc3966_exact_event_property_contains, | |||
) | |||
users = rules_by_user.keys() | |||
@@ -502,18 +503,22 @@ RulesByUser = Dict[str, List[Rule]] | |||
StateGroup = Union[object, int] | |||
def _is_simple_value(value: Any) -> bool: | |||
return isinstance(value, (bool, str)) or type(value) is int or value is None | |||
def _flatten_dict( | |||
d: Union[EventBase, Mapping[str, Any]], | |||
prefix: Optional[List[str]] = None, | |||
result: Optional[Dict[str, SimpleJsonValue]] = None, | |||
result: Optional[Dict[str, JsonValue]] = None, | |||
*, | |||
msc3783_escape_event_match_key: bool = False, | |||
) -> Dict[str, SimpleJsonValue]: | |||
) -> Dict[str, JsonValue]: | |||
""" | |||
Given a JSON dictionary (or event) which might contain sub dictionaries, | |||
flatten it into a single layer dictionary by combining the keys & sub-keys. | |||
String, integer, boolean, and null values are kept. All others are dropped. | |||
String, integer, boolean, null or lists of those values are kept. All others are dropped. | |||
Transforms: | |||
@@ -542,8 +547,10 @@ def _flatten_dict( | |||
# nested fields. | |||
key = key.replace("\\", "\\\\").replace(".", "\\.") | |||
if isinstance(value, (bool, str)) or type(value) is int or value is None: | |||
if _is_simple_value(value): | |||
result[".".join(prefix + [key])] = value | |||
elif isinstance(value, (list, tuple)): | |||
result[".".join(prefix + [key])] = [v for v in value if _is_simple_value(v)] | |||
elif isinstance(value, Mapping): | |||
# do not set `room_version` due to recursion considerations below | |||
_flatten_dict( | |||
@@ -71,6 +71,7 @@ MutableStateMap = MutableMapping[StateKey, T] | |||
# JSON types. These could be made stronger, but will do for now. | |||
# A "simple" (canonical) JSON value. | |||
SimpleJsonValue = Optional[Union[str, int, bool]] | |||
JsonValue = Union[List[SimpleJsonValue], Tuple[SimpleJsonValue, ...], SimpleJsonValue] | |||
# A JSON-serialisable dict. | |||
JsonDict = Dict[str, Any] | |||
# A JSON-serialisable mapping; roughly speaking an immutable JSONDict. | |||
@@ -32,6 +32,7 @@ from synapse.storage.databases.main.appservice import _make_exclusive_regex | |||
from synapse.synapse_rust.push import PushRuleEvaluator | |||
from synapse.types import JsonDict, JsonMapping, UserID | |||
from synapse.util import Clock | |||
from synapse.util.frozenutils import freeze | |||
from tests import unittest | |||
from tests.test_utils.event_injection import create_event, inject_member_event | |||
@@ -57,17 +58,24 @@ class FlattenDictTestCase(unittest.TestCase): | |||
) | |||
def test_non_string(self) -> None: | |||
"""Booleans, ints, and nulls should be kept while other items are dropped.""" | |||
"""String, booleans, ints, nulls and list of those should be kept while other items are dropped.""" | |||
input: Dict[str, Any] = { | |||
"woo": "woo", | |||
"foo": True, | |||
"bar": 1, | |||
"baz": None, | |||
"fuzz": [], | |||
"fuzz": ["woo", True, 1, None, [], {}], | |||
"boo": {}, | |||
} | |||
self.assertEqual( | |||
{"woo": "woo", "foo": True, "bar": 1, "baz": None}, _flatten_dict(input) | |||
{ | |||
"woo": "woo", | |||
"foo": True, | |||
"bar": 1, | |||
"baz": None, | |||
"fuzz": ["woo", True, 1, None], | |||
}, | |||
_flatten_dict(input), | |||
) | |||
def test_event(self) -> None: | |||
@@ -117,6 +125,7 @@ class FlattenDictTestCase(unittest.TestCase): | |||
"room_id": "!test:test", | |||
"sender": "@alice:test", | |||
"type": "m.room.message", | |||
"content.org.matrix.msc1767.markup": [], | |||
} | |||
self.assertEqual(expected, _flatten_dict(event)) | |||
@@ -128,6 +137,7 @@ class FlattenDictTestCase(unittest.TestCase): | |||
"room_id": "!test:test", | |||
"sender": "@alice:test", | |||
"type": "m.room.message", | |||
"content.org.matrix.msc1767.markup": [], | |||
} | |||
self.assertEqual(expected, _flatten_dict(event)) | |||
@@ -169,6 +179,7 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): | |||
room_version_feature_flags=event.room_version.msc3931_push_features, | |||
msc3931_enabled=True, | |||
msc3758_exact_event_match=True, | |||
msc3966_exact_event_property_contains=True, | |||
) | |||
def test_display_name(self) -> None: | |||
@@ -549,6 +560,42 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): | |||
"incorrect types should not match", | |||
) | |||
def test_exact_event_property_contains(self) -> None: | |||
"""Check that exact_event_property_contains conditions work as expected.""" | |||
condition = { | |||
"kind": "org.matrix.msc3966.exact_event_property_contains", | |||
"key": "content.value", | |||
"value": "foobaz", | |||
} | |||
self._assert_matches( | |||
condition, | |||
{"value": ["foobaz"]}, | |||
"exact value should match", | |||
) | |||
self._assert_matches( | |||
condition, | |||
{"value": ["foobaz", "bugz"]}, | |||
"extra values should match", | |||
) | |||
self._assert_not_matches( | |||
condition, | |||
{"value": ["FoobaZ"]}, | |||
"values should match and be case-sensitive", | |||
) | |||
self._assert_not_matches( | |||
condition, | |||
{"value": "foobaz"}, | |||
"does not search in a string", | |||
) | |||
# it should work on frozendicts too | |||
self._assert_matches( | |||
condition, | |||
freeze({"value": ["foobaz"]}), | |||
"values should match on frozendicts", | |||
) | |||
def test_no_body(self) -> None: | |||
"""Not having a body shouldn't break the evaluator.""" | |||
evaluator = self._get_evaluator({}) | |||