kumo_api_types/
shaping.rs

1use crate::egress_path::EgressPathConfig;
2#[cfg(feature = "lua")]
3use anyhow::Context;
4#[cfg(feature = "lua")]
5use config::any_err;
6#[cfg(feature = "lua")]
7use config::serialize_options;
8#[cfg(feature = "lua")]
9use dns_resolver::fully_qualify;
10#[cfg(feature = "lua")]
11use kumo_log_types::JsonLogRecord;
12#[cfg(feature = "lua")]
13use mailexchanger::MailExchanger;
14#[cfg(feature = "lua")]
15use mlua::prelude::LuaUserData;
16#[cfg(feature = "lua")]
17use mlua::{LuaSerdeExt, UserDataMethods};
18use ordermap::OrderMap;
19use serde::de::{SeqAccess, Visitor};
20use serde::{Deserialize, Deserializer, Serialize};
21use serde_with::formats::PreferOne;
22use serde_with::{serde_as, DeserializeAs, OneOrMany};
23#[cfg(feature = "lua")]
24use sha2::{Digest, Sha256};
25#[cfg(feature = "lua")]
26use std::collections::BTreeMap;
27use std::hash::{Hash, Hasher};
28use std::marker::PhantomData;
29#[cfg(feature = "lua")]
30use std::sync::Arc;
31use std::time::Duration;
32#[cfg(feature = "lua")]
33use throttle::LimitSpec;
34use throttle::ThrottleSpec;
35
36#[derive(Deserialize, Serialize, Debug, Clone)]
37#[serde(try_from = "String", into = "String")]
38pub struct Regex(fancy_regex::Regex);
39
40impl TryFrom<String> for Regex {
41    type Error = fancy_regex::Error;
42
43    fn try_from(s: String) -> fancy_regex::Result<Self> {
44        Ok(Self(fancy_regex::Regex::new(&s)?))
45    }
46}
47
48impl From<Regex> for String {
49    fn from(r: Regex) -> String {
50        r.0.as_str().to_string()
51    }
52}
53
54impl std::ops::Deref for Regex {
55    type Target = fancy_regex::Regex;
56    fn deref(&self) -> &Self::Target {
57        &self.0
58    }
59}
60
61impl std::hash::Hash for Regex {
62    fn hash<H: std::hash::Hasher>(&self, hasher: &mut H) {
63        self.0.as_str().hash(hasher)
64    }
65}
66
67/// toml::Value is not Hash because it may contain floating
68/// point numbers, which are problematic from a Ord and Eq
69/// perspective. We're okay with skirting around that for
70/// our purposes here, so we implement our own hashable
71/// wrapper around the toml value.
72#[derive(Deserialize, Serialize, Debug, Clone)]
73#[serde(from = "toml::Value", into = "toml::Value")]
74pub struct HashableTomlValue {
75    value: toml::Value,
76}
77
78impl From<toml::Value> for HashableTomlValue {
79    fn from(value: toml::Value) -> Self {
80        Self { value }
81    }
82}
83
84impl From<HashableTomlValue> for toml::Value {
85    fn from(value: HashableTomlValue) -> toml::Value {
86        value.value
87    }
88}
89
90impl std::ops::Deref for HashableTomlValue {
91    type Target = toml::Value;
92    fn deref(&self) -> &toml::Value {
93        &self.value
94    }
95}
96
97fn hash_toml<H>(value: &toml::Value, h: &mut H)
98where
99    H: Hasher,
100{
101    match value {
102        toml::Value::Boolean(v) => v.hash(h),
103        toml::Value::Datetime(v) => {
104            if let Some(d) = &v.date {
105                d.year.hash(h);
106                d.month.hash(h);
107                d.day.hash(h);
108            }
109            if let Some(t) = &v.time {
110                t.hour.hash(h);
111                t.minute.hash(h);
112                t.second.hash(h);
113                t.nanosecond.hash(h);
114            }
115            if let Some(toml::value::Offset::Custom { minutes }) = &v.offset {
116                minutes.hash(h);
117            }
118        }
119        toml::Value::String(v) => v.hash(h),
120        toml::Value::Integer(v) => v.hash(h),
121        toml::Value::Float(v) => v.to_ne_bytes().hash(h),
122        toml::Value::Array(a) => {
123            for v in a.iter() {
124                hash_toml(v, h);
125            }
126        }
127        toml::Value::Table(m) => {
128            for (k, v) in m.iter() {
129                k.hash(h);
130                hash_toml(v, h);
131            }
132        }
133    }
134}
135
136impl Hash for HashableTomlValue {
137    fn hash<H>(&self, h: &mut H)
138    where
139        H: Hasher,
140    {
141        hash_toml(&self.value, h);
142    }
143}
144
145/// Represents an individual EgressPathConfig field name and value.
146/// It only allows deserializing from valid EgressPathConfig field + values.
147#[derive(Deserialize, Serialize, Debug, Clone, Hash)]
148#[serde(
149    try_from = "EgressPathConfigValueUnchecked",
150    into = "EgressPathConfigValueUnchecked"
151)]
152pub struct EgressPathConfigValue {
153    pub name: String,
154    pub value: HashableTomlValue,
155}
156
157/// This is the type that we actually use to deserialize EgressPathConfigValue items.
158/// It doesn't care about validity; it is used solely to tell serde what shape of
159/// data to expect.
160/// The validation is performed by the TryFrom impl that is used to convert to the
161/// checked form below.
162#[derive(Deserialize, Serialize, Debug, Clone)]
163pub struct EgressPathConfigValueUnchecked {
164    pub name: String,
165    pub value: toml::Value,
166}
167
168impl TryFrom<EgressPathConfigValueUnchecked> for EgressPathConfigValue {
169    type Error = anyhow::Error;
170    fn try_from(config: EgressPathConfigValueUnchecked) -> anyhow::Result<EgressPathConfigValue> {
171        let mut map = toml::map::Map::new();
172        map.insert(config.name.clone(), config.value.clone());
173        let table = toml::Value::Table(map);
174
175        // Attempt to deserialize as EgressPathConfig.
176        // If it fails, then the field name/value are invalid
177        EgressPathConfig::deserialize(table)?;
178
179        // If we reach this point, we can pass along the name/value
180        Ok(EgressPathConfigValue {
181            name: config.name,
182            value: HashableTomlValue {
183                value: config.value,
184            },
185        })
186    }
187}
188
189impl From<EgressPathConfigValue> for EgressPathConfigValueUnchecked {
190    fn from(config: EgressPathConfigValue) -> EgressPathConfigValueUnchecked {
191        EgressPathConfigValueUnchecked {
192            name: config.name,
193            value: config.value.value,
194        }
195    }
196}
197
198#[derive(Deserialize, Serialize, Debug, Clone, Hash)]
199pub enum Action {
200    Suspend,
201    SetConfig(EgressPathConfigValue),
202    SuspendTenant,
203    SuspendCampaign,
204    SetDomainConfig(EgressPathConfigValue),
205    SetProviderConfig(EgressPathConfigValue),
206    Bounce,
207    BounceTenant,
208    BounceCampaign,
209}
210
211#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash, Default)]
212pub enum Trigger {
213    /// Trigger on the first match, immediately
214    #[default]
215    Immediate,
216    /// Trigger when a certain number of matches occur
217    /// over a certain time period.
218    Threshold(ThrottleSpec),
219}
220
221#[serde_as]
222#[derive(Deserialize, Serialize, Debug, Hash, Clone)]
223pub struct Rule {
224    #[serde(deserialize_with = "regex_string_or_array")]
225    pub regex: Vec<Regex>,
226
227    #[serde(deserialize_with = "one_or_many_action")]
228    pub action: Vec<Action>,
229
230    #[serde(default)]
231    pub trigger: Trigger,
232
233    #[serde(with = "duration_serde")]
234    pub duration: Duration,
235
236    /// if true, this rule can match kumomta internally generated
237    /// Response messages, otherwise, the rule will skip testing
238    /// against those.
239    #[serde(default)]
240    pub match_internal: bool,
241}
242
243impl Rule {
244    pub fn matches(&self, is_internal: bool, response: &str) -> bool {
245        if is_internal && !self.match_internal {
246            return false;
247        }
248        self.regex
249            .iter()
250            .any(|r| r.is_match(response).unwrap_or(false))
251    }
252}
253
254/// Which kind of shaping block a [`Rule`] was reached through while
255/// matching one record.
256#[cfg(feature = "lua")]
257#[derive(Debug, Clone, Hash, PartialEq, Eq)]
258pub enum MatchScope {
259    /// The `default` block or a site-name block, applying across the whole
260    /// site_name rollup.
261    Rollup,
262    /// A specific domain block.
263    Domain,
264    /// A provider block, named by the provider.
265    Provider(String),
266}
267
268/// Pairs a [`Rule`] with the [`MatchScope`] it was found in while matching.
269#[cfg(feature = "lua")]
270#[derive(Debug, Clone)]
271pub struct MatchedRule {
272    pub rule: Rule,
273    pub scope: MatchScope,
274}
275
276#[cfg(feature = "lua")]
277impl MatchedRule {
278    /// Writes a hash of the rule and its [`MatchScope`] together.
279    pub fn hash_into<H: std::hash::Hasher>(&self, hasher: &mut H) {
280        self.rule.hash(hasher);
281        self.scope.hash(hasher);
282    }
283}
284
285#[cfg(feature = "lua")]
286#[derive(Debug, Default)]
287struct ShapingInner {
288    by_site: OrderMap<String, PartialEntry>,
289    by_domain: OrderMap<String, PartialEntry>,
290    by_provider: OrderMap<String, ProviderEntry>,
291    warnings: Vec<String>,
292    errors: Vec<String>,
293    hash: String,
294}
295
296#[cfg(feature = "lua")]
297impl ShapingInner {
298    pub async fn get_egress_path_config(
299        &self,
300        domain: &str,
301        egress_source: &str,
302        site_name: &str,
303    ) -> PartialEntry {
304        let mut params = PartialEntry::default();
305
306        // Apply basic/default configuration
307        if let Some(default) = self.by_domain.get("default") {
308            params.merge_from(default.clone());
309        }
310
311        // Provider rules come next.
312        // They can only match valid domain names, so we'll
313        // skip processing them if we have something like our
314        // TSA http "domain name" here
315        let is_domain_name = dns_resolver::Name::from_str_relaxed(domain).is_ok();
316        if is_domain_name {
317            let mut prov_with_sources = vec![];
318
319            for prov in self.by_provider.values() {
320                if prov.domain_matches(domain).await {
321                    toml_table_merge_from(&mut params.params, &prov.params);
322                    prov.apply_provider_params_to(egress_source, &mut params.params);
323
324                    if !prov.sources.is_empty() {
325                        // Remember this matching provider, so that we
326                        // can apply any source rules after we've applied
327                        // any/all base provider rules for this domain
328                        prov_with_sources.push(prov);
329                    }
330                }
331            }
332
333            // Then Provider source rules
334            for prov in prov_with_sources {
335                if let Some(source) = prov.sources.get(egress_source) {
336                    toml_table_merge_from(&mut params.params, source);
337                    prov.apply_provider_params_to(egress_source, &mut params.params);
338                }
339            }
340        }
341
342        // Then site config
343        if let Some(by_site) = self.by_site.get(site_name) {
344            params.merge_from(by_site.clone());
345        }
346
347        // Then domain config
348        if let Some(by_domain) = self.by_domain.get(domain) {
349            params.merge_from(by_domain.clone());
350        }
351
352        // Then source config for the site
353        if let Some(by_site) = self.by_site.get(site_name) {
354            if let Some(source) = by_site.sources.get(egress_source) {
355                toml_table_merge_from(&mut params.params, source);
356            }
357        }
358
359        // Then source config for the domain
360        if let Some(by_domain) = self.by_domain.get(domain) {
361            if let Some(source) = by_domain.sources.get(egress_source) {
362                toml_table_merge_from(&mut params.params, source);
363            }
364        }
365
366        params
367    }
368
369    pub async fn match_rules(&self, record: &JsonLogRecord) -> anyhow::Result<Vec<MatchedRule>> {
370        use rfc5321::parser::ForwardPath;
371        // Extract the domain from the recipient.
372        let recipient = ForwardPath::try_from(
373            record
374                .recipient
375                .first()
376                .ok_or_else(|| anyhow::anyhow!("no recipients!?"))?
377                .as_str(),
378        )
379        .map_err(|err| anyhow::anyhow!("parsing record.recipient: {err}"))?;
380
381        let recipient = match recipient {
382            ForwardPath::Postmaster => {
383                // It doesn't make sense to apply automation on the
384                // local postmaster address, so we ignore this.
385                return Ok(vec![]);
386            }
387            ForwardPath::Path(path) => path.mailbox,
388        };
389        let domain = recipient.domain.to_string();
390
391        // Track events/outcomes by site.
392        let source = record.egress_source.as_deref().unwrap_or("unspecified");
393        // record.site is poorly named; it is really an identifier for the
394        // egress path. For matching purposes, we want just the site_name
395        // in the form produced by our MX resolution process.
396        // In an earlier incarnation of this logic, we would resolve the
397        // site_name for ourselves based on other data in the record,
398        // but that could lead to over-resolution of some names and
399        // yield surprising results.
400        // What we do here is extract the egress path decoration from
401        // record.site to arrive at something that looks like the
402        // mx site_name.
403        // NOTE: this is coupled with the logic in
404        // ReadyQueueManager::compute_queue_name
405        let site_name = record
406            .site
407            .trim_start_matches(&format!("{source}->"))
408            .trim_end_matches("@smtp_client")
409            .to_string();
410
411        Ok(self.match_rules_impl(record, &domain, &site_name).await)
412    }
413
414    pub async fn match_rules_impl(
415        &self,
416        record: &JsonLogRecord,
417        domain: &str,
418        site_name: &str,
419    ) -> Vec<MatchedRule> {
420        let mut result = vec![];
421        let response = record.response.to_single_line();
422        tracing::trace!("Consider rules for {response}");
423
424        let is_internal = record.response.content.starts_with("KumoMTA internal: ");
425
426        if let Some(default) = self.by_domain.get("default") {
427            for rule in &default.automation {
428                tracing::trace!("Consider \"default\" rule {rule:?} for {response}");
429                if rule.matches(is_internal, &response) {
430                    // For automation under `default`, we always
431                    // assume that mx_rollup should be true.
432                    // If you somehow have a domain where that isn't
433                    // true, you should avoid using `default` for
434                    // automation.  Honestly, it's best to avoid
435                    // using `default` for automation.
436                    result.push(MatchedRule {
437                        rule: rule.clone(),
438                        scope: MatchScope::Rollup,
439                    });
440                }
441            }
442        }
443
444        for prov in self.by_provider.values() {
445            if prov.domain_matches(domain).await {
446                for rule in &prov.automation {
447                    tracing::trace!(
448                        "Consider provider \"{}\" rule {rule:?} for {response}",
449                        prov.provider_name
450                    );
451                    if rule.matches(is_internal, &response) {
452                        result.push(MatchedRule {
453                            rule: rule.clone(),
454                            scope: MatchScope::Provider(prov.provider_name.clone()),
455                        });
456                    }
457                }
458            }
459        }
460
461        // Then site config
462        if let Some(by_site) = self.by_site.get(site_name) {
463            for rule in &by_site.automation {
464                tracing::trace!("Consider \"{site_name}\" rule {rule:?} for {response}");
465                if rule.matches(is_internal, &response) {
466                    result.push(MatchedRule {
467                        rule: rule.clone(),
468                        scope: MatchScope::Rollup,
469                    });
470                }
471            }
472        }
473
474        // Then domain config
475        if let Some(by_domain) = self.by_domain.get(domain) {
476            for rule in &by_domain.automation {
477                tracing::trace!("Consider \"{domain}\" rule {rule:?} for {response}");
478                if rule.matches(is_internal, &response) {
479                    result.push(MatchedRule {
480                        rule: rule.clone(),
481                        scope: MatchScope::Domain,
482                    });
483                }
484            }
485        }
486
487        result
488    }
489}
490
491#[cfg(feature = "lua")]
492#[derive(Debug, Default, Clone, mlua::FromLua)]
493pub struct Shaping {
494    inner: Arc<ShapingInner>,
495}
496
497#[cfg(feature = "lua")]
498fn from_json<'a, T: Deserialize<'a>>(json: &'a str) -> anyhow::Result<T> {
499    let d = &mut serde_json::Deserializer::from_str(json);
500    Ok(serde_path_to_error::deserialize(d)?)
501}
502
503#[cfg(feature = "lua")]
504fn from_toml<'a, T: Deserialize<'a>>(toml: &'a str) -> anyhow::Result<T> {
505    let d = toml::Deserializer::parse(toml)?;
506    Ok(serde_path_to_error::deserialize(d)?)
507}
508
509#[cfg(feature = "lua")]
510#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
511pub enum CheckLevel {
512    #[default]
513    Ignore,
514    Warn,
515    Error,
516}
517#[cfg(feature = "lua")]
518impl std::str::FromStr for CheckLevel {
519    type Err = String;
520
521    fn from_str(s: &str) -> Result<Self, String> {
522        if s.eq_ignore_ascii_case("ignore") {
523            Ok(Self::Ignore)
524        } else if s.eq_ignore_ascii_case("warn") {
525            Ok(Self::Warn)
526        } else if s.eq_ignore_ascii_case("error") {
527            Ok(Self::Error)
528        } else {
529            Err(format!(
530                "Expected one of `Ignore`, `Warn` or `Error`, got `{s}`"
531            ))
532        }
533    }
534}
535
536#[cfg(feature = "lua")]
537#[derive(Default)]
538struct Collector {
539    warnings: Vec<String>,
540    errors: Vec<String>,
541}
542
543#[cfg(feature = "lua")]
544impl Collector {
545    fn push<S: Into<String>>(&mut self, level: CheckLevel, msg: S) {
546        match level {
547            CheckLevel::Ignore => {}
548            CheckLevel::Warn => self.warnings.push(msg.into()),
549            CheckLevel::Error => self.errors.push(msg.into()),
550        }
551    }
552}
553
554/// Reject a `SetProviderConfig` action found outside a provider block.
555/// The override it generates is attached to the provider that matched the
556/// rule, and a rule outside a provider block never matches one.
557#[cfg(feature = "lua")]
558fn reject_provider_only_actions(automation: &[Rule], scope: &str, collector: &mut Collector) {
559    for rule in automation {
560        if rule
561            .action
562            .iter()
563            .any(|action| matches!(action, Action::SetProviderConfig(_)))
564        {
565            collector.push(
566                CheckLevel::Error,
567                format!("SetProviderConfig is only valid within a provider block, but was used in {scope}"),
568            );
569        }
570    }
571}
572
573#[cfg(feature = "lua")]
574#[derive(Debug, Clone, Deserialize)]
575#[serde_as]
576#[serde(deny_unknown_fields)]
577pub struct ShapingMergeOptions {
578    #[serde(default)]
579    pub provider_overlap: CheckLevel,
580    #[serde(default)]
581    pub dns_fail: CheckLevel,
582    #[serde(default)]
583    pub null_mx: CheckLevel,
584    #[serde(default)]
585    pub aliased_site: CheckLevel,
586    #[serde(default)]
587    pub skip_remote: bool,
588    #[serde(default)]
589    pub remote_load: CheckLevel,
590    #[serde(default)]
591    pub local_load: CheckLevel,
592    #[serde(default, with = "duration_serde")]
593    pub http_timeout: Option<Duration>,
594}
595
596#[cfg(feature = "lua")]
597impl Default for ShapingMergeOptions {
598    fn default() -> Self {
599        Self {
600            provider_overlap: CheckLevel::Ignore,
601            dns_fail: CheckLevel::Ignore,
602            null_mx: CheckLevel::Ignore,
603            aliased_site: CheckLevel::Ignore,
604            skip_remote: false,
605            remote_load: CheckLevel::Ignore,
606            local_load: CheckLevel::Error,
607            http_timeout: None,
608        }
609    }
610}
611
612#[cfg(feature = "lua")]
613impl Shaping {
614    async fn load_from_file(
615        path: &str,
616        options: &ShapingMergeOptions,
617        collector: &mut Collector,
618    ) -> anyhow::Result<ShapingFile> {
619        let (data, level): (String, CheckLevel) =
620            if path.starts_with("http://") || path.starts_with("https://") {
621                if options.skip_remote {
622                    collector.push(
623                        CheckLevel::Warn,
624                        format!("Ignoring {path} because skip_remote is set to true"),
625                    );
626                    return Ok(ShapingFile::default());
627                }
628
629                // To facilitate startup ordering races, and listing multiple subscription
630                // host replicas and allowing one or more of them to be temporarily down,
631                // we allow the http request to fail.
632                // We'll log the error message but consider it to be an empty map
633
634                async fn http_get(url: &str, timeout: Duration) -> anyhow::Result<String> {
635                    tokio::time::timeout(timeout, async {
636                        let response = kumo_http_client::builder()
637                            .timeout(timeout)
638                            .connect_timeout(timeout)
639                            .read_timeout(timeout)
640                            .build()?
641                            .get(url)
642                            .send()
643                            .await
644                            .with_context(|| format!("making HTTP request to {url}"))?;
645                        let status = response.status();
646                        if !status.is_success() {
647                            let err_text = match response.text().await {
648                                Ok(text) => text,
649                                Err(err) => {
650                                    format!("Additional error {err:#} while reading response body")
651                                }
652                            };
653                            anyhow::bail!(
654                                "HTTP request to {url} failed: status {status} {reason} {err_text}",
655                                status = status.as_u16(),
656                                reason = status.canonical_reason().unwrap_or("")
657                            );
658                        }
659                        response
660                            .text()
661                            .await
662                            .with_context(|| format!("reading text from {url}"))
663                    })
664                    .await
665                    .with_context(|| format!("timeout making HTTP request to {url}"))?
666                }
667
668                let timeout = options.http_timeout.unwrap_or(Duration::from_secs(5));
669
670                match http_get(path, timeout).await {
671                    Ok(s) => (s, options.remote_load),
672                    Err(err) => {
673                        tracing::error!("{err:#}. Ignoring this shaping source for now");
674                        collector.push(
675                            options.remote_load,
676                            format!("remote shaping source {path} error: {err:#}"),
677                        );
678                        return Ok(ShapingFile::default());
679                    }
680                }
681            } else {
682                match std::fs::read_to_string(path)
683                    .with_context(|| format!("loading data from file {path}"))
684                {
685                    Err(err) => {
686                        collector.push(
687                            options.local_load,
688                            format!("local shaping source {path} error: {err:#}"),
689                        );
690                        return Ok(ShapingFile::default());
691                    }
692                    Ok(s) => (s, options.local_load),
693                }
694            };
695
696        if path.ends_with(".toml") {
697            from_toml(&data).with_context(|| format!("parsing toml from file {path}"))
698        } else if path.ends_with(".json") {
699            from_json(&data).with_context(|| format!("parsing json from file {path}"))
700        } else {
701            // Try parsing both ways and see which wins
702            let mut errors = vec![];
703            match from_toml(&data) {
704                Ok(s) => return Ok(s),
705                Err(err) => errors.push(format!("as toml: {err:#}")),
706            }
707            match from_json(&data) {
708                Ok(s) => return Ok(s),
709                Err(err) => errors.push(format!("as json: {err:#}")),
710            }
711
712            collector.push(level, format!("parsing {path}: {}", errors.join(", ")));
713            Ok(ShapingFile::default())
714        }
715    }
716
717    pub async fn merge_files(
718        files: &[String],
719        options: &ShapingMergeOptions,
720    ) -> anyhow::Result<Self> {
721        use futures_util::stream::FuturesUnordered;
722        use futures_util::StreamExt;
723
724        let mut collector = Collector::default();
725        let mut loaded = vec![];
726        for p in files {
727            loaded.push(Self::load_from_file(p, options, &mut collector).await?);
728        }
729
730        let mut by_site: OrderMap<String, PartialEntry> = OrderMap::new();
731        let mut by_domain: OrderMap<String, PartialEntry> = OrderMap::new();
732        let mut by_provider: OrderMap<String, ProviderEntry> = OrderMap::new();
733        let mut site_aliases: OrderMap<String, Vec<String>> = OrderMap::new();
734
735        // Pre-resolve domains. We don't interleave the resolution with
736        // the work below, because we want to ensure that the ordering
737        // is preserved
738        let mut mx = std::collections::HashMap::new();
739        let mut lookups = FuturesUnordered::new();
740        for item in &loaded {
741            for (domain, partial) in &item.domains {
742                if partial.mx_rollup {
743                    let domain = domain.to_string();
744                    lookups.push(tokio::spawn(async move {
745                        let mx_result = MailExchanger::resolve(&domain).await;
746                        (domain, mx_result)
747                    }));
748                }
749            }
750        }
751
752        while let Some(Ok((domain, result))) = lookups.next().await {
753            mx.insert(domain, result);
754        }
755
756        for mut item in loaded {
757            if let Some(mut partial) = item.default.take() {
758                let domain = "default";
759                partial.domain_name.replace(domain.to_string());
760                reject_provider_only_actions(
761                    &partial.automation,
762                    "the default block",
763                    &mut collector,
764                );
765                match by_domain.get_mut(domain) {
766                    Some(existing) => {
767                        existing.merge_from(partial);
768                    }
769                    None => {
770                        by_domain.insert(domain.to_string(), partial);
771                    }
772                }
773            }
774
775            for (domain, mut partial) in item.domains {
776                partial.domain_name.replace(domain.clone());
777                reject_provider_only_actions(
778                    &partial.automation,
779                    &format!("domain '{domain}'"),
780                    &mut collector,
781                );
782
783                if let Ok(name) = fully_qualify(&domain) {
784                    if name.num_labels() == 1 {
785                        collector.push(
786                            CheckLevel::Warn,
787                            format!(
788                                "Entry for domain '{domain}' consists of a \
789                                 single DNS label. Domain names in TOML sections \
790                                 need to be quoted like '[\"{domain}.com\"]` otherwise \
791                                 the '.' will create a nested table rather than being \
792                                 added to the domain name."
793                            ),
794                        );
795                    }
796                }
797
798                #[cfg(test)]
799                if partial._treat_domain_name_as_site_name {
800                    match by_site.get_mut(&domain) {
801                        Some(existing) => {
802                            existing.merge_from(partial);
803                        }
804                        None => {
805                            by_site.insert(domain.to_string(), partial);
806                        }
807                    }
808                    continue;
809                }
810                if partial.mx_rollup {
811                    let mx = match mx.get(&domain) {
812                        Some(Ok(mx)) => mx,
813                        Some(Err(err)) => {
814                            tracing::debug!(
815                                target: "shaping_load",
816                                %domain,
817                                "dropping domain from shaping: MX resolve failed: {err:#}"
818                            );
819                            collector.push(
820                                options.dns_fail,
821                                format!(
822                                    "error resolving MX for {domain}: {err:#}. \
823                                 Ignoring the shaping config for that domain."
824                                ),
825                            );
826                            continue;
827                        }
828                        None => {
829                            tracing::debug!(
830                                target: "shaping_load",
831                                %domain,
832                                "dropping domain from shaping: MX was not resolved"
833                            );
834                            collector.push(
835                                options.dns_fail,
836                                format!(
837                                "We didn't try to resolve the MX for {domain} for some reason!?. \
838                                 Ignoring the shaping config for that domain."
839                            ),
840                            );
841                            continue;
842                        }
843                    };
844
845                    if mx.site_name.is_empty() {
846                        collector.push(
847                            options.null_mx,
848                            format!(
849                            "domain {domain} has a NULL MX and cannot be used with mx_rollup=true. \
850                             Ignoring the shaping config for that domain."),
851                        );
852                        continue;
853                    }
854
855                    tracing::trace!(
856                        target: "shaping_load",
857                        %domain,
858                        site_name = %mx.site_name,
859                        "domain resolved to site_name"
860                    );
861
862                    site_aliases
863                        .entry(mx.site_name.to_string())
864                        .or_default()
865                        .push(domain.to_string());
866
867                    match by_site.get_mut(&mx.site_name) {
868                        Some(existing) => {
869                            existing.merge_from(partial);
870                        }
871                        None => {
872                            by_site.insert(mx.site_name.clone(), partial);
873                        }
874                    }
875                } else {
876                    match by_domain.get_mut(&domain) {
877                        Some(existing) => {
878                            existing.merge_from(partial);
879                        }
880                        None => {
881                            by_domain.insert(domain, partial);
882                        }
883                    }
884                }
885            }
886
887            for (provider, mut prov) in item.provider {
888                prov.provider_name = provider.to_string();
889                match by_provider.get_mut(&provider) {
890                    Some(existing) => {
891                        existing.merge_from(prov);
892                    }
893                    None => {
894                        by_provider.insert(provider.to_string(), prov);
895                    }
896                }
897            }
898        }
899
900        for (site, partial) in &by_site {
901            partial
902                .clone()
903                .finish()
904                .with_context(|| format!("site: {site}"))?;
905        }
906
907        for (domain, partial) in &by_domain {
908            partial
909                .clone()
910                .finish()
911                .with_context(|| format!("domain: {domain}"))?;
912        }
913
914        for (provider, prov) in &by_provider {
915            prov.finish_params()
916                .with_context(|| format!("provider: {provider}"))?;
917        }
918
919        if options.aliased_site != CheckLevel::Ignore {
920            for (site, aliases) in site_aliases {
921                if aliases.len() > 1 {
922                    collector.push(
923                        options.aliased_site,
924                        format!(
925                            "multiple domain blocks alias to the same site: {site}: {}",
926                            aliases.join(", ")
927                        ),
928                    );
929                }
930            }
931        }
932
933        if options.provider_overlap != CheckLevel::Ignore {
934            for domain in mx.keys() {
935                let mut matching_providers = vec![];
936                for (prov_name, prov) in &by_provider {
937                    if prov.domain_matches(domain).await {
938                        matching_providers.push(prov_name.to_string());
939                    }
940                }
941                if !matching_providers.is_empty() {
942                    collector.push(
943                        options.provider_overlap,
944                        format!(
945                            "domain {domain} is also matched by provider(s): {}",
946                            matching_providers.join(", ")
947                        ),
948                    );
949                }
950            }
951        }
952
953        let mut ctx = Sha256::new();
954        ctx.update("by_site");
955        for (site, entry) in &by_site {
956            ctx.update(site);
957            entry.hash_into(&mut ctx);
958        }
959        ctx.update("by_domain");
960        for (domain, entry) in &by_domain {
961            ctx.update(domain);
962            entry.hash_into(&mut ctx);
963        }
964        ctx.update("by_provider");
965        for (provider, prov) in &by_provider {
966            ctx.update(provider);
967            prov.hash_into(&mut ctx);
968        }
969        // collector.warnings and collector.errors are deliberately left out of
970        // the hash: we want hash() to identify the configuration, not this
971        // particular load of it. Some diagnostic text includes details such as
972        // elapsed query time that vary between otherwise-identical loads.
973        let hash = ctx.finalize();
974        let hash = data_encoding::HEXLOWER.encode(&hash);
975
976        tracing::debug!(
977            target: "shaping_load",
978            %hash,
979            sites = by_site.len(),
980            domains = by_domain.len(),
981            providers = by_provider.len(),
982            warnings = collector.warnings.len(),
983            errors = collector.errors.len(),
984            "merged shaping config"
985        );
986
987        Ok(Self {
988            inner: Arc::new(ShapingInner {
989                by_site,
990                by_domain,
991                by_provider,
992                warnings: collector.warnings,
993                errors: collector.errors,
994                hash,
995            }),
996        })
997    }
998
999    async fn get_egress_path_config(
1000        &self,
1001        domain: &str,
1002        egress_source: &str,
1003        site_name: &str,
1004    ) -> PartialEntry {
1005        self.inner
1006            .get_egress_path_config(domain, egress_source, site_name)
1007            .await
1008    }
1009
1010    pub async fn get_egress_path_config_value(
1011        &self,
1012        domain: &str,
1013        egress_source: &str,
1014        site_name: &str,
1015    ) -> anyhow::Result<serde_json::Value> {
1016        let partial = self
1017            .get_egress_path_config(domain, egress_source, site_name)
1018            .await;
1019        Ok(serde_json::to_value(&partial)?)
1020    }
1021
1022    pub fn get_errors(&self) -> &[String] {
1023        &self.inner.errors
1024    }
1025
1026    pub fn get_warnings(&self) -> &[String] {
1027        &self.inner.warnings
1028    }
1029
1030    pub async fn match_rules(&self, record: &JsonLogRecord) -> anyhow::Result<Vec<MatchedRule>> {
1031        self.inner.match_rules(record).await
1032    }
1033
1034    pub fn get_referenced_sources(&self) -> BTreeMap<String, Vec<String>> {
1035        let mut result = BTreeMap::new();
1036
1037        for (site_name, site) in &self.inner.by_site {
1038            for source_name in site.sources.keys() {
1039                result
1040                    .entry(source_name.to_string())
1041                    .or_insert(vec![])
1042                    .push(format!("site:{site_name}"));
1043            }
1044        }
1045        for (domain_name, domain) in &self.inner.by_domain {
1046            for source_name in domain.sources.keys() {
1047                result
1048                    .entry(source_name.to_string())
1049                    .or_insert(vec![])
1050                    .push(format!("domain:{domain_name}"));
1051            }
1052        }
1053
1054        result
1055    }
1056
1057    pub fn hash(&self) -> String {
1058        self.inner.hash.clone()
1059    }
1060}
1061
1062#[cfg(feature = "lua")]
1063impl LuaUserData for Shaping {
1064    fn add_methods<M: UserDataMethods<Self>>(methods: &mut M) {
1065        mod_memoize::Memoized::impl_memoize(methods);
1066        methods.add_async_method(
1067            "get_egress_path_config",
1068            |lua, this, (domain, egress_source, site_name): (String, String, String)| async move {
1069                let params = this
1070                    .get_egress_path_config(&domain, &egress_source, &site_name)
1071                    .await;
1072                lua.to_value_with(&params.params, serialize_options())
1073            },
1074        );
1075
1076        methods.add_method("get_errors", move |_lua, this, ()| {
1077            let errors: Vec<String> = this.get_errors().iter().map(|s| s.to_string()).collect();
1078            Ok(errors)
1079        });
1080
1081        methods.add_method("get_warnings", move |_lua, this, ()| {
1082            let warnings: Vec<String> = this.get_warnings().iter().map(|s| s.to_string()).collect();
1083            Ok(warnings)
1084        });
1085
1086        methods.add_method("get_referenced_sources", move |_lua, this, ()| {
1087            Ok(this.get_referenced_sources())
1088        });
1089
1090        methods.add_async_method("match_rules", |lua, this, record: mlua::Value| async move {
1091            let record: JsonLogRecord = lua.from_value(record)?;
1092            let rules = this.match_rules(&record).await.map_err(any_err)?;
1093            let mut result = vec![];
1094            for matched in rules {
1095                // Only the rule, in its original shape, crosses into lua. The
1096                // match-time scope stays internal to Rust.
1097                result.push(lua.to_value(&matched.rule)?);
1098            }
1099            Ok(result)
1100        });
1101
1102        methods.add_method("hash", move |_, this, ()| Ok(this.hash()));
1103    }
1104}
1105
1106#[derive(Default, Debug)]
1107pub struct MergedEntry {
1108    pub params: EgressPathConfig,
1109    pub sources: OrderMap<String, EgressPathConfig>,
1110    pub automation: Vec<Rule>,
1111}
1112
1113#[cfg(feature = "lua")]
1114#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1115struct ShapingFile {
1116    pub default: Option<PartialEntry>,
1117    #[serde(flatten, default)]
1118    pub domains: OrderMap<String, PartialEntry>,
1119    #[serde(default)]
1120    pub provider: OrderMap<String, ProviderEntry>,
1121}
1122
1123#[cfg(feature = "lua")]
1124#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1125struct PartialEntry {
1126    #[serde(skip)]
1127    pub domain_name: Option<String>,
1128
1129    #[serde(flatten)]
1130    pub params: toml::Table,
1131
1132    #[serde(default = "default_true")]
1133    pub mx_rollup: bool,
1134
1135    // This is present to facilitate unit testing without requiring
1136    // DNS to resolve the site_name. When set to true, the domain_name
1137    // is considered to be the site_name for this entry.
1138    #[cfg(test)]
1139    #[serde(default)]
1140    pub _treat_domain_name_as_site_name: bool,
1141
1142    #[serde(default)]
1143    pub replace_base: bool,
1144
1145    #[serde(default)]
1146    pub automation: Vec<Rule>,
1147
1148    #[serde(default)]
1149    pub sources: OrderMap<String, toml::Table>,
1150}
1151
1152#[cfg(feature = "lua")]
1153#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1154pub struct ProviderEntry {
1155    #[serde(skip, default)]
1156    pub provider_name: String,
1157
1158    #[serde(default)]
1159    pub provider_connection_limit: Option<LimitSpec>,
1160
1161    #[serde(default)]
1162    pub provider_max_message_rate: Option<ThrottleSpec>,
1163
1164    #[serde(default, rename = "match")]
1165    pub matches: Vec<ProviderMatch>,
1166
1167    #[serde(default)]
1168    pub replace_base: bool,
1169
1170    #[serde(flatten)]
1171    pub params: toml::Table,
1172
1173    #[serde(default)]
1174    pub automation: Vec<Rule>,
1175
1176    #[serde(default)]
1177    pub sources: OrderMap<String, toml::Table>,
1178}
1179
1180#[cfg(feature = "lua")]
1181fn suffix_matches(candidate: &str, suffix: &str) -> bool {
1182    // Remove trailing dot from candidate, as our resolver tends
1183    // to leave the canonical dot on the input host name
1184    let candidate = candidate.strip_suffix(".").unwrap_or(candidate);
1185    // We DON'T handle case mismatches in this code.
1186    // We assume that the dns-resolver crate normalized the
1187    // names to lowercase and that the input suffix is lowercase as well.
1188    candidate.ends_with(suffix)
1189}
1190
1191#[cfg(feature = "lua")]
1192#[cfg(test)]
1193#[test]
1194fn test_suffix_matches() {
1195    assert!(suffix_matches("a", "a"));
1196    assert!(suffix_matches("foo.com", "foo.com"));
1197    assert!(!suffix_matches("foo.com", ".foo.com"));
1198    assert!(!suffix_matches("foo.com", "longer.com"));
1199    assert!(!suffix_matches("réputation.net", ".mx.microsoft"));
1200
1201    // We DON'T handle case mismatches in this code.
1202    // We assume that the dns-resolver crate normalized the
1203    // names to lowercase and that the input suffix is lowercase as well.
1204    assert!(!suffix_matches("foo.Com", ".com"));
1205    assert!(!suffix_matches("foo.Cam", ".com"));
1206}
1207
1208#[cfg(feature = "lua")]
1209fn host_matches(candidate: &str, name: &str) -> bool {
1210    // Remove trailing dot from candidate, as our resolver tends
1211    // to leave the canonical dot on the input host name
1212    let candidate = candidate.strip_suffix(".").unwrap_or(candidate);
1213    candidate == name
1214}
1215
1216#[cfg(feature = "lua")]
1217#[cfg(test)]
1218#[test]
1219fn test_host_matches() {
1220    assert!(host_matches("foo.com", "foo.com"));
1221    assert!(host_matches("foo.com.", "foo.com"));
1222    assert!(!host_matches("foo.com", "notfoo.com"));
1223}
1224
1225#[cfg(feature = "lua")]
1226impl ProviderEntry {
1227    async fn domain_matches(&self, domain: &str) -> bool {
1228        // We'd like to avoid doing DNS if we can do a simple suffix match,
1229        // so we bias to looking at those first
1230        let mut need_mx = false;
1231
1232        tracing::trace!(
1233            "ProviderEntry::domain_matches({domain}) vs {:?}",
1234            self.matches
1235        );
1236
1237        for rule in &self.matches {
1238            match rule {
1239                ProviderMatch::DomainSuffix(suffix) => {
1240                    if suffix_matches(domain, suffix) {
1241                        tracing::trace!("{domain} suffix matches {suffix}");
1242                        return true;
1243                    }
1244                }
1245                ProviderMatch::HostName(_) | ProviderMatch::MXSuffix(_) => {
1246                    need_mx = true;
1247                }
1248            }
1249        }
1250
1251        if !need_mx {
1252            return false;
1253        }
1254
1255        // Now we can consider DNS
1256        match MailExchanger::resolve(domain).await {
1257            Err(err) => {
1258                // Didn't resolve; could be legit, for example, could
1259                // be an internal or fake name that is handled via
1260                // smart host or custom routing, so we log what
1261                // happened as a trace rather than polluting the
1262                // logs about it
1263                tracing::trace!(
1264                    "Error resolving MX for {domain}: {err:#}. \
1265                    Provider {} match rules will be ignored",
1266                    self.provider_name
1267                );
1268                false
1269            }
1270            Ok(mx) => {
1271                tracing::trace!("Consider MXSuffix rules");
1272                for host in &mx.hosts {
1273                    let mut matched = false;
1274
1275                    for rule in &self.matches {
1276                        match rule {
1277                            ProviderMatch::MXSuffix(suffix) => {
1278                                // For a given MX suffix rule, all hosts must match
1279                                // it for it to be valid. This is so that we don't
1280                                // falsely lump a vanity domain that blends providers
1281                                // together.
1282                                tracing::trace!("suffix={suffix} vs host {host}");
1283                                if suffix_matches(host, suffix) {
1284                                    matched = true;
1285                                    break;
1286                                }
1287                            }
1288                            ProviderMatch::HostName(name) => {
1289                                if host_matches(host, name) {
1290                                    matched = true;
1291                                    break;
1292                                }
1293                            }
1294                            ProviderMatch::DomainSuffix(_) => {}
1295                        }
1296                    }
1297
1298                    if !matched {
1299                        tracing::trace!("host didn't match any of these rules");
1300                        return false;
1301                    }
1302                }
1303
1304                true
1305            }
1306        }
1307    }
1308
1309    fn merge_from(&mut self, mut other: Self) {
1310        if other.replace_base {
1311            self.provider_connection_limit = other.provider_connection_limit;
1312            self.matches = other.matches;
1313            self.params = other.params;
1314            self.sources = other.sources;
1315            self.automation = other.automation;
1316        } else {
1317            if other.provider_connection_limit.is_some() {
1318                self.provider_connection_limit = other.provider_connection_limit;
1319            }
1320
1321            toml_table_merge_from(&mut self.params, &other.params);
1322
1323            for (source, tbl) in other.sources {
1324                match self.sources.get_mut(&source) {
1325                    Some(existing) => {
1326                        toml_table_merge_from(existing, &tbl);
1327                    }
1328                    None => {
1329                        self.sources.insert(source, tbl);
1330                    }
1331                }
1332            }
1333
1334            self.matches.append(&mut other.matches);
1335            self.automation.append(&mut other.automation);
1336        }
1337    }
1338
1339    fn apply_provider_params_to(&self, source: &str, target: &mut toml::Table) {
1340        let mut implied = toml::Table::new();
1341        implied.insert(
1342            "provider_name".to_string(),
1343            toml::Value::String(self.provider_name.to_string()),
1344        );
1345
1346        if let Some(limit) = &self.provider_connection_limit {
1347            let mut limits = toml::Table::new();
1348            limits.insert(
1349                format!("shaping-provider-{}-{source}-limit", self.provider_name),
1350                toml::Value::String(limit.to_string()),
1351            );
1352            implied.insert(
1353                "additional_connection_limits".to_string(),
1354                toml::Value::Table(limits),
1355            );
1356        }
1357        if let Some(rate) = &self.provider_max_message_rate {
1358            let rate = rate.as_string();
1359            let mut limits = toml::Table::new();
1360            limits.insert(
1361                format!("shaping-provider-{}-{source}-rate", self.provider_name),
1362                rate.into(),
1363            );
1364            implied.insert(
1365                "additional_message_rate_throttles".to_string(),
1366                toml::Value::Table(limits),
1367            );
1368        }
1369
1370        if let Some(rate) = target.remove("provider_source_selection_rate") {
1371            let mut limits = toml::Table::new();
1372            limits.insert(
1373                format!(
1374                    "shaping-provider-{}-{source}-selection-rate",
1375                    self.provider_name
1376                ),
1377                rate,
1378            );
1379            implied.insert(
1380                "additional_source_selection_rates".to_string(),
1381                toml::Value::Table(limits),
1382            );
1383        }
1384
1385        toml_table_merge_from(target, &implied);
1386    }
1387
1388    fn finish_params(&self) -> anyhow::Result<MergedEntry> {
1389        let provider_name = &self.provider_name;
1390
1391        let params = EgressPathConfig::deserialize(self.params.clone()).with_context(|| {
1392            format!(
1393                "interpreting provider '{provider_name}' params {:#?} as EgressPathConfig",
1394                self.params
1395            )
1396        })?;
1397        let mut sources = OrderMap::new();
1398
1399        for (source, params) in &self.sources {
1400            let mut params = params.clone();
1401            // I don't really like this remove call. The issue is that we don't
1402            // have an alternative way to filter this out of the partial source
1403            // definition, and this provider_ option is not valid in the
1404            // EgressPathConfig struct itself; it is a shaping source-specific
1405            // addition to help with setting up shared throttles across providers
1406            params.remove("provider_source_selection_rate");
1407            sources.insert(
1408                source.clone(),
1409                EgressPathConfig::deserialize(params.clone()).with_context(|| {
1410                    format!("interpreting provider '{provider_name}' source '{source}' {params:#} as EgressPathConfig")
1411                })?,
1412            );
1413        }
1414
1415        Ok(MergedEntry {
1416            params,
1417            sources,
1418            automation: self.automation.clone(),
1419        })
1420    }
1421
1422    fn hash_into(&self, ctx: &mut Sha256) {
1423        ctx.update(&self.provider_name);
1424        ctx.update(serde_json::to_string(self).unwrap_or_else(|_| String::new()));
1425    }
1426}
1427
1428#[cfg(feature = "lua")]
1429#[derive(Deserialize, Serialize, Debug, Clone)]
1430pub enum ProviderMatch {
1431    MXSuffix(String),
1432    DomainSuffix(String),
1433    HostName(String),
1434}
1435
1436#[cfg(feature = "lua")]
1437fn toml_table_merge_from(tbl: &mut toml::Table, source: &toml::Table) {
1438    // Limit merging to just the throttle related fields, as their purpose
1439    // is for creating broader scoped limits that cut across normal boundaries
1440    fn is_mergeable(name: &str) -> bool {
1441        match name {
1442            "additional_connection_limits"
1443            | "additional_message_rate_throttles"
1444            | "additional_source_selection_rates" => true,
1445            _ => false,
1446        }
1447    }
1448
1449    for (k, v) in source {
1450        match (tbl.get_mut(k), v.as_table()) {
1451            // Merge Table values together, rather than simply replacing them.
1452            (Some(toml::Value::Table(existing)), Some(v)) if is_mergeable(k) => {
1453                for (inner_k, inner_v) in v {
1454                    existing.insert(inner_k.clone(), inner_v.clone());
1455                }
1456            }
1457            _ => {
1458                tbl.insert(k.clone(), v.clone());
1459            }
1460        }
1461    }
1462}
1463
1464#[cfg(feature = "lua")]
1465impl PartialEntry {
1466    fn merge_from(&mut self, mut other: Self) {
1467        if other.replace_base {
1468            self.params = other.params;
1469            self.automation = other.automation;
1470            self.sources = other.sources;
1471        } else {
1472            toml_table_merge_from(&mut self.params, &other.params);
1473
1474            for (source, tbl) in other.sources {
1475                match self.sources.get_mut(&source) {
1476                    Some(existing) => {
1477                        toml_table_merge_from(existing, &tbl);
1478                    }
1479                    None => {
1480                        self.sources.insert(source, tbl);
1481                    }
1482                }
1483            }
1484
1485            self.automation.append(&mut other.automation);
1486        }
1487    }
1488
1489    fn finish(self) -> anyhow::Result<MergedEntry> {
1490        let domain = self.domain_name.unwrap_or_default();
1491
1492        let params = EgressPathConfig::deserialize(self.params.clone()).with_context(|| {
1493            format!(
1494                "interpreting domain '{domain}' params {:#?} as EgressPathConfig",
1495                self.params
1496            )
1497        })?;
1498        let mut sources = OrderMap::new();
1499
1500        for (source, params) in self.sources {
1501            sources.insert(
1502                source.clone(),
1503                EgressPathConfig::deserialize(params.clone()).with_context(|| {
1504                    format!("interpreting domain '{domain}' source '{source}' {params:#} as EgressPathConfig")
1505                })?,
1506            );
1507        }
1508
1509        Ok(MergedEntry {
1510            params,
1511            sources,
1512            automation: self.automation,
1513        })
1514    }
1515
1516    fn hash_into(&self, ctx: &mut Sha256) {
1517        if let Some(name) = self.domain_name.as_ref() {
1518            ctx.update(name)
1519        }
1520        ctx.update(serde_json::to_string(self).unwrap_or_else(|_| String::new()));
1521    }
1522}
1523
1524fn one_or_many<'de, T, D>(deserializer: D, expecting: &str) -> Result<Vec<T>, D::Error>
1525where
1526    T: Deserialize<'de>,
1527    D: Deserializer<'de>,
1528{
1529    let result: Result<Vec<T>, _> =
1530        OneOrMany::<serde_with::Same, PreferOne>::deserialize_as(deserializer);
1531    match result {
1532        Ok(r) => Ok(r),
1533        Err(err) => Err(serde::de::Error::custom(format!(
1534            "{expecting}.\nThe underlying error message is:\n{err:#}"
1535        ))),
1536    }
1537}
1538
1539fn one_or_many_action<'de, D>(deserializer: D) -> Result<Vec<Action>, D::Error>
1540where
1541    D: Deserializer<'de>,
1542{
1543    one_or_many(
1544        deserializer,
1545        "\"action\" field expected either a single Action or an array of Actions",
1546    )
1547}
1548
1549fn regex_string_or_array<'de, D>(deserializer: D) -> Result<Vec<Regex>, D::Error>
1550where
1551    D: Deserializer<'de>,
1552{
1553    string_or_array(
1554        deserializer,
1555        "regex string or array of regex strings for field regex",
1556    )
1557}
1558
1559fn string_or_array<'de, T, D>(deserializer: D, expecting: &'static str) -> Result<Vec<T>, D::Error>
1560where
1561    T: Deserialize<'de> + TryFrom<String>,
1562    <T as TryFrom<String>>::Error: std::fmt::Debug,
1563    D: Deserializer<'de>,
1564{
1565    // This is a Visitor that forwards string types to T's `TryFrom<String>` impl and
1566    // forwards map types to T's `Deserialize` impl. The `PhantomData` is to
1567    // keep the compiler from complaining about T being an unused generic type
1568    // parameter. We need T in order to know the Value type for the Visitor
1569    // impl.
1570    struct StringOrArray<T>(PhantomData<fn() -> T>, &'static str);
1571
1572    impl<'de, T> Visitor<'de> for StringOrArray<T>
1573    where
1574        T: Deserialize<'de> + TryFrom<String>,
1575        <T as TryFrom<String>>::Error: std::fmt::Debug,
1576    {
1577        type Value = Vec<T>;
1578
1579        fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
1580            formatter.write_str(self.1)
1581        }
1582
1583        fn visit_str<E>(self, value: &str) -> Result<Vec<T>, E>
1584        where
1585            E: serde::de::Error,
1586        {
1587            Ok(vec![
1588                T::try_from(value.to_string()).map_err(|e| E::custom(format!("{e:?}")))?
1589            ])
1590        }
1591
1592        fn visit_seq<S>(self, seq: S) -> Result<Vec<T>, S::Error>
1593        where
1594            S: SeqAccess<'de>,
1595        {
1596            Deserialize::deserialize(serde::de::value::SeqAccessDeserializer::new(seq))
1597        }
1598    }
1599
1600    deserializer.deserialize_any(StringOrArray(PhantomData, expecting))
1601}
1602
1603#[cfg(feature = "lua")]
1604fn default_true() -> bool {
1605    true
1606}
1607
1608#[cfg(feature = "lua")]
1609pub fn register(lua: &mlua::Lua) -> anyhow::Result<()> {
1610    let shaping_mod = config::get_or_create_sub_module(lua, "shaping")?;
1611
1612    shaping_mod.set(
1613        "load",
1614        lua.create_async_function(
1615            move |lua, (paths, options): (Vec<String>, Option<mlua::Value>)| async move {
1616                let options = match options {
1617                    Some(v) => lua.from_value(v)?,
1618                    None => Default::default(),
1619                };
1620                let shaping = Shaping::merge_files(&paths, &options)
1621                    .await
1622                    .map_err(any_err)?;
1623                Ok(shaping)
1624            },
1625        )?,
1626    )?;
1627
1628    Ok(())
1629}
1630
1631#[cfg(test)]
1632mod test {
1633    use super::*;
1634    use kumo_log_types::RecordType;
1635    use rfc5321::Response;
1636    use std::io::Write;
1637    use tempfile::NamedTempFile;
1638    use uuid::Uuid;
1639
1640    async fn make_shaping_configs(inputs: &[&str]) -> Shaping {
1641        let mut files = vec![];
1642        let mut file_names = vec![];
1643
1644        for (i, content) in inputs.iter().enumerate() {
1645            let mut shaping_file = NamedTempFile::with_prefix(format!("file{i}")).unwrap();
1646            shaping_file.write_all(content.as_bytes()).unwrap();
1647            file_names.push(shaping_file.path().to_str().unwrap().to_string());
1648            files.push(shaping_file);
1649        }
1650
1651        Shaping::merge_files(&file_names, &ShapingMergeOptions::default())
1652            .await
1653            .unwrap()
1654    }
1655
1656    #[tokio::test]
1657    async fn hash_excludes_diagnostics() {
1658        let content = r#"
1659["example.com"]
1660mx_rollup = false
1661connection_limit = 5
1662"#;
1663        let mut file = NamedTempFile::with_prefix("shaping").unwrap();
1664        file.write_all(content.as_bytes()).unwrap();
1665        let path = file.path().to_str().unwrap().to_string();
1666
1667        // skip_remote records a warning for a remote source without fetching
1668        // it, giving a warning whose presence is independent of the merged
1669        // content. This isolates the warning from the by_site/by_domain/
1670        // by_provider data the hash is computed over.
1671        let options = ShapingMergeOptions {
1672            skip_remote: true,
1673            ..ShapingMergeOptions::default()
1674        };
1675
1676        let without_warning = Shaping::merge_files(std::slice::from_ref(&path), &options)
1677            .await
1678            .unwrap();
1679        let with_warning = Shaping::merge_files(
1680            &[
1681                path.clone(),
1682                "https://example.invalid/shaping.toml".to_string(),
1683            ],
1684            &options,
1685        )
1686        .await
1687        .unwrap();
1688
1689        assert!(
1690            without_warning.get_warnings().is_empty(),
1691            "the content-only load should record no warnings"
1692        );
1693        assert!(
1694            !with_warning.get_warnings().is_empty(),
1695            "the skipped remote source should record a warning"
1696        );
1697        // Identical merged content with and without a warning must hash the
1698        // same: the hash covers configuration only.
1699        k9::assert_equal!(without_warning.hash(), with_warning.hash());
1700
1701        // A real content change must still change the hash.
1702        let changed = content.replace("connection_limit = 5", "connection_limit = 6");
1703        let mut changed_file = NamedTempFile::with_prefix("shaping").unwrap();
1704        changed_file.write_all(changed.as_bytes()).unwrap();
1705        let changed_path = changed_file.path().to_str().unwrap().to_string();
1706        let changed = Shaping::merge_files(&[changed_path], &options)
1707            .await
1708            .unwrap();
1709        assert_ne!(without_warning.hash(), changed.hash());
1710    }
1711
1712    #[tokio::test]
1713    async fn test_merge_additional() {
1714        let shaping = make_shaping_configs(&[
1715            r#"
1716["example.com"]
1717mx_rollup = false
1718additional_connection_limits = {"first"=10}
1719        "#,
1720            r#"
1721["example.com"]
1722mx_rollup = false
1723additional_connection_limits = {"second"=32}
1724additional_message_rate_throttles = {"second"="100/hr"}
1725        "#,
1726        ])
1727        .await;
1728
1729        let resolved = shaping
1730            .get_egress_path_config("example.com", "invalid.source", "invalid.site")
1731            .await
1732            .finish()
1733            .unwrap();
1734
1735        k9::snapshot!(
1736            resolved.params.additional_connection_limits,
1737            r#"
1738{
1739    "first": 10,
1740    "second": 32,
1741}
1742"#
1743        );
1744        k9::snapshot!(
1745            resolved.params.additional_message_rate_throttles,
1746            r#"
1747{
1748    "second": 100/h,
1749}
1750"#
1751        );
1752    }
1753
1754    #[tokio::test]
1755    async fn test_provider_multi_hostname() {
1756        let shaping = make_shaping_configs(&[r#"
1757[provider."yahoo"]
1758match=[{HostName="mta5.am0.yahoodns.net"},{HostName="mta6.am0.yahoodns.net"},{HostName="mta7.am0.yahoodns.net"}]
1759enable_tls = "Required"
1760        "#])
1761        .await;
1762
1763        let resolved = shaping
1764            .get_egress_path_config("yahoo.com", "invalid.source", "invalid.site")
1765            .await
1766            .finish()
1767            .unwrap();
1768
1769        k9::assert_equal!(
1770            resolved.params.enable_tls,
1771            crate::egress_path::Tls::Required
1772        );
1773        k9::assert_equal!(resolved.params.provider_name.unwrap(), "yahoo");
1774    }
1775
1776    #[tokio::test]
1777    async fn test_provider_multi_suffix() {
1778        let shaping = make_shaping_configs(&[r#"
1779[provider."yahoo"]
1780match=[{MXSuffix="mta5.am0.yahoodns.net"},{MXSuffix="mta6.am0.yahoodns.net"},{MXSuffix="mta7.am0.yahoodns.net"}]
1781enable_tls = "Required"
1782        "#])
1783        .await;
1784
1785        let resolved = shaping
1786            .get_egress_path_config("yahoo.com", "invalid.source", "invalid.site")
1787            .await
1788            .finish()
1789            .unwrap();
1790
1791        k9::assert_equal!(
1792            resolved.params.enable_tls,
1793            crate::egress_path::Tls::Required
1794        );
1795        k9::assert_equal!(resolved.params.provider_name.unwrap(), "yahoo");
1796    }
1797
1798    #[tokio::test]
1799    async fn test_set_provider_config_rejected_outside_provider() {
1800        // SetProviderConfig attaches its override to the matched provider.
1801        let shaping = make_shaping_configs(&[r#"
1802["example.com"]
1803mx_rollup = false
1804
1805[["example.com".automation]]
1806regex = "x"
1807action = {SetProviderConfig={name="connection_limit", value=3}}
1808duration = "1hr"
1809"#])
1810        .await;
1811
1812        k9::assert_equal!(
1813            shaping.get_errors(),
1814            ["SetProviderConfig is only valid within a provider block, but was used in domain 'example.com'".to_string()]
1815                .as_slice()
1816        );
1817    }
1818
1819    #[tokio::test]
1820    async fn test_set_provider_config_accepted_in_provider() {
1821        let shaping = make_shaping_configs(&[r#"
1822[provider."acme"]
1823match=[{DomainSuffix=".acme"}]
1824
1825[[provider."acme".automation]]
1826regex = "x"
1827action = {SetProviderConfig={name="connection_limit", value=3}}
1828duration = "1hr"
1829"#])
1830        .await;
1831
1832        k9::assert_equal!(shaping.get_errors(), ([] as [String; 0]).as_slice());
1833    }
1834
1835    #[tokio::test]
1836    async fn test_provider() {
1837        let shaping = make_shaping_configs(&[r#"
1838[provider."Office 365"]
1839match=[{MXSuffix=".olc.protection.outlook.com"},{DomainSuffix=".outlook.com"}]
1840enable_tls = "Required"
1841provider_connection_limit = 10
1842provider_max_message_rate = "120/s"
1843
1844[provider."Office 365".sources."new-source"]
1845provider_source_selection_rate = "500/d,max_burst=1"
1846        "#])
1847        .await;
1848
1849        let resolved = shaping
1850            .get_egress_path_config("outlook.com", "invalid.source", "invalid.site")
1851            .await
1852            .finish()
1853            .unwrap();
1854
1855        k9::assert_equal!(
1856            resolved.params.enable_tls,
1857            crate::egress_path::Tls::Required
1858        );
1859        k9::assert_equal!(resolved.params.provider_name.unwrap(), "Office 365");
1860
1861        k9::snapshot!(
1862            resolved.params.additional_connection_limits,
1863            r#"
1864{
1865    "shaping-provider-Office 365-invalid.source-limit": 10,
1866}
1867"#
1868        );
1869        k9::snapshot!(
1870            resolved.params.additional_message_rate_throttles,
1871            r#"
1872{
1873    "shaping-provider-Office 365-invalid.source-rate": 120/s,
1874}
1875"#
1876        );
1877        assert!(resolved.params.source_selection_rate.is_none());
1878        assert!(resolved.params.additional_source_selection_rates.is_empty());
1879
1880        let resolved = shaping
1881            .get_egress_path_config("outlook.com", "new-source", "invalid.site")
1882            .await
1883            .finish()
1884            .unwrap();
1885        assert!(resolved.params.source_selection_rate.is_none());
1886        k9::snapshot!(
1887            resolved.params.additional_source_selection_rates,
1888            r#"
1889{
1890    "shaping-provider-Office 365-new-source-selection-rate": 500/d,max_burst=1,
1891}
1892"#
1893        );
1894    }
1895
1896    #[tokio::test]
1897    async fn test_rule_matching() {
1898        let shaping = make_shaping_configs(&[r#"
1899[["default".automation]]
1900regex="default"
1901action = {SetConfig={name="connection_limit", value=1}}
1902duration = "1hr"
1903
1904["fake.site"]
1905_treat_domain_name_as_site_name = true
1906
1907[["fake.site".automation]]
1908regex="fake_rollup"
1909action = {SetConfig={name="connection_limit", value=2}}
1910duration = "1hr"
1911
1912["((a|b).x|c.y).targets.test"]
1913_treat_domain_name_as_site_name = true
1914connection_limit = 11
1915
1916[["((a|b).x|c.y).targets.test".automation]]
1917regex="nested_rollup"
1918action = {SetConfig={name="connection_limit", value=3}}
1919duration = "1hr"
1920
1921["((a|c).x|b.y).targets.test"]
1922_treat_domain_name_as_site_name = true
1923connection_limit = 22
1924
1925["woot.provider"]
1926mx_rollup = false
1927
1928[["woot.provider".automation]]
1929regex="woot_domain"
1930action = {SetConfig={name="connection_limit", value=2}}
1931duration = "1hr"
1932
1933[provider."provider"]
1934match=[{DomainSuffix=".provider"}]
1935
1936[[provider."provider".automation]]
1937regex="provider"
1938action = {SetConfig={name="connection_limit", value=3}}
1939duration = "1hr"
1940match_internal = true
1941
1942"#])
1943        .await;
1944
1945        eprintln!("{:?}", shaping.inner.warnings);
1946
1947        fn make_record(content: &str, recipient: &str, site: &str) -> JsonLogRecord {
1948            JsonLogRecord {
1949                kind: RecordType::TransientFailure,
1950                id: String::new(),
1951                sender: String::new(),
1952                recipient: vec![recipient.to_string()],
1953                queue: String::new(),
1954                site: site.to_string(),
1955                size: 0,
1956                response: Response {
1957                    code: 400,
1958                    command: None,
1959                    enhanced_code: None,
1960                    content: content.to_string(),
1961                },
1962                peer_address: None,
1963                timestamp: Default::default(),
1964                created: Default::default(),
1965                num_attempts: 1,
1966                bounce_classification: Default::default(),
1967                egress_pool: None,
1968                egress_source: None,
1969                source_address: None,
1970                feedback_report: None,
1971                meta: Default::default(),
1972                headers: Default::default(),
1973                delivery_protocol: None,
1974                reception_protocol: None,
1975                nodeid: Uuid::default(),
1976                tls_cipher: None,
1977                tls_protocol_version: None,
1978                tls_peer_subject_name: None,
1979                provider_name: None,
1980                session_id: None,
1981            }
1982        }
1983
1984        let matches = shaping
1985            .match_rules(&make_record("default", "user@example.com", "dummy_site"))
1986            .await
1987            .unwrap();
1988        k9::assert_equal!(
1989            matches[0].rule.regex[0].to_string(),
1990            "default",
1991            "matches against default automation rule"
1992        );
1993
1994        let matches = shaping
1995            .match_rules(&make_record(
1996                "KumoMTA internal: default",
1997                "user@example.com",
1998                "dummy_site",
1999            ))
2000            .await
2001            .unwrap();
2002        assert!(matches.is_empty(), "internal bounce should not match");
2003
2004        let matches = shaping
2005            .match_rules(&make_record(
2006                "woot_domain",
2007                "user@woot.provider",
2008                "dummy_site",
2009            ))
2010            .await
2011            .unwrap();
2012        k9::assert_equal!(
2013            matches[0].rule.regex[0].to_string(),
2014            "woot_domain",
2015            "matches against domain rule with mx_rollup=false"
2016        );
2017
2018        let matches = shaping
2019            .match_rules(&make_record("fake_rollup", "user@fake.rollup", "fake.site"))
2020            .await
2021            .unwrap();
2022        k9::assert_equal!(
2023            matches[0].rule.regex[0].to_string(),
2024            "fake_rollup",
2025            "matches against domain rule with mx_rollup=true"
2026        );
2027
2028        let nested_a = "((a|b).x|c.y).targets.test";
2029        let nested_b = "((a|c).x|b.y).targets.test";
2030        for (site, expected) in [(nested_a, 11), (nested_b, 22)] {
2031            let config = shaping
2032                .get_egress_path_config("example.test", "unspecified", site)
2033                .await
2034                .finish()
2035                .unwrap();
2036            assert_eq!(config.params.connection_limit.limit, expected);
2037        }
2038        let matches = shaping
2039            .match_rules(&make_record(
2040                "nested_rollup",
2041                "user@example.test",
2042                &format!("unspecified->{nested_a}@smtp_client"),
2043            ))
2044            .await
2045            .unwrap();
2046        assert_eq!(matches.len(), 1);
2047        assert_eq!(matches[0].rule.regex[0].to_string(), "nested_rollup");
2048        assert!(shaping
2049            .match_rules(&make_record(
2050                "nested_rollup",
2051                "user@example.test",
2052                &format!("unspecified->{nested_b}@smtp_client")
2053            ))
2054            .await
2055            .unwrap()
2056            .is_empty());
2057
2058        let matches = shaping
2059            .match_rules(&make_record("provider", "user@woot.provider", "dummy_site"))
2060            .await
2061            .unwrap();
2062        k9::assert_equal!(
2063            matches[0].rule.regex[0].to_string(),
2064            "provider",
2065            "matches against provider rule"
2066        );
2067        k9::assert_equal!(
2068            matches[0].scope,
2069            MatchScope::Provider("provider".to_string()),
2070            "provider rule's scope names the matched provider"
2071        );
2072
2073        let matches = shaping
2074            .match_rules(&make_record(
2075                "KumoMTA internal: provider",
2076                "user@woot.provider",
2077                "dummy_site",
2078            ))
2079            .await
2080            .unwrap();
2081        k9::assert_equal!(
2082            matches[0].rule.regex[0].to_string(),
2083            "provider",
2084            "internal response matches against provider rule"
2085        );
2086    }
2087
2088    #[tokio::test]
2089    async fn test_defaults() {
2090        let shaping = make_shaping_configs(&[r#"
2091["default"]
2092connection_limit = 10
2093max_connection_rate = "100/min"
2094max_deliveries_per_connection = 100
2095max_message_rate = "100/s"
2096idle_timeout = "60s"
2097data_timeout = "30s"
2098data_dot_timeout = "60s"
2099enable_tls = "Opportunistic"
2100consecutive_connection_failures_before_delay = 100
2101
2102[["default".automation]]
2103regex=[
2104        '/Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/',
2105        '/All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/',
2106        '/has been temporarily rate limited due to IP reputation/',
2107        '/Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/',
2108        '/Server busy\. Please try again later from/'
2109]
2110action = [
2111        {SetConfig={name="max_message_rate", value="1/minute"}},
2112        {SetConfig={name="connection_limit", value=1}}
2113]
2114duration = "90m"
2115
2116["gmail.com"]
2117max_deliveries_per_connection = 50
2118connection_limit = 5
2119enable_tls = "Required"
2120consecutive_connection_failures_before_delay = 5
2121
2122["yahoo.com"]
2123max_deliveries_per_connection = 20
2124
2125[["yahoo.com".automation]]
2126regex = "\\[TS04\\]"
2127action = "Suspend"
2128duration = "2 hours"
2129
2130["comcast.net"]
2131connection_limit = 25
2132max_deliveries_per_connection = 250
2133enable_tls = "Required"
2134idle_timeout = "30s"
2135consecutive_connection_failures_before_delay = 24
2136
2137["mail.com"]
2138max_deliveries_per_connection = 100
2139
2140["orange.fr"]
2141connection_limit = 3
2142
2143["smtp.mailgun.com"]
2144connection_limit = 7000
2145max_deliveries_per_connection = 3
2146
2147["example.com"]
2148mx_rollup = false
2149max_deliveries_per_connection = 100
2150connection_limit = 3
2151
2152["example.com".sources."my source name"]
2153connection_limit = 5
2154        "#])
2155        .await;
2156
2157        let default = shaping
2158            .get_egress_path_config("invalid.domain", "invalid.source", "invalid.site")
2159            .await
2160            .finish()
2161            .unwrap();
2162        k9::snapshot!(
2163            default,
2164            r#"
2165MergedEntry {
2166    params: EgressPathConfig {
2167        connection_limit: 10,
2168        additional_connection_limits: {},
2169        enable_tls: Opportunistic,
2170        tls_mode: StartTls,
2171        enable_mta_sts: true,
2172        enable_dane: false,
2173        enable_pipelining: true,
2174        enable_rset: true,
2175        tls_prefer_openssl: false,
2176        tls_certificate: None,
2177        tls_private_key: None,
2178        openssl_cipher_list: None,
2179        openssl_cipher_suites: None,
2180        openssl_options: None,
2181        rustls_cipher_suites: [],
2182        client_timeouts: SmtpClientTimeouts {
2183            connect_timeout: 60s,
2184            banner_timeout: 60s,
2185            ehlo_timeout: 300s,
2186            mail_from_timeout: 300s,
2187            rcpt_to_timeout: 300s,
2188            data_timeout: 30s,
2189            data_dot_timeout: 60s,
2190            rset_timeout: 5s,
2191            idle_timeout: 60s,
2192            starttls_timeout: 5s,
2193            auth_timeout: 60s,
2194        },
2195        system_shutdown_timeout: None,
2196        max_ready: 1024,
2197        consecutive_connection_failures_before_delay: 100,
2198        smtp_port: 25,
2199        smtp_auth_plain_username: None,
2200        smtp_auth_plain_password: None,
2201        allow_smtp_auth_plain_without_tls: false,
2202        allow_smtp_auth_plain_without_valid_certificate: false,
2203        max_message_rate: Some(
2204            100/s,
2205        ),
2206        additional_message_rate_throttles: {},
2207        source_selection_rate: None,
2208        additional_source_selection_rates: {},
2209        max_connection_rate: Some(
2210            100/m,
2211        ),
2212        max_deliveries_per_connection: 100,
2213        max_recipients_per_batch: 100,
2214        prohibited_hosts: {
2215            "0.0.0.0",
2216            "127.0.0.0/8",
2217            "::/127",
2218        },
2219        skip_hosts: {},
2220        ip_lookup_strategy: Ipv4AndIpv6,
2221        max_mx_plan_size: 50,
2222        max_mx_addresses_per_host: 10,
2223        ehlo_domain: None,
2224        aggressive_connection_opening: false,
2225        refresh_interval: 60s,
2226        refresh_strategy: Ttl,
2227        dispatcher_wakeup_strategy: Aggressive,
2228        maintainer_wakeup_strategy: Aggressive,
2229        provider_name: None,
2230        remember_broken_tls: None,
2231        opportunistic_tls_reconnect_on_failed_handshake: false,
2232        use_lmtp: false,
2233        reconnect_strategy: ConnectNextHost,
2234        readyq_pool_name: None,
2235        low_memory_reduction_policy: ShrinkDataAndMeta,
2236        no_memory_reduction_policy: ShrinkDataAndMeta,
2237        try_next_host_on_transport_error: false,
2238        ignore_8bit_checks: false,
2239        dispatcher_progress_watchdog_timeout: None,
2240    },
2241    sources: {},
2242    automation: [
2243        Rule {
2244            regex: [
2245                Regex(
2246                    /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2247                ),
2248                Regex(
2249                    /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2250                ),
2251                Regex(
2252                    /has been temporarily rate limited due to IP reputation/,
2253                ),
2254                Regex(
2255                    /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2256                ),
2257                Regex(
2258                    /Server busy\. Please try again later from/,
2259                ),
2260            ],
2261            action: [
2262                SetConfig(
2263                    EgressPathConfigValue {
2264                        name: "max_message_rate",
2265                        value: HashableTomlValue {
2266                            value: String(
2267                                "1/minute",
2268                            ),
2269                        },
2270                    },
2271                ),
2272                SetConfig(
2273                    EgressPathConfigValue {
2274                        name: "connection_limit",
2275                        value: HashableTomlValue {
2276                            value: Integer(
2277                                1,
2278                            ),
2279                        },
2280                    },
2281                ),
2282            ],
2283            trigger: Immediate,
2284            duration: 5400s,
2285            match_internal: false,
2286        },
2287    ],
2288}
2289"#
2290        );
2291
2292        let example_com = shaping
2293            .get_egress_path_config("example.com", "invalid.source", "invalid.site")
2294            .await
2295            .finish()
2296            .unwrap();
2297        k9::snapshot!(
2298            example_com,
2299            r#"
2300MergedEntry {
2301    params: EgressPathConfig {
2302        connection_limit: 3,
2303        additional_connection_limits: {},
2304        enable_tls: Opportunistic,
2305        tls_mode: StartTls,
2306        enable_mta_sts: true,
2307        enable_dane: false,
2308        enable_pipelining: true,
2309        enable_rset: true,
2310        tls_prefer_openssl: false,
2311        tls_certificate: None,
2312        tls_private_key: None,
2313        openssl_cipher_list: None,
2314        openssl_cipher_suites: None,
2315        openssl_options: None,
2316        rustls_cipher_suites: [],
2317        client_timeouts: SmtpClientTimeouts {
2318            connect_timeout: 60s,
2319            banner_timeout: 60s,
2320            ehlo_timeout: 300s,
2321            mail_from_timeout: 300s,
2322            rcpt_to_timeout: 300s,
2323            data_timeout: 30s,
2324            data_dot_timeout: 60s,
2325            rset_timeout: 5s,
2326            idle_timeout: 60s,
2327            starttls_timeout: 5s,
2328            auth_timeout: 60s,
2329        },
2330        system_shutdown_timeout: None,
2331        max_ready: 1024,
2332        consecutive_connection_failures_before_delay: 100,
2333        smtp_port: 25,
2334        smtp_auth_plain_username: None,
2335        smtp_auth_plain_password: None,
2336        allow_smtp_auth_plain_without_tls: false,
2337        allow_smtp_auth_plain_without_valid_certificate: false,
2338        max_message_rate: Some(
2339            100/s,
2340        ),
2341        additional_message_rate_throttles: {},
2342        source_selection_rate: None,
2343        additional_source_selection_rates: {},
2344        max_connection_rate: Some(
2345            100/m,
2346        ),
2347        max_deliveries_per_connection: 100,
2348        max_recipients_per_batch: 100,
2349        prohibited_hosts: {
2350            "0.0.0.0",
2351            "127.0.0.0/8",
2352            "::/127",
2353        },
2354        skip_hosts: {},
2355        ip_lookup_strategy: Ipv4AndIpv6,
2356        max_mx_plan_size: 50,
2357        max_mx_addresses_per_host: 10,
2358        ehlo_domain: None,
2359        aggressive_connection_opening: false,
2360        refresh_interval: 60s,
2361        refresh_strategy: Ttl,
2362        dispatcher_wakeup_strategy: Aggressive,
2363        maintainer_wakeup_strategy: Aggressive,
2364        provider_name: None,
2365        remember_broken_tls: None,
2366        opportunistic_tls_reconnect_on_failed_handshake: false,
2367        use_lmtp: false,
2368        reconnect_strategy: ConnectNextHost,
2369        readyq_pool_name: None,
2370        low_memory_reduction_policy: ShrinkDataAndMeta,
2371        no_memory_reduction_policy: ShrinkDataAndMeta,
2372        try_next_host_on_transport_error: false,
2373        ignore_8bit_checks: false,
2374        dispatcher_progress_watchdog_timeout: None,
2375    },
2376    sources: {
2377        "my source name": EgressPathConfig {
2378            connection_limit: 5,
2379            additional_connection_limits: {},
2380            enable_tls: Opportunistic,
2381            tls_mode: StartTls,
2382            enable_mta_sts: true,
2383            enable_dane: false,
2384            enable_pipelining: true,
2385            enable_rset: true,
2386            tls_prefer_openssl: false,
2387            tls_certificate: None,
2388            tls_private_key: None,
2389            openssl_cipher_list: None,
2390            openssl_cipher_suites: None,
2391            openssl_options: None,
2392            rustls_cipher_suites: [],
2393            client_timeouts: SmtpClientTimeouts {
2394                connect_timeout: 60s,
2395                banner_timeout: 60s,
2396                ehlo_timeout: 300s,
2397                mail_from_timeout: 300s,
2398                rcpt_to_timeout: 300s,
2399                data_timeout: 300s,
2400                data_dot_timeout: 300s,
2401                rset_timeout: 5s,
2402                idle_timeout: 5s,
2403                starttls_timeout: 5s,
2404                auth_timeout: 60s,
2405            },
2406            system_shutdown_timeout: None,
2407            max_ready: 1024,
2408            consecutive_connection_failures_before_delay: 100,
2409            smtp_port: 25,
2410            smtp_auth_plain_username: None,
2411            smtp_auth_plain_password: None,
2412            allow_smtp_auth_plain_without_tls: false,
2413            allow_smtp_auth_plain_without_valid_certificate: false,
2414            max_message_rate: None,
2415            additional_message_rate_throttles: {},
2416            source_selection_rate: None,
2417            additional_source_selection_rates: {},
2418            max_connection_rate: None,
2419            max_deliveries_per_connection: 1024,
2420            max_recipients_per_batch: 100,
2421            prohibited_hosts: {
2422                "0.0.0.0",
2423                "127.0.0.0/8",
2424                "::/127",
2425            },
2426            skip_hosts: {},
2427            ip_lookup_strategy: Ipv4AndIpv6,
2428            max_mx_plan_size: 50,
2429            max_mx_addresses_per_host: 10,
2430            ehlo_domain: None,
2431            aggressive_connection_opening: false,
2432            refresh_interval: 60s,
2433            refresh_strategy: Ttl,
2434            dispatcher_wakeup_strategy: Aggressive,
2435            maintainer_wakeup_strategy: Aggressive,
2436            provider_name: None,
2437            remember_broken_tls: None,
2438            opportunistic_tls_reconnect_on_failed_handshake: false,
2439            use_lmtp: false,
2440            reconnect_strategy: ConnectNextHost,
2441            readyq_pool_name: None,
2442            low_memory_reduction_policy: ShrinkDataAndMeta,
2443            no_memory_reduction_policy: ShrinkDataAndMeta,
2444            try_next_host_on_transport_error: false,
2445            ignore_8bit_checks: false,
2446            dispatcher_progress_watchdog_timeout: None,
2447        },
2448    },
2449    automation: [
2450        Rule {
2451            regex: [
2452                Regex(
2453                    /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2454                ),
2455                Regex(
2456                    /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2457                ),
2458                Regex(
2459                    /has been temporarily rate limited due to IP reputation/,
2460                ),
2461                Regex(
2462                    /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2463                ),
2464                Regex(
2465                    /Server busy\. Please try again later from/,
2466                ),
2467            ],
2468            action: [
2469                SetConfig(
2470                    EgressPathConfigValue {
2471                        name: "max_message_rate",
2472                        value: HashableTomlValue {
2473                            value: String(
2474                                "1/minute",
2475                            ),
2476                        },
2477                    },
2478                ),
2479                SetConfig(
2480                    EgressPathConfigValue {
2481                        name: "connection_limit",
2482                        value: HashableTomlValue {
2483                            value: Integer(
2484                                1,
2485                            ),
2486                        },
2487                    },
2488                ),
2489            ],
2490            trigger: Immediate,
2491            duration: 5400s,
2492            match_internal: false,
2493        },
2494    ],
2495}
2496"#
2497        );
2498
2499        // The site name here will need to be updated if yahoo changes
2500        // their MX records
2501        let yahoo_com = shaping
2502            .get_egress_path_config(
2503                "yahoo.com",
2504                "invalid.source",
2505                "(mta5|mta6|mta7).am0.yahoodns.net",
2506            )
2507            .await
2508            .finish()
2509            .unwrap();
2510        k9::snapshot!(
2511            yahoo_com,
2512            r#"
2513MergedEntry {
2514    params: EgressPathConfig {
2515        connection_limit: 10,
2516        additional_connection_limits: {},
2517        enable_tls: Opportunistic,
2518        tls_mode: StartTls,
2519        enable_mta_sts: true,
2520        enable_dane: false,
2521        enable_pipelining: true,
2522        enable_rset: true,
2523        tls_prefer_openssl: false,
2524        tls_certificate: None,
2525        tls_private_key: None,
2526        openssl_cipher_list: None,
2527        openssl_cipher_suites: None,
2528        openssl_options: None,
2529        rustls_cipher_suites: [],
2530        client_timeouts: SmtpClientTimeouts {
2531            connect_timeout: 60s,
2532            banner_timeout: 60s,
2533            ehlo_timeout: 300s,
2534            mail_from_timeout: 300s,
2535            rcpt_to_timeout: 300s,
2536            data_timeout: 30s,
2537            data_dot_timeout: 60s,
2538            rset_timeout: 5s,
2539            idle_timeout: 60s,
2540            starttls_timeout: 5s,
2541            auth_timeout: 60s,
2542        },
2543        system_shutdown_timeout: None,
2544        max_ready: 1024,
2545        consecutive_connection_failures_before_delay: 100,
2546        smtp_port: 25,
2547        smtp_auth_plain_username: None,
2548        smtp_auth_plain_password: None,
2549        allow_smtp_auth_plain_without_tls: false,
2550        allow_smtp_auth_plain_without_valid_certificate: false,
2551        max_message_rate: Some(
2552            100/s,
2553        ),
2554        additional_message_rate_throttles: {},
2555        source_selection_rate: None,
2556        additional_source_selection_rates: {},
2557        max_connection_rate: Some(
2558            100/m,
2559        ),
2560        max_deliveries_per_connection: 20,
2561        max_recipients_per_batch: 100,
2562        prohibited_hosts: {
2563            "0.0.0.0",
2564            "127.0.0.0/8",
2565            "::/127",
2566        },
2567        skip_hosts: {},
2568        ip_lookup_strategy: Ipv4AndIpv6,
2569        max_mx_plan_size: 50,
2570        max_mx_addresses_per_host: 10,
2571        ehlo_domain: None,
2572        aggressive_connection_opening: false,
2573        refresh_interval: 60s,
2574        refresh_strategy: Ttl,
2575        dispatcher_wakeup_strategy: Aggressive,
2576        maintainer_wakeup_strategy: Aggressive,
2577        provider_name: None,
2578        remember_broken_tls: None,
2579        opportunistic_tls_reconnect_on_failed_handshake: false,
2580        use_lmtp: false,
2581        reconnect_strategy: ConnectNextHost,
2582        readyq_pool_name: None,
2583        low_memory_reduction_policy: ShrinkDataAndMeta,
2584        no_memory_reduction_policy: ShrinkDataAndMeta,
2585        try_next_host_on_transport_error: false,
2586        ignore_8bit_checks: false,
2587        dispatcher_progress_watchdog_timeout: None,
2588    },
2589    sources: {},
2590    automation: [
2591        Rule {
2592            regex: [
2593                Regex(
2594                    /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2595                ),
2596                Regex(
2597                    /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2598                ),
2599                Regex(
2600                    /has been temporarily rate limited due to IP reputation/,
2601                ),
2602                Regex(
2603                    /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2604                ),
2605                Regex(
2606                    /Server busy\. Please try again later from/,
2607                ),
2608            ],
2609            action: [
2610                SetConfig(
2611                    EgressPathConfigValue {
2612                        name: "max_message_rate",
2613                        value: HashableTomlValue {
2614                            value: String(
2615                                "1/minute",
2616                            ),
2617                        },
2618                    },
2619                ),
2620                SetConfig(
2621                    EgressPathConfigValue {
2622                        name: "connection_limit",
2623                        value: HashableTomlValue {
2624                            value: Integer(
2625                                1,
2626                            ),
2627                        },
2628                    },
2629                ),
2630            ],
2631            trigger: Immediate,
2632            duration: 5400s,
2633            match_internal: false,
2634        },
2635        Rule {
2636            regex: [
2637                Regex(
2638                    \[TS04\],
2639                ),
2640            ],
2641            action: [
2642                Suspend,
2643            ],
2644            trigger: Immediate,
2645            duration: 7200s,
2646            match_internal: false,
2647        },
2648    ],
2649}
2650"#
2651        );
2652    }
2653
2654    #[tokio::test]
2655    async fn test_load_default_shaping_toml() {
2656        Shaping::merge_files(
2657            &["../../assets/policy-extras/shaping.toml".into()],
2658            &ShapingMergeOptions::default(),
2659        )
2660        .await
2661        .unwrap();
2662    }
2663}