1use crate::egress_path::EgressPathConfig;
2#[cfg(feature = "lua")]
3use anyhow::Context;
4#[cfg(feature = "lua")]
5use config::any_err;
6#[cfg(feature = "lua")]
7use config::serialize_options;
8#[cfg(feature = "lua")]
9use dns_resolver::fully_qualify;
10#[cfg(feature = "lua")]
11use kumo_log_types::JsonLogRecord;
12#[cfg(feature = "lua")]
13use mailexchanger::MailExchanger;
14#[cfg(feature = "lua")]
15use mlua::prelude::LuaUserData;
16#[cfg(feature = "lua")]
17use mlua::{LuaSerdeExt, UserDataMethods};
18use ordermap::OrderMap;
19use serde::de::{SeqAccess, Visitor};
20use serde::{Deserialize, Deserializer, Serialize};
21use serde_with::formats::PreferOne;
22use serde_with::{serde_as, DeserializeAs, OneOrMany};
23#[cfg(feature = "lua")]
24use sha2::{Digest, Sha256};
25#[cfg(feature = "lua")]
26use std::collections::BTreeMap;
27use std::hash::{Hash, Hasher};
28use std::marker::PhantomData;
29#[cfg(feature = "lua")]
30use std::sync::Arc;
31use std::time::Duration;
32#[cfg(feature = "lua")]
33use throttle::LimitSpec;
34use throttle::ThrottleSpec;
35
36#[derive(Deserialize, Serialize, Debug, Clone)]
37#[serde(try_from = "String", into = "String")]
38pub struct Regex(fancy_regex::Regex);
39
40impl TryFrom<String> for Regex {
41 type Error = fancy_regex::Error;
42
43 fn try_from(s: String) -> fancy_regex::Result<Self> {
44 Ok(Self(fancy_regex::Regex::new(&s)?))
45 }
46}
47
48impl From<Regex> for String {
49 fn from(r: Regex) -> String {
50 r.0.as_str().to_string()
51 }
52}
53
54impl std::ops::Deref for Regex {
55 type Target = fancy_regex::Regex;
56 fn deref(&self) -> &Self::Target {
57 &self.0
58 }
59}
60
61impl std::hash::Hash for Regex {
62 fn hash<H: std::hash::Hasher>(&self, hasher: &mut H) {
63 self.0.as_str().hash(hasher)
64 }
65}
66
67#[derive(Deserialize, Serialize, Debug, Clone)]
73#[serde(from = "toml::Value", into = "toml::Value")]
74pub struct HashableTomlValue {
75 value: toml::Value,
76}
77
78impl From<toml::Value> for HashableTomlValue {
79 fn from(value: toml::Value) -> Self {
80 Self { value }
81 }
82}
83
84impl From<HashableTomlValue> for toml::Value {
85 fn from(value: HashableTomlValue) -> toml::Value {
86 value.value
87 }
88}
89
90impl std::ops::Deref for HashableTomlValue {
91 type Target = toml::Value;
92 fn deref(&self) -> &toml::Value {
93 &self.value
94 }
95}
96
97fn hash_toml<H>(value: &toml::Value, h: &mut H)
98where
99 H: Hasher,
100{
101 match value {
102 toml::Value::Boolean(v) => v.hash(h),
103 toml::Value::Datetime(v) => {
104 if let Some(d) = &v.date {
105 d.year.hash(h);
106 d.month.hash(h);
107 d.day.hash(h);
108 }
109 if let Some(t) = &v.time {
110 t.hour.hash(h);
111 t.minute.hash(h);
112 t.second.hash(h);
113 t.nanosecond.hash(h);
114 }
115 if let Some(toml::value::Offset::Custom { minutes }) = &v.offset {
116 minutes.hash(h);
117 }
118 }
119 toml::Value::String(v) => v.hash(h),
120 toml::Value::Integer(v) => v.hash(h),
121 toml::Value::Float(v) => v.to_ne_bytes().hash(h),
122 toml::Value::Array(a) => {
123 for v in a.iter() {
124 hash_toml(v, h);
125 }
126 }
127 toml::Value::Table(m) => {
128 for (k, v) in m.iter() {
129 k.hash(h);
130 hash_toml(v, h);
131 }
132 }
133 }
134}
135
136impl Hash for HashableTomlValue {
137 fn hash<H>(&self, h: &mut H)
138 where
139 H: Hasher,
140 {
141 hash_toml(&self.value, h);
142 }
143}
144
145#[derive(Deserialize, Serialize, Debug, Clone, Hash)]
148#[serde(
149 try_from = "EgressPathConfigValueUnchecked",
150 into = "EgressPathConfigValueUnchecked"
151)]
152pub struct EgressPathConfigValue {
153 pub name: String,
154 pub value: HashableTomlValue,
155}
156
157#[derive(Deserialize, Serialize, Debug, Clone)]
163pub struct EgressPathConfigValueUnchecked {
164 pub name: String,
165 pub value: toml::Value,
166}
167
168impl TryFrom<EgressPathConfigValueUnchecked> for EgressPathConfigValue {
169 type Error = anyhow::Error;
170 fn try_from(config: EgressPathConfigValueUnchecked) -> anyhow::Result<EgressPathConfigValue> {
171 let mut map = toml::map::Map::new();
172 map.insert(config.name.clone(), config.value.clone());
173 let table = toml::Value::Table(map);
174
175 EgressPathConfig::deserialize(table)?;
178
179 Ok(EgressPathConfigValue {
181 name: config.name,
182 value: HashableTomlValue {
183 value: config.value,
184 },
185 })
186 }
187}
188
189impl From<EgressPathConfigValue> for EgressPathConfigValueUnchecked {
190 fn from(config: EgressPathConfigValue) -> EgressPathConfigValueUnchecked {
191 EgressPathConfigValueUnchecked {
192 name: config.name,
193 value: config.value.value,
194 }
195 }
196}
197
198#[derive(Deserialize, Serialize, Debug, Clone, Hash)]
199pub enum Action {
200 Suspend,
201 SetConfig(EgressPathConfigValue),
202 SuspendTenant,
203 SuspendCampaign,
204 SetDomainConfig(EgressPathConfigValue),
205 SetProviderConfig(EgressPathConfigValue),
206 Bounce,
207 BounceTenant,
208 BounceCampaign,
209}
210
211#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash, Default)]
212pub enum Trigger {
213 #[default]
215 Immediate,
216 Threshold(ThrottleSpec),
219}
220
221#[serde_as]
222#[derive(Deserialize, Serialize, Debug, Hash, Clone)]
223pub struct Rule {
224 #[serde(deserialize_with = "regex_string_or_array")]
225 pub regex: Vec<Regex>,
226
227 #[serde(deserialize_with = "one_or_many_action")]
228 pub action: Vec<Action>,
229
230 #[serde(default)]
231 pub trigger: Trigger,
232
233 #[serde(with = "duration_serde")]
234 pub duration: Duration,
235
236 #[serde(default)]
240 pub match_internal: bool,
241}
242
243impl Rule {
244 pub fn matches(&self, is_internal: bool, response: &str) -> bool {
245 if is_internal && !self.match_internal {
246 return false;
247 }
248 self.regex
249 .iter()
250 .any(|r| r.is_match(response).unwrap_or(false))
251 }
252}
253
254#[cfg(feature = "lua")]
257#[derive(Debug, Clone, Hash, PartialEq, Eq)]
258pub enum MatchScope {
259 Rollup,
262 Domain,
264 Provider(String),
266}
267
268#[cfg(feature = "lua")]
270#[derive(Debug, Clone)]
271pub struct MatchedRule {
272 pub rule: Rule,
273 pub scope: MatchScope,
274}
275
276#[cfg(feature = "lua")]
277impl MatchedRule {
278 pub fn hash_into<H: std::hash::Hasher>(&self, hasher: &mut H) {
280 self.rule.hash(hasher);
281 self.scope.hash(hasher);
282 }
283}
284
285#[cfg(feature = "lua")]
286#[derive(Debug, Default)]
287struct ShapingInner {
288 by_site: OrderMap<String, PartialEntry>,
289 by_domain: OrderMap<String, PartialEntry>,
290 by_provider: OrderMap<String, ProviderEntry>,
291 warnings: Vec<String>,
292 errors: Vec<String>,
293 hash: String,
294}
295
296#[cfg(feature = "lua")]
297impl ShapingInner {
298 pub async fn get_egress_path_config(
299 &self,
300 domain: &str,
301 egress_source: &str,
302 site_name: &str,
303 ) -> PartialEntry {
304 let mut params = PartialEntry::default();
305
306 if let Some(default) = self.by_domain.get("default") {
308 params.merge_from(default.clone());
309 }
310
311 let is_domain_name = dns_resolver::Name::from_str_relaxed(domain).is_ok();
316 if is_domain_name {
317 let mut prov_with_sources = vec![];
318
319 for prov in self.by_provider.values() {
320 if prov.domain_matches(domain).await {
321 toml_table_merge_from(&mut params.params, &prov.params);
322 prov.apply_provider_params_to(egress_source, &mut params.params);
323
324 if !prov.sources.is_empty() {
325 prov_with_sources.push(prov);
329 }
330 }
331 }
332
333 for prov in prov_with_sources {
335 if let Some(source) = prov.sources.get(egress_source) {
336 toml_table_merge_from(&mut params.params, source);
337 prov.apply_provider_params_to(egress_source, &mut params.params);
338 }
339 }
340 }
341
342 if let Some(by_site) = self.by_site.get(site_name) {
344 params.merge_from(by_site.clone());
345 }
346
347 if let Some(by_domain) = self.by_domain.get(domain) {
349 params.merge_from(by_domain.clone());
350 }
351
352 if let Some(by_site) = self.by_site.get(site_name) {
354 if let Some(source) = by_site.sources.get(egress_source) {
355 toml_table_merge_from(&mut params.params, source);
356 }
357 }
358
359 if let Some(by_domain) = self.by_domain.get(domain) {
361 if let Some(source) = by_domain.sources.get(egress_source) {
362 toml_table_merge_from(&mut params.params, source);
363 }
364 }
365
366 params
367 }
368
369 pub async fn match_rules(&self, record: &JsonLogRecord) -> anyhow::Result<Vec<MatchedRule>> {
370 use rfc5321::parser::ForwardPath;
371 let recipient = ForwardPath::try_from(
373 record
374 .recipient
375 .first()
376 .ok_or_else(|| anyhow::anyhow!("no recipients!?"))?
377 .as_str(),
378 )
379 .map_err(|err| anyhow::anyhow!("parsing record.recipient: {err}"))?;
380
381 let recipient = match recipient {
382 ForwardPath::Postmaster => {
383 return Ok(vec![]);
386 }
387 ForwardPath::Path(path) => path.mailbox,
388 };
389 let domain = recipient.domain.to_string();
390
391 let source = record.egress_source.as_deref().unwrap_or("unspecified");
393 let site_name = record
406 .site
407 .trim_start_matches(&format!("{source}->"))
408 .trim_end_matches("@smtp_client")
409 .to_string();
410
411 Ok(self.match_rules_impl(record, &domain, &site_name).await)
412 }
413
414 pub async fn match_rules_impl(
415 &self,
416 record: &JsonLogRecord,
417 domain: &str,
418 site_name: &str,
419 ) -> Vec<MatchedRule> {
420 let mut result = vec![];
421 let response = record.response.to_single_line();
422 tracing::trace!("Consider rules for {response}");
423
424 let is_internal = record.response.content.starts_with("KumoMTA internal: ");
425
426 if let Some(default) = self.by_domain.get("default") {
427 for rule in &default.automation {
428 tracing::trace!("Consider \"default\" rule {rule:?} for {response}");
429 if rule.matches(is_internal, &response) {
430 result.push(MatchedRule {
437 rule: rule.clone(),
438 scope: MatchScope::Rollup,
439 });
440 }
441 }
442 }
443
444 for prov in self.by_provider.values() {
445 if prov.domain_matches(domain).await {
446 for rule in &prov.automation {
447 tracing::trace!(
448 "Consider provider \"{}\" rule {rule:?} for {response}",
449 prov.provider_name
450 );
451 if rule.matches(is_internal, &response) {
452 result.push(MatchedRule {
453 rule: rule.clone(),
454 scope: MatchScope::Provider(prov.provider_name.clone()),
455 });
456 }
457 }
458 }
459 }
460
461 if let Some(by_site) = self.by_site.get(site_name) {
463 for rule in &by_site.automation {
464 tracing::trace!("Consider \"{site_name}\" rule {rule:?} for {response}");
465 if rule.matches(is_internal, &response) {
466 result.push(MatchedRule {
467 rule: rule.clone(),
468 scope: MatchScope::Rollup,
469 });
470 }
471 }
472 }
473
474 if let Some(by_domain) = self.by_domain.get(domain) {
476 for rule in &by_domain.automation {
477 tracing::trace!("Consider \"{domain}\" rule {rule:?} for {response}");
478 if rule.matches(is_internal, &response) {
479 result.push(MatchedRule {
480 rule: rule.clone(),
481 scope: MatchScope::Domain,
482 });
483 }
484 }
485 }
486
487 result
488 }
489}
490
491#[cfg(feature = "lua")]
492#[derive(Debug, Default, Clone, mlua::FromLua)]
493pub struct Shaping {
494 inner: Arc<ShapingInner>,
495}
496
497#[cfg(feature = "lua")]
498fn from_json<'a, T: Deserialize<'a>>(json: &'a str) -> anyhow::Result<T> {
499 let d = &mut serde_json::Deserializer::from_str(json);
500 Ok(serde_path_to_error::deserialize(d)?)
501}
502
503#[cfg(feature = "lua")]
504fn from_toml<'a, T: Deserialize<'a>>(toml: &'a str) -> anyhow::Result<T> {
505 let d = toml::Deserializer::parse(toml)?;
506 Ok(serde_path_to_error::deserialize(d)?)
507}
508
509#[cfg(feature = "lua")]
510#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
511pub enum CheckLevel {
512 #[default]
513 Ignore,
514 Warn,
515 Error,
516}
517#[cfg(feature = "lua")]
518impl std::str::FromStr for CheckLevel {
519 type Err = String;
520
521 fn from_str(s: &str) -> Result<Self, String> {
522 if s.eq_ignore_ascii_case("ignore") {
523 Ok(Self::Ignore)
524 } else if s.eq_ignore_ascii_case("warn") {
525 Ok(Self::Warn)
526 } else if s.eq_ignore_ascii_case("error") {
527 Ok(Self::Error)
528 } else {
529 Err(format!(
530 "Expected one of `Ignore`, `Warn` or `Error`, got `{s}`"
531 ))
532 }
533 }
534}
535
536#[cfg(feature = "lua")]
537#[derive(Default)]
538struct Collector {
539 warnings: Vec<String>,
540 errors: Vec<String>,
541}
542
543#[cfg(feature = "lua")]
544impl Collector {
545 fn push<S: Into<String>>(&mut self, level: CheckLevel, msg: S) {
546 match level {
547 CheckLevel::Ignore => {}
548 CheckLevel::Warn => self.warnings.push(msg.into()),
549 CheckLevel::Error => self.errors.push(msg.into()),
550 }
551 }
552}
553
554#[cfg(feature = "lua")]
558fn reject_provider_only_actions(automation: &[Rule], scope: &str, collector: &mut Collector) {
559 for rule in automation {
560 if rule
561 .action
562 .iter()
563 .any(|action| matches!(action, Action::SetProviderConfig(_)))
564 {
565 collector.push(
566 CheckLevel::Error,
567 format!("SetProviderConfig is only valid within a provider block, but was used in {scope}"),
568 );
569 }
570 }
571}
572
573#[cfg(feature = "lua")]
574#[derive(Debug, Clone, Deserialize)]
575#[serde_as]
576#[serde(deny_unknown_fields)]
577pub struct ShapingMergeOptions {
578 #[serde(default)]
579 pub provider_overlap: CheckLevel,
580 #[serde(default)]
581 pub dns_fail: CheckLevel,
582 #[serde(default)]
583 pub null_mx: CheckLevel,
584 #[serde(default)]
585 pub aliased_site: CheckLevel,
586 #[serde(default)]
587 pub skip_remote: bool,
588 #[serde(default)]
589 pub remote_load: CheckLevel,
590 #[serde(default)]
591 pub local_load: CheckLevel,
592 #[serde(default, with = "duration_serde")]
593 pub http_timeout: Option<Duration>,
594}
595
596#[cfg(feature = "lua")]
597impl Default for ShapingMergeOptions {
598 fn default() -> Self {
599 Self {
600 provider_overlap: CheckLevel::Ignore,
601 dns_fail: CheckLevel::Ignore,
602 null_mx: CheckLevel::Ignore,
603 aliased_site: CheckLevel::Ignore,
604 skip_remote: false,
605 remote_load: CheckLevel::Ignore,
606 local_load: CheckLevel::Error,
607 http_timeout: None,
608 }
609 }
610}
611
612#[cfg(feature = "lua")]
613impl Shaping {
614 async fn load_from_file(
615 path: &str,
616 options: &ShapingMergeOptions,
617 collector: &mut Collector,
618 ) -> anyhow::Result<ShapingFile> {
619 let (data, level): (String, CheckLevel) =
620 if path.starts_with("http://") || path.starts_with("https://") {
621 if options.skip_remote {
622 collector.push(
623 CheckLevel::Warn,
624 format!("Ignoring {path} because skip_remote is set to true"),
625 );
626 return Ok(ShapingFile::default());
627 }
628
629 async fn http_get(url: &str, timeout: Duration) -> anyhow::Result<String> {
635 tokio::time::timeout(timeout, async {
636 let response = kumo_http_client::builder()
637 .timeout(timeout)
638 .connect_timeout(timeout)
639 .read_timeout(timeout)
640 .build()?
641 .get(url)
642 .send()
643 .await
644 .with_context(|| format!("making HTTP request to {url}"))?;
645 let status = response.status();
646 if !status.is_success() {
647 let err_text = match response.text().await {
648 Ok(text) => text,
649 Err(err) => {
650 format!("Additional error {err:#} while reading response body")
651 }
652 };
653 anyhow::bail!(
654 "HTTP request to {url} failed: status {status} {reason} {err_text}",
655 status = status.as_u16(),
656 reason = status.canonical_reason().unwrap_or("")
657 );
658 }
659 response
660 .text()
661 .await
662 .with_context(|| format!("reading text from {url}"))
663 })
664 .await
665 .with_context(|| format!("timeout making HTTP request to {url}"))?
666 }
667
668 let timeout = options.http_timeout.unwrap_or(Duration::from_secs(5));
669
670 match http_get(path, timeout).await {
671 Ok(s) => (s, options.remote_load),
672 Err(err) => {
673 tracing::error!("{err:#}. Ignoring this shaping source for now");
674 collector.push(
675 options.remote_load,
676 format!("remote shaping source {path} error: {err:#}"),
677 );
678 return Ok(ShapingFile::default());
679 }
680 }
681 } else {
682 match std::fs::read_to_string(path)
683 .with_context(|| format!("loading data from file {path}"))
684 {
685 Err(err) => {
686 collector.push(
687 options.local_load,
688 format!("local shaping source {path} error: {err:#}"),
689 );
690 return Ok(ShapingFile::default());
691 }
692 Ok(s) => (s, options.local_load),
693 }
694 };
695
696 if path.ends_with(".toml") {
697 from_toml(&data).with_context(|| format!("parsing toml from file {path}"))
698 } else if path.ends_with(".json") {
699 from_json(&data).with_context(|| format!("parsing json from file {path}"))
700 } else {
701 let mut errors = vec![];
703 match from_toml(&data) {
704 Ok(s) => return Ok(s),
705 Err(err) => errors.push(format!("as toml: {err:#}")),
706 }
707 match from_json(&data) {
708 Ok(s) => return Ok(s),
709 Err(err) => errors.push(format!("as json: {err:#}")),
710 }
711
712 collector.push(level, format!("parsing {path}: {}", errors.join(", ")));
713 Ok(ShapingFile::default())
714 }
715 }
716
717 pub async fn merge_files(
718 files: &[String],
719 options: &ShapingMergeOptions,
720 ) -> anyhow::Result<Self> {
721 use futures_util::stream::FuturesUnordered;
722 use futures_util::StreamExt;
723
724 let mut collector = Collector::default();
725 let mut loaded = vec![];
726 for p in files {
727 loaded.push(Self::load_from_file(p, options, &mut collector).await?);
728 }
729
730 let mut by_site: OrderMap<String, PartialEntry> = OrderMap::new();
731 let mut by_domain: OrderMap<String, PartialEntry> = OrderMap::new();
732 let mut by_provider: OrderMap<String, ProviderEntry> = OrderMap::new();
733 let mut site_aliases: OrderMap<String, Vec<String>> = OrderMap::new();
734
735 let mut mx = std::collections::HashMap::new();
739 let mut lookups = FuturesUnordered::new();
740 for item in &loaded {
741 for (domain, partial) in &item.domains {
742 if partial.mx_rollup {
743 let domain = domain.to_string();
744 lookups.push(tokio::spawn(async move {
745 let mx_result = MailExchanger::resolve(&domain).await;
746 (domain, mx_result)
747 }));
748 }
749 }
750 }
751
752 while let Some(Ok((domain, result))) = lookups.next().await {
753 mx.insert(domain, result);
754 }
755
756 for mut item in loaded {
757 if let Some(mut partial) = item.default.take() {
758 let domain = "default";
759 partial.domain_name.replace(domain.to_string());
760 reject_provider_only_actions(
761 &partial.automation,
762 "the default block",
763 &mut collector,
764 );
765 match by_domain.get_mut(domain) {
766 Some(existing) => {
767 existing.merge_from(partial);
768 }
769 None => {
770 by_domain.insert(domain.to_string(), partial);
771 }
772 }
773 }
774
775 for (domain, mut partial) in item.domains {
776 partial.domain_name.replace(domain.clone());
777 reject_provider_only_actions(
778 &partial.automation,
779 &format!("domain '{domain}'"),
780 &mut collector,
781 );
782
783 if let Ok(name) = fully_qualify(&domain) {
784 if name.num_labels() == 1 {
785 collector.push(
786 CheckLevel::Warn,
787 format!(
788 "Entry for domain '{domain}' consists of a \
789 single DNS label. Domain names in TOML sections \
790 need to be quoted like '[\"{domain}.com\"]` otherwise \
791 the '.' will create a nested table rather than being \
792 added to the domain name."
793 ),
794 );
795 }
796 }
797
798 #[cfg(test)]
799 if partial._treat_domain_name_as_site_name {
800 match by_site.get_mut(&domain) {
801 Some(existing) => {
802 existing.merge_from(partial);
803 }
804 None => {
805 by_site.insert(domain.to_string(), partial);
806 }
807 }
808 continue;
809 }
810 if partial.mx_rollup {
811 let mx = match mx.get(&domain) {
812 Some(Ok(mx)) => mx,
813 Some(Err(err)) => {
814 tracing::debug!(
815 target: "shaping_load",
816 %domain,
817 "dropping domain from shaping: MX resolve failed: {err:#}"
818 );
819 collector.push(
820 options.dns_fail,
821 format!(
822 "error resolving MX for {domain}: {err:#}. \
823 Ignoring the shaping config for that domain."
824 ),
825 );
826 continue;
827 }
828 None => {
829 tracing::debug!(
830 target: "shaping_load",
831 %domain,
832 "dropping domain from shaping: MX was not resolved"
833 );
834 collector.push(
835 options.dns_fail,
836 format!(
837 "We didn't try to resolve the MX for {domain} for some reason!?. \
838 Ignoring the shaping config for that domain."
839 ),
840 );
841 continue;
842 }
843 };
844
845 if mx.site_name.is_empty() {
846 collector.push(
847 options.null_mx,
848 format!(
849 "domain {domain} has a NULL MX and cannot be used with mx_rollup=true. \
850 Ignoring the shaping config for that domain."),
851 );
852 continue;
853 }
854
855 tracing::trace!(
856 target: "shaping_load",
857 %domain,
858 site_name = %mx.site_name,
859 "domain resolved to site_name"
860 );
861
862 site_aliases
863 .entry(mx.site_name.to_string())
864 .or_default()
865 .push(domain.to_string());
866
867 match by_site.get_mut(&mx.site_name) {
868 Some(existing) => {
869 existing.merge_from(partial);
870 }
871 None => {
872 by_site.insert(mx.site_name.clone(), partial);
873 }
874 }
875 } else {
876 match by_domain.get_mut(&domain) {
877 Some(existing) => {
878 existing.merge_from(partial);
879 }
880 None => {
881 by_domain.insert(domain, partial);
882 }
883 }
884 }
885 }
886
887 for (provider, mut prov) in item.provider {
888 prov.provider_name = provider.to_string();
889 match by_provider.get_mut(&provider) {
890 Some(existing) => {
891 existing.merge_from(prov);
892 }
893 None => {
894 by_provider.insert(provider.to_string(), prov);
895 }
896 }
897 }
898 }
899
900 for (site, partial) in &by_site {
901 partial
902 .clone()
903 .finish()
904 .with_context(|| format!("site: {site}"))?;
905 }
906
907 for (domain, partial) in &by_domain {
908 partial
909 .clone()
910 .finish()
911 .with_context(|| format!("domain: {domain}"))?;
912 }
913
914 for (provider, prov) in &by_provider {
915 prov.finish_params()
916 .with_context(|| format!("provider: {provider}"))?;
917 }
918
919 if options.aliased_site != CheckLevel::Ignore {
920 for (site, aliases) in site_aliases {
921 if aliases.len() > 1 {
922 collector.push(
923 options.aliased_site,
924 format!(
925 "multiple domain blocks alias to the same site: {site}: {}",
926 aliases.join(", ")
927 ),
928 );
929 }
930 }
931 }
932
933 if options.provider_overlap != CheckLevel::Ignore {
934 for domain in mx.keys() {
935 let mut matching_providers = vec![];
936 for (prov_name, prov) in &by_provider {
937 if prov.domain_matches(domain).await {
938 matching_providers.push(prov_name.to_string());
939 }
940 }
941 if !matching_providers.is_empty() {
942 collector.push(
943 options.provider_overlap,
944 format!(
945 "domain {domain} is also matched by provider(s): {}",
946 matching_providers.join(", ")
947 ),
948 );
949 }
950 }
951 }
952
953 let mut ctx = Sha256::new();
954 ctx.update("by_site");
955 for (site, entry) in &by_site {
956 ctx.update(site);
957 entry.hash_into(&mut ctx);
958 }
959 ctx.update("by_domain");
960 for (domain, entry) in &by_domain {
961 ctx.update(domain);
962 entry.hash_into(&mut ctx);
963 }
964 ctx.update("by_provider");
965 for (provider, prov) in &by_provider {
966 ctx.update(provider);
967 prov.hash_into(&mut ctx);
968 }
969 let hash = ctx.finalize();
974 let hash = data_encoding::HEXLOWER.encode(&hash);
975
976 tracing::debug!(
977 target: "shaping_load",
978 %hash,
979 sites = by_site.len(),
980 domains = by_domain.len(),
981 providers = by_provider.len(),
982 warnings = collector.warnings.len(),
983 errors = collector.errors.len(),
984 "merged shaping config"
985 );
986
987 Ok(Self {
988 inner: Arc::new(ShapingInner {
989 by_site,
990 by_domain,
991 by_provider,
992 warnings: collector.warnings,
993 errors: collector.errors,
994 hash,
995 }),
996 })
997 }
998
999 async fn get_egress_path_config(
1000 &self,
1001 domain: &str,
1002 egress_source: &str,
1003 site_name: &str,
1004 ) -> PartialEntry {
1005 self.inner
1006 .get_egress_path_config(domain, egress_source, site_name)
1007 .await
1008 }
1009
1010 pub async fn get_egress_path_config_value(
1011 &self,
1012 domain: &str,
1013 egress_source: &str,
1014 site_name: &str,
1015 ) -> anyhow::Result<serde_json::Value> {
1016 let partial = self
1017 .get_egress_path_config(domain, egress_source, site_name)
1018 .await;
1019 Ok(serde_json::to_value(&partial)?)
1020 }
1021
1022 pub fn get_errors(&self) -> &[String] {
1023 &self.inner.errors
1024 }
1025
1026 pub fn get_warnings(&self) -> &[String] {
1027 &self.inner.warnings
1028 }
1029
1030 pub async fn match_rules(&self, record: &JsonLogRecord) -> anyhow::Result<Vec<MatchedRule>> {
1031 self.inner.match_rules(record).await
1032 }
1033
1034 pub fn get_referenced_sources(&self) -> BTreeMap<String, Vec<String>> {
1035 let mut result = BTreeMap::new();
1036
1037 for (site_name, site) in &self.inner.by_site {
1038 for source_name in site.sources.keys() {
1039 result
1040 .entry(source_name.to_string())
1041 .or_insert(vec![])
1042 .push(format!("site:{site_name}"));
1043 }
1044 }
1045 for (domain_name, domain) in &self.inner.by_domain {
1046 for source_name in domain.sources.keys() {
1047 result
1048 .entry(source_name.to_string())
1049 .or_insert(vec![])
1050 .push(format!("domain:{domain_name}"));
1051 }
1052 }
1053
1054 result
1055 }
1056
1057 pub fn hash(&self) -> String {
1058 self.inner.hash.clone()
1059 }
1060}
1061
1062#[cfg(feature = "lua")]
1063impl LuaUserData for Shaping {
1064 fn add_methods<M: UserDataMethods<Self>>(methods: &mut M) {
1065 mod_memoize::Memoized::impl_memoize(methods);
1066 methods.add_async_method(
1067 "get_egress_path_config",
1068 |lua, this, (domain, egress_source, site_name): (String, String, String)| async move {
1069 let params = this
1070 .get_egress_path_config(&domain, &egress_source, &site_name)
1071 .await;
1072 lua.to_value_with(¶ms.params, serialize_options())
1073 },
1074 );
1075
1076 methods.add_method("get_errors", move |_lua, this, ()| {
1077 let errors: Vec<String> = this.get_errors().iter().map(|s| s.to_string()).collect();
1078 Ok(errors)
1079 });
1080
1081 methods.add_method("get_warnings", move |_lua, this, ()| {
1082 let warnings: Vec<String> = this.get_warnings().iter().map(|s| s.to_string()).collect();
1083 Ok(warnings)
1084 });
1085
1086 methods.add_method("get_referenced_sources", move |_lua, this, ()| {
1087 Ok(this.get_referenced_sources())
1088 });
1089
1090 methods.add_async_method("match_rules", |lua, this, record: mlua::Value| async move {
1091 let record: JsonLogRecord = lua.from_value(record)?;
1092 let rules = this.match_rules(&record).await.map_err(any_err)?;
1093 let mut result = vec![];
1094 for matched in rules {
1095 result.push(lua.to_value(&matched.rule)?);
1098 }
1099 Ok(result)
1100 });
1101
1102 methods.add_method("hash", move |_, this, ()| Ok(this.hash()));
1103 }
1104}
1105
1106#[derive(Default, Debug)]
1107pub struct MergedEntry {
1108 pub params: EgressPathConfig,
1109 pub sources: OrderMap<String, EgressPathConfig>,
1110 pub automation: Vec<Rule>,
1111}
1112
1113#[cfg(feature = "lua")]
1114#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1115struct ShapingFile {
1116 pub default: Option<PartialEntry>,
1117 #[serde(flatten, default)]
1118 pub domains: OrderMap<String, PartialEntry>,
1119 #[serde(default)]
1120 pub provider: OrderMap<String, ProviderEntry>,
1121}
1122
1123#[cfg(feature = "lua")]
1124#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1125struct PartialEntry {
1126 #[serde(skip)]
1127 pub domain_name: Option<String>,
1128
1129 #[serde(flatten)]
1130 pub params: toml::Table,
1131
1132 #[serde(default = "default_true")]
1133 pub mx_rollup: bool,
1134
1135 #[cfg(test)]
1139 #[serde(default)]
1140 pub _treat_domain_name_as_site_name: bool,
1141
1142 #[serde(default)]
1143 pub replace_base: bool,
1144
1145 #[serde(default)]
1146 pub automation: Vec<Rule>,
1147
1148 #[serde(default)]
1149 pub sources: OrderMap<String, toml::Table>,
1150}
1151
1152#[cfg(feature = "lua")]
1153#[derive(Deserialize, Serialize, Debug, Clone, Default)]
1154pub struct ProviderEntry {
1155 #[serde(skip, default)]
1156 pub provider_name: String,
1157
1158 #[serde(default)]
1159 pub provider_connection_limit: Option<LimitSpec>,
1160
1161 #[serde(default)]
1162 pub provider_max_message_rate: Option<ThrottleSpec>,
1163
1164 #[serde(default, rename = "match")]
1165 pub matches: Vec<ProviderMatch>,
1166
1167 #[serde(default)]
1168 pub replace_base: bool,
1169
1170 #[serde(flatten)]
1171 pub params: toml::Table,
1172
1173 #[serde(default)]
1174 pub automation: Vec<Rule>,
1175
1176 #[serde(default)]
1177 pub sources: OrderMap<String, toml::Table>,
1178}
1179
1180#[cfg(feature = "lua")]
1181fn suffix_matches(candidate: &str, suffix: &str) -> bool {
1182 let candidate = candidate.strip_suffix(".").unwrap_or(candidate);
1185 candidate.ends_with(suffix)
1189}
1190
1191#[cfg(feature = "lua")]
1192#[cfg(test)]
1193#[test]
1194fn test_suffix_matches() {
1195 assert!(suffix_matches("a", "a"));
1196 assert!(suffix_matches("foo.com", "foo.com"));
1197 assert!(!suffix_matches("foo.com", ".foo.com"));
1198 assert!(!suffix_matches("foo.com", "longer.com"));
1199 assert!(!suffix_matches("réputation.net", ".mx.microsoft"));
1200
1201 assert!(!suffix_matches("foo.Com", ".com"));
1205 assert!(!suffix_matches("foo.Cam", ".com"));
1206}
1207
1208#[cfg(feature = "lua")]
1209fn host_matches(candidate: &str, name: &str) -> bool {
1210 let candidate = candidate.strip_suffix(".").unwrap_or(candidate);
1213 candidate == name
1214}
1215
1216#[cfg(feature = "lua")]
1217#[cfg(test)]
1218#[test]
1219fn test_host_matches() {
1220 assert!(host_matches("foo.com", "foo.com"));
1221 assert!(host_matches("foo.com.", "foo.com"));
1222 assert!(!host_matches("foo.com", "notfoo.com"));
1223}
1224
1225#[cfg(feature = "lua")]
1226impl ProviderEntry {
1227 async fn domain_matches(&self, domain: &str) -> bool {
1228 let mut need_mx = false;
1231
1232 tracing::trace!(
1233 "ProviderEntry::domain_matches({domain}) vs {:?}",
1234 self.matches
1235 );
1236
1237 for rule in &self.matches {
1238 match rule {
1239 ProviderMatch::DomainSuffix(suffix) => {
1240 if suffix_matches(domain, suffix) {
1241 tracing::trace!("{domain} suffix matches {suffix}");
1242 return true;
1243 }
1244 }
1245 ProviderMatch::HostName(_) | ProviderMatch::MXSuffix(_) => {
1246 need_mx = true;
1247 }
1248 }
1249 }
1250
1251 if !need_mx {
1252 return false;
1253 }
1254
1255 match MailExchanger::resolve(domain).await {
1257 Err(err) => {
1258 tracing::trace!(
1264 "Error resolving MX for {domain}: {err:#}. \
1265 Provider {} match rules will be ignored",
1266 self.provider_name
1267 );
1268 false
1269 }
1270 Ok(mx) => {
1271 tracing::trace!("Consider MXSuffix rules");
1272 for host in &mx.hosts {
1273 let mut matched = false;
1274
1275 for rule in &self.matches {
1276 match rule {
1277 ProviderMatch::MXSuffix(suffix) => {
1278 tracing::trace!("suffix={suffix} vs host {host}");
1283 if suffix_matches(host, suffix) {
1284 matched = true;
1285 break;
1286 }
1287 }
1288 ProviderMatch::HostName(name) => {
1289 if host_matches(host, name) {
1290 matched = true;
1291 break;
1292 }
1293 }
1294 ProviderMatch::DomainSuffix(_) => {}
1295 }
1296 }
1297
1298 if !matched {
1299 tracing::trace!("host didn't match any of these rules");
1300 return false;
1301 }
1302 }
1303
1304 true
1305 }
1306 }
1307 }
1308
1309 fn merge_from(&mut self, mut other: Self) {
1310 if other.replace_base {
1311 self.provider_connection_limit = other.provider_connection_limit;
1312 self.matches = other.matches;
1313 self.params = other.params;
1314 self.sources = other.sources;
1315 self.automation = other.automation;
1316 } else {
1317 if other.provider_connection_limit.is_some() {
1318 self.provider_connection_limit = other.provider_connection_limit;
1319 }
1320
1321 toml_table_merge_from(&mut self.params, &other.params);
1322
1323 for (source, tbl) in other.sources {
1324 match self.sources.get_mut(&source) {
1325 Some(existing) => {
1326 toml_table_merge_from(existing, &tbl);
1327 }
1328 None => {
1329 self.sources.insert(source, tbl);
1330 }
1331 }
1332 }
1333
1334 self.matches.append(&mut other.matches);
1335 self.automation.append(&mut other.automation);
1336 }
1337 }
1338
1339 fn apply_provider_params_to(&self, source: &str, target: &mut toml::Table) {
1340 let mut implied = toml::Table::new();
1341 implied.insert(
1342 "provider_name".to_string(),
1343 toml::Value::String(self.provider_name.to_string()),
1344 );
1345
1346 if let Some(limit) = &self.provider_connection_limit {
1347 let mut limits = toml::Table::new();
1348 limits.insert(
1349 format!("shaping-provider-{}-{source}-limit", self.provider_name),
1350 toml::Value::String(limit.to_string()),
1351 );
1352 implied.insert(
1353 "additional_connection_limits".to_string(),
1354 toml::Value::Table(limits),
1355 );
1356 }
1357 if let Some(rate) = &self.provider_max_message_rate {
1358 let rate = rate.as_string();
1359 let mut limits = toml::Table::new();
1360 limits.insert(
1361 format!("shaping-provider-{}-{source}-rate", self.provider_name),
1362 rate.into(),
1363 );
1364 implied.insert(
1365 "additional_message_rate_throttles".to_string(),
1366 toml::Value::Table(limits),
1367 );
1368 }
1369
1370 if let Some(rate) = target.remove("provider_source_selection_rate") {
1371 let mut limits = toml::Table::new();
1372 limits.insert(
1373 format!(
1374 "shaping-provider-{}-{source}-selection-rate",
1375 self.provider_name
1376 ),
1377 rate,
1378 );
1379 implied.insert(
1380 "additional_source_selection_rates".to_string(),
1381 toml::Value::Table(limits),
1382 );
1383 }
1384
1385 toml_table_merge_from(target, &implied);
1386 }
1387
1388 fn finish_params(&self) -> anyhow::Result<MergedEntry> {
1389 let provider_name = &self.provider_name;
1390
1391 let params = EgressPathConfig::deserialize(self.params.clone()).with_context(|| {
1392 format!(
1393 "interpreting provider '{provider_name}' params {:#?} as EgressPathConfig",
1394 self.params
1395 )
1396 })?;
1397 let mut sources = OrderMap::new();
1398
1399 for (source, params) in &self.sources {
1400 let mut params = params.clone();
1401 params.remove("provider_source_selection_rate");
1407 sources.insert(
1408 source.clone(),
1409 EgressPathConfig::deserialize(params.clone()).with_context(|| {
1410 format!("interpreting provider '{provider_name}' source '{source}' {params:#} as EgressPathConfig")
1411 })?,
1412 );
1413 }
1414
1415 Ok(MergedEntry {
1416 params,
1417 sources,
1418 automation: self.automation.clone(),
1419 })
1420 }
1421
1422 fn hash_into(&self, ctx: &mut Sha256) {
1423 ctx.update(&self.provider_name);
1424 ctx.update(serde_json::to_string(self).unwrap_or_else(|_| String::new()));
1425 }
1426}
1427
1428#[cfg(feature = "lua")]
1429#[derive(Deserialize, Serialize, Debug, Clone)]
1430pub enum ProviderMatch {
1431 MXSuffix(String),
1432 DomainSuffix(String),
1433 HostName(String),
1434}
1435
1436#[cfg(feature = "lua")]
1437fn toml_table_merge_from(tbl: &mut toml::Table, source: &toml::Table) {
1438 fn is_mergeable(name: &str) -> bool {
1441 match name {
1442 "additional_connection_limits"
1443 | "additional_message_rate_throttles"
1444 | "additional_source_selection_rates" => true,
1445 _ => false,
1446 }
1447 }
1448
1449 for (k, v) in source {
1450 match (tbl.get_mut(k), v.as_table()) {
1451 (Some(toml::Value::Table(existing)), Some(v)) if is_mergeable(k) => {
1453 for (inner_k, inner_v) in v {
1454 existing.insert(inner_k.clone(), inner_v.clone());
1455 }
1456 }
1457 _ => {
1458 tbl.insert(k.clone(), v.clone());
1459 }
1460 }
1461 }
1462}
1463
1464#[cfg(feature = "lua")]
1465impl PartialEntry {
1466 fn merge_from(&mut self, mut other: Self) {
1467 if other.replace_base {
1468 self.params = other.params;
1469 self.automation = other.automation;
1470 self.sources = other.sources;
1471 } else {
1472 toml_table_merge_from(&mut self.params, &other.params);
1473
1474 for (source, tbl) in other.sources {
1475 match self.sources.get_mut(&source) {
1476 Some(existing) => {
1477 toml_table_merge_from(existing, &tbl);
1478 }
1479 None => {
1480 self.sources.insert(source, tbl);
1481 }
1482 }
1483 }
1484
1485 self.automation.append(&mut other.automation);
1486 }
1487 }
1488
1489 fn finish(self) -> anyhow::Result<MergedEntry> {
1490 let domain = self.domain_name.unwrap_or_default();
1491
1492 let params = EgressPathConfig::deserialize(self.params.clone()).with_context(|| {
1493 format!(
1494 "interpreting domain '{domain}' params {:#?} as EgressPathConfig",
1495 self.params
1496 )
1497 })?;
1498 let mut sources = OrderMap::new();
1499
1500 for (source, params) in self.sources {
1501 sources.insert(
1502 source.clone(),
1503 EgressPathConfig::deserialize(params.clone()).with_context(|| {
1504 format!("interpreting domain '{domain}' source '{source}' {params:#} as EgressPathConfig")
1505 })?,
1506 );
1507 }
1508
1509 Ok(MergedEntry {
1510 params,
1511 sources,
1512 automation: self.automation,
1513 })
1514 }
1515
1516 fn hash_into(&self, ctx: &mut Sha256) {
1517 if let Some(name) = self.domain_name.as_ref() {
1518 ctx.update(name)
1519 }
1520 ctx.update(serde_json::to_string(self).unwrap_or_else(|_| String::new()));
1521 }
1522}
1523
1524fn one_or_many<'de, T, D>(deserializer: D, expecting: &str) -> Result<Vec<T>, D::Error>
1525where
1526 T: Deserialize<'de>,
1527 D: Deserializer<'de>,
1528{
1529 let result: Result<Vec<T>, _> =
1530 OneOrMany::<serde_with::Same, PreferOne>::deserialize_as(deserializer);
1531 match result {
1532 Ok(r) => Ok(r),
1533 Err(err) => Err(serde::de::Error::custom(format!(
1534 "{expecting}.\nThe underlying error message is:\n{err:#}"
1535 ))),
1536 }
1537}
1538
1539fn one_or_many_action<'de, D>(deserializer: D) -> Result<Vec<Action>, D::Error>
1540where
1541 D: Deserializer<'de>,
1542{
1543 one_or_many(
1544 deserializer,
1545 "\"action\" field expected either a single Action or an array of Actions",
1546 )
1547}
1548
1549fn regex_string_or_array<'de, D>(deserializer: D) -> Result<Vec<Regex>, D::Error>
1550where
1551 D: Deserializer<'de>,
1552{
1553 string_or_array(
1554 deserializer,
1555 "regex string or array of regex strings for field regex",
1556 )
1557}
1558
1559fn string_or_array<'de, T, D>(deserializer: D, expecting: &'static str) -> Result<Vec<T>, D::Error>
1560where
1561 T: Deserialize<'de> + TryFrom<String>,
1562 <T as TryFrom<String>>::Error: std::fmt::Debug,
1563 D: Deserializer<'de>,
1564{
1565 struct StringOrArray<T>(PhantomData<fn() -> T>, &'static str);
1571
1572 impl<'de, T> Visitor<'de> for StringOrArray<T>
1573 where
1574 T: Deserialize<'de> + TryFrom<String>,
1575 <T as TryFrom<String>>::Error: std::fmt::Debug,
1576 {
1577 type Value = Vec<T>;
1578
1579 fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
1580 formatter.write_str(self.1)
1581 }
1582
1583 fn visit_str<E>(self, value: &str) -> Result<Vec<T>, E>
1584 where
1585 E: serde::de::Error,
1586 {
1587 Ok(vec![
1588 T::try_from(value.to_string()).map_err(|e| E::custom(format!("{e:?}")))?
1589 ])
1590 }
1591
1592 fn visit_seq<S>(self, seq: S) -> Result<Vec<T>, S::Error>
1593 where
1594 S: SeqAccess<'de>,
1595 {
1596 Deserialize::deserialize(serde::de::value::SeqAccessDeserializer::new(seq))
1597 }
1598 }
1599
1600 deserializer.deserialize_any(StringOrArray(PhantomData, expecting))
1601}
1602
1603#[cfg(feature = "lua")]
1604fn default_true() -> bool {
1605 true
1606}
1607
1608#[cfg(feature = "lua")]
1609pub fn register(lua: &mlua::Lua) -> anyhow::Result<()> {
1610 let shaping_mod = config::get_or_create_sub_module(lua, "shaping")?;
1611
1612 shaping_mod.set(
1613 "load",
1614 lua.create_async_function(
1615 move |lua, (paths, options): (Vec<String>, Option<mlua::Value>)| async move {
1616 let options = match options {
1617 Some(v) => lua.from_value(v)?,
1618 None => Default::default(),
1619 };
1620 let shaping = Shaping::merge_files(&paths, &options)
1621 .await
1622 .map_err(any_err)?;
1623 Ok(shaping)
1624 },
1625 )?,
1626 )?;
1627
1628 Ok(())
1629}
1630
1631#[cfg(test)]
1632mod test {
1633 use super::*;
1634 use kumo_log_types::RecordType;
1635 use rfc5321::Response;
1636 use std::io::Write;
1637 use tempfile::NamedTempFile;
1638 use uuid::Uuid;
1639
1640 async fn make_shaping_configs(inputs: &[&str]) -> Shaping {
1641 let mut files = vec![];
1642 let mut file_names = vec![];
1643
1644 for (i, content) in inputs.iter().enumerate() {
1645 let mut shaping_file = NamedTempFile::with_prefix(format!("file{i}")).unwrap();
1646 shaping_file.write_all(content.as_bytes()).unwrap();
1647 file_names.push(shaping_file.path().to_str().unwrap().to_string());
1648 files.push(shaping_file);
1649 }
1650
1651 Shaping::merge_files(&file_names, &ShapingMergeOptions::default())
1652 .await
1653 .unwrap()
1654 }
1655
1656 #[tokio::test]
1657 async fn hash_excludes_diagnostics() {
1658 let content = r#"
1659["example.com"]
1660mx_rollup = false
1661connection_limit = 5
1662"#;
1663 let mut file = NamedTempFile::with_prefix("shaping").unwrap();
1664 file.write_all(content.as_bytes()).unwrap();
1665 let path = file.path().to_str().unwrap().to_string();
1666
1667 let options = ShapingMergeOptions {
1672 skip_remote: true,
1673 ..ShapingMergeOptions::default()
1674 };
1675
1676 let without_warning = Shaping::merge_files(std::slice::from_ref(&path), &options)
1677 .await
1678 .unwrap();
1679 let with_warning = Shaping::merge_files(
1680 &[
1681 path.clone(),
1682 "https://example.invalid/shaping.toml".to_string(),
1683 ],
1684 &options,
1685 )
1686 .await
1687 .unwrap();
1688
1689 assert!(
1690 without_warning.get_warnings().is_empty(),
1691 "the content-only load should record no warnings"
1692 );
1693 assert!(
1694 !with_warning.get_warnings().is_empty(),
1695 "the skipped remote source should record a warning"
1696 );
1697 k9::assert_equal!(without_warning.hash(), with_warning.hash());
1700
1701 let changed = content.replace("connection_limit = 5", "connection_limit = 6");
1703 let mut changed_file = NamedTempFile::with_prefix("shaping").unwrap();
1704 changed_file.write_all(changed.as_bytes()).unwrap();
1705 let changed_path = changed_file.path().to_str().unwrap().to_string();
1706 let changed = Shaping::merge_files(&[changed_path], &options)
1707 .await
1708 .unwrap();
1709 assert_ne!(without_warning.hash(), changed.hash());
1710 }
1711
1712 #[tokio::test]
1713 async fn test_merge_additional() {
1714 let shaping = make_shaping_configs(&[
1715 r#"
1716["example.com"]
1717mx_rollup = false
1718additional_connection_limits = {"first"=10}
1719 "#,
1720 r#"
1721["example.com"]
1722mx_rollup = false
1723additional_connection_limits = {"second"=32}
1724additional_message_rate_throttles = {"second"="100/hr"}
1725 "#,
1726 ])
1727 .await;
1728
1729 let resolved = shaping
1730 .get_egress_path_config("example.com", "invalid.source", "invalid.site")
1731 .await
1732 .finish()
1733 .unwrap();
1734
1735 k9::snapshot!(
1736 resolved.params.additional_connection_limits,
1737 r#"
1738{
1739 "first": 10,
1740 "second": 32,
1741}
1742"#
1743 );
1744 k9::snapshot!(
1745 resolved.params.additional_message_rate_throttles,
1746 r#"
1747{
1748 "second": 100/h,
1749}
1750"#
1751 );
1752 }
1753
1754 #[tokio::test]
1755 async fn test_provider_multi_hostname() {
1756 let shaping = make_shaping_configs(&[r#"
1757[provider."yahoo"]
1758match=[{HostName="mta5.am0.yahoodns.net"},{HostName="mta6.am0.yahoodns.net"},{HostName="mta7.am0.yahoodns.net"}]
1759enable_tls = "Required"
1760 "#])
1761 .await;
1762
1763 let resolved = shaping
1764 .get_egress_path_config("yahoo.com", "invalid.source", "invalid.site")
1765 .await
1766 .finish()
1767 .unwrap();
1768
1769 k9::assert_equal!(
1770 resolved.params.enable_tls,
1771 crate::egress_path::Tls::Required
1772 );
1773 k9::assert_equal!(resolved.params.provider_name.unwrap(), "yahoo");
1774 }
1775
1776 #[tokio::test]
1777 async fn test_provider_multi_suffix() {
1778 let shaping = make_shaping_configs(&[r#"
1779[provider."yahoo"]
1780match=[{MXSuffix="mta5.am0.yahoodns.net"},{MXSuffix="mta6.am0.yahoodns.net"},{MXSuffix="mta7.am0.yahoodns.net"}]
1781enable_tls = "Required"
1782 "#])
1783 .await;
1784
1785 let resolved = shaping
1786 .get_egress_path_config("yahoo.com", "invalid.source", "invalid.site")
1787 .await
1788 .finish()
1789 .unwrap();
1790
1791 k9::assert_equal!(
1792 resolved.params.enable_tls,
1793 crate::egress_path::Tls::Required
1794 );
1795 k9::assert_equal!(resolved.params.provider_name.unwrap(), "yahoo");
1796 }
1797
1798 #[tokio::test]
1799 async fn test_set_provider_config_rejected_outside_provider() {
1800 let shaping = make_shaping_configs(&[r#"
1802["example.com"]
1803mx_rollup = false
1804
1805[["example.com".automation]]
1806regex = "x"
1807action = {SetProviderConfig={name="connection_limit", value=3}}
1808duration = "1hr"
1809"#])
1810 .await;
1811
1812 k9::assert_equal!(
1813 shaping.get_errors(),
1814 ["SetProviderConfig is only valid within a provider block, but was used in domain 'example.com'".to_string()]
1815 .as_slice()
1816 );
1817 }
1818
1819 #[tokio::test]
1820 async fn test_set_provider_config_accepted_in_provider() {
1821 let shaping = make_shaping_configs(&[r#"
1822[provider."acme"]
1823match=[{DomainSuffix=".acme"}]
1824
1825[[provider."acme".automation]]
1826regex = "x"
1827action = {SetProviderConfig={name="connection_limit", value=3}}
1828duration = "1hr"
1829"#])
1830 .await;
1831
1832 k9::assert_equal!(shaping.get_errors(), ([] as [String; 0]).as_slice());
1833 }
1834
1835 #[tokio::test]
1836 async fn test_provider() {
1837 let shaping = make_shaping_configs(&[r#"
1838[provider."Office 365"]
1839match=[{MXSuffix=".olc.protection.outlook.com"},{DomainSuffix=".outlook.com"}]
1840enable_tls = "Required"
1841provider_connection_limit = 10
1842provider_max_message_rate = "120/s"
1843
1844[provider."Office 365".sources."new-source"]
1845provider_source_selection_rate = "500/d,max_burst=1"
1846 "#])
1847 .await;
1848
1849 let resolved = shaping
1850 .get_egress_path_config("outlook.com", "invalid.source", "invalid.site")
1851 .await
1852 .finish()
1853 .unwrap();
1854
1855 k9::assert_equal!(
1856 resolved.params.enable_tls,
1857 crate::egress_path::Tls::Required
1858 );
1859 k9::assert_equal!(resolved.params.provider_name.unwrap(), "Office 365");
1860
1861 k9::snapshot!(
1862 resolved.params.additional_connection_limits,
1863 r#"
1864{
1865 "shaping-provider-Office 365-invalid.source-limit": 10,
1866}
1867"#
1868 );
1869 k9::snapshot!(
1870 resolved.params.additional_message_rate_throttles,
1871 r#"
1872{
1873 "shaping-provider-Office 365-invalid.source-rate": 120/s,
1874}
1875"#
1876 );
1877 assert!(resolved.params.source_selection_rate.is_none());
1878 assert!(resolved.params.additional_source_selection_rates.is_empty());
1879
1880 let resolved = shaping
1881 .get_egress_path_config("outlook.com", "new-source", "invalid.site")
1882 .await
1883 .finish()
1884 .unwrap();
1885 assert!(resolved.params.source_selection_rate.is_none());
1886 k9::snapshot!(
1887 resolved.params.additional_source_selection_rates,
1888 r#"
1889{
1890 "shaping-provider-Office 365-new-source-selection-rate": 500/d,max_burst=1,
1891}
1892"#
1893 );
1894 }
1895
1896 #[tokio::test]
1897 async fn test_rule_matching() {
1898 let shaping = make_shaping_configs(&[r#"
1899[["default".automation]]
1900regex="default"
1901action = {SetConfig={name="connection_limit", value=1}}
1902duration = "1hr"
1903
1904["fake.site"]
1905_treat_domain_name_as_site_name = true
1906
1907[["fake.site".automation]]
1908regex="fake_rollup"
1909action = {SetConfig={name="connection_limit", value=2}}
1910duration = "1hr"
1911
1912["((a|b).x|c.y).targets.test"]
1913_treat_domain_name_as_site_name = true
1914connection_limit = 11
1915
1916[["((a|b).x|c.y).targets.test".automation]]
1917regex="nested_rollup"
1918action = {SetConfig={name="connection_limit", value=3}}
1919duration = "1hr"
1920
1921["((a|c).x|b.y).targets.test"]
1922_treat_domain_name_as_site_name = true
1923connection_limit = 22
1924
1925["woot.provider"]
1926mx_rollup = false
1927
1928[["woot.provider".automation]]
1929regex="woot_domain"
1930action = {SetConfig={name="connection_limit", value=2}}
1931duration = "1hr"
1932
1933[provider."provider"]
1934match=[{DomainSuffix=".provider"}]
1935
1936[[provider."provider".automation]]
1937regex="provider"
1938action = {SetConfig={name="connection_limit", value=3}}
1939duration = "1hr"
1940match_internal = true
1941
1942"#])
1943 .await;
1944
1945 eprintln!("{:?}", shaping.inner.warnings);
1946
1947 fn make_record(content: &str, recipient: &str, site: &str) -> JsonLogRecord {
1948 JsonLogRecord {
1949 kind: RecordType::TransientFailure,
1950 id: String::new(),
1951 sender: String::new(),
1952 recipient: vec![recipient.to_string()],
1953 queue: String::new(),
1954 site: site.to_string(),
1955 size: 0,
1956 response: Response {
1957 code: 400,
1958 command: None,
1959 enhanced_code: None,
1960 content: content.to_string(),
1961 },
1962 peer_address: None,
1963 timestamp: Default::default(),
1964 created: Default::default(),
1965 num_attempts: 1,
1966 bounce_classification: Default::default(),
1967 egress_pool: None,
1968 egress_source: None,
1969 source_address: None,
1970 feedback_report: None,
1971 meta: Default::default(),
1972 headers: Default::default(),
1973 delivery_protocol: None,
1974 reception_protocol: None,
1975 nodeid: Uuid::default(),
1976 tls_cipher: None,
1977 tls_protocol_version: None,
1978 tls_peer_subject_name: None,
1979 provider_name: None,
1980 session_id: None,
1981 }
1982 }
1983
1984 let matches = shaping
1985 .match_rules(&make_record("default", "user@example.com", "dummy_site"))
1986 .await
1987 .unwrap();
1988 k9::assert_equal!(
1989 matches[0].rule.regex[0].to_string(),
1990 "default",
1991 "matches against default automation rule"
1992 );
1993
1994 let matches = shaping
1995 .match_rules(&make_record(
1996 "KumoMTA internal: default",
1997 "user@example.com",
1998 "dummy_site",
1999 ))
2000 .await
2001 .unwrap();
2002 assert!(matches.is_empty(), "internal bounce should not match");
2003
2004 let matches = shaping
2005 .match_rules(&make_record(
2006 "woot_domain",
2007 "user@woot.provider",
2008 "dummy_site",
2009 ))
2010 .await
2011 .unwrap();
2012 k9::assert_equal!(
2013 matches[0].rule.regex[0].to_string(),
2014 "woot_domain",
2015 "matches against domain rule with mx_rollup=false"
2016 );
2017
2018 let matches = shaping
2019 .match_rules(&make_record("fake_rollup", "user@fake.rollup", "fake.site"))
2020 .await
2021 .unwrap();
2022 k9::assert_equal!(
2023 matches[0].rule.regex[0].to_string(),
2024 "fake_rollup",
2025 "matches against domain rule with mx_rollup=true"
2026 );
2027
2028 let nested_a = "((a|b).x|c.y).targets.test";
2029 let nested_b = "((a|c).x|b.y).targets.test";
2030 for (site, expected) in [(nested_a, 11), (nested_b, 22)] {
2031 let config = shaping
2032 .get_egress_path_config("example.test", "unspecified", site)
2033 .await
2034 .finish()
2035 .unwrap();
2036 assert_eq!(config.params.connection_limit.limit, expected);
2037 }
2038 let matches = shaping
2039 .match_rules(&make_record(
2040 "nested_rollup",
2041 "user@example.test",
2042 &format!("unspecified->{nested_a}@smtp_client"),
2043 ))
2044 .await
2045 .unwrap();
2046 assert_eq!(matches.len(), 1);
2047 assert_eq!(matches[0].rule.regex[0].to_string(), "nested_rollup");
2048 assert!(shaping
2049 .match_rules(&make_record(
2050 "nested_rollup",
2051 "user@example.test",
2052 &format!("unspecified->{nested_b}@smtp_client")
2053 ))
2054 .await
2055 .unwrap()
2056 .is_empty());
2057
2058 let matches = shaping
2059 .match_rules(&make_record("provider", "user@woot.provider", "dummy_site"))
2060 .await
2061 .unwrap();
2062 k9::assert_equal!(
2063 matches[0].rule.regex[0].to_string(),
2064 "provider",
2065 "matches against provider rule"
2066 );
2067 k9::assert_equal!(
2068 matches[0].scope,
2069 MatchScope::Provider("provider".to_string()),
2070 "provider rule's scope names the matched provider"
2071 );
2072
2073 let matches = shaping
2074 .match_rules(&make_record(
2075 "KumoMTA internal: provider",
2076 "user@woot.provider",
2077 "dummy_site",
2078 ))
2079 .await
2080 .unwrap();
2081 k9::assert_equal!(
2082 matches[0].rule.regex[0].to_string(),
2083 "provider",
2084 "internal response matches against provider rule"
2085 );
2086 }
2087
2088 #[tokio::test]
2089 async fn test_defaults() {
2090 let shaping = make_shaping_configs(&[r#"
2091["default"]
2092connection_limit = 10
2093max_connection_rate = "100/min"
2094max_deliveries_per_connection = 100
2095max_message_rate = "100/s"
2096idle_timeout = "60s"
2097data_timeout = "30s"
2098data_dot_timeout = "60s"
2099enable_tls = "Opportunistic"
2100consecutive_connection_failures_before_delay = 100
2101
2102[["default".automation]]
2103regex=[
2104 '/Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/',
2105 '/All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/',
2106 '/has been temporarily rate limited due to IP reputation/',
2107 '/Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/',
2108 '/Server busy\. Please try again later from/'
2109]
2110action = [
2111 {SetConfig={name="max_message_rate", value="1/minute"}},
2112 {SetConfig={name="connection_limit", value=1}}
2113]
2114duration = "90m"
2115
2116["gmail.com"]
2117max_deliveries_per_connection = 50
2118connection_limit = 5
2119enable_tls = "Required"
2120consecutive_connection_failures_before_delay = 5
2121
2122["yahoo.com"]
2123max_deliveries_per_connection = 20
2124
2125[["yahoo.com".automation]]
2126regex = "\\[TS04\\]"
2127action = "Suspend"
2128duration = "2 hours"
2129
2130["comcast.net"]
2131connection_limit = 25
2132max_deliveries_per_connection = 250
2133enable_tls = "Required"
2134idle_timeout = "30s"
2135consecutive_connection_failures_before_delay = 24
2136
2137["mail.com"]
2138max_deliveries_per_connection = 100
2139
2140["orange.fr"]
2141connection_limit = 3
2142
2143["smtp.mailgun.com"]
2144connection_limit = 7000
2145max_deliveries_per_connection = 3
2146
2147["example.com"]
2148mx_rollup = false
2149max_deliveries_per_connection = 100
2150connection_limit = 3
2151
2152["example.com".sources."my source name"]
2153connection_limit = 5
2154 "#])
2155 .await;
2156
2157 let default = shaping
2158 .get_egress_path_config("invalid.domain", "invalid.source", "invalid.site")
2159 .await
2160 .finish()
2161 .unwrap();
2162 k9::snapshot!(
2163 default,
2164 r#"
2165MergedEntry {
2166 params: EgressPathConfig {
2167 connection_limit: 10,
2168 additional_connection_limits: {},
2169 enable_tls: Opportunistic,
2170 tls_mode: StartTls,
2171 enable_mta_sts: true,
2172 enable_dane: false,
2173 enable_pipelining: true,
2174 enable_rset: true,
2175 tls_prefer_openssl: false,
2176 tls_certificate: None,
2177 tls_private_key: None,
2178 openssl_cipher_list: None,
2179 openssl_cipher_suites: None,
2180 openssl_options: None,
2181 rustls_cipher_suites: [],
2182 client_timeouts: SmtpClientTimeouts {
2183 connect_timeout: 60s,
2184 banner_timeout: 60s,
2185 ehlo_timeout: 300s,
2186 mail_from_timeout: 300s,
2187 rcpt_to_timeout: 300s,
2188 data_timeout: 30s,
2189 data_dot_timeout: 60s,
2190 rset_timeout: 5s,
2191 idle_timeout: 60s,
2192 starttls_timeout: 5s,
2193 auth_timeout: 60s,
2194 },
2195 system_shutdown_timeout: None,
2196 max_ready: 1024,
2197 consecutive_connection_failures_before_delay: 100,
2198 smtp_port: 25,
2199 smtp_auth_plain_username: None,
2200 smtp_auth_plain_password: None,
2201 allow_smtp_auth_plain_without_tls: false,
2202 allow_smtp_auth_plain_without_valid_certificate: false,
2203 max_message_rate: Some(
2204 100/s,
2205 ),
2206 additional_message_rate_throttles: {},
2207 source_selection_rate: None,
2208 additional_source_selection_rates: {},
2209 max_connection_rate: Some(
2210 100/m,
2211 ),
2212 max_deliveries_per_connection: 100,
2213 max_recipients_per_batch: 100,
2214 prohibited_hosts: {
2215 "0.0.0.0",
2216 "127.0.0.0/8",
2217 "::/127",
2218 },
2219 skip_hosts: {},
2220 ip_lookup_strategy: Ipv4AndIpv6,
2221 max_mx_plan_size: 50,
2222 max_mx_addresses_per_host: 10,
2223 ehlo_domain: None,
2224 aggressive_connection_opening: false,
2225 refresh_interval: 60s,
2226 refresh_strategy: Ttl,
2227 dispatcher_wakeup_strategy: Aggressive,
2228 maintainer_wakeup_strategy: Aggressive,
2229 provider_name: None,
2230 remember_broken_tls: None,
2231 opportunistic_tls_reconnect_on_failed_handshake: false,
2232 use_lmtp: false,
2233 reconnect_strategy: ConnectNextHost,
2234 readyq_pool_name: None,
2235 low_memory_reduction_policy: ShrinkDataAndMeta,
2236 no_memory_reduction_policy: ShrinkDataAndMeta,
2237 try_next_host_on_transport_error: false,
2238 ignore_8bit_checks: false,
2239 dispatcher_progress_watchdog_timeout: None,
2240 },
2241 sources: {},
2242 automation: [
2243 Rule {
2244 regex: [
2245 Regex(
2246 /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2247 ),
2248 Regex(
2249 /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2250 ),
2251 Regex(
2252 /has been temporarily rate limited due to IP reputation/,
2253 ),
2254 Regex(
2255 /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2256 ),
2257 Regex(
2258 /Server busy\. Please try again later from/,
2259 ),
2260 ],
2261 action: [
2262 SetConfig(
2263 EgressPathConfigValue {
2264 name: "max_message_rate",
2265 value: HashableTomlValue {
2266 value: String(
2267 "1/minute",
2268 ),
2269 },
2270 },
2271 ),
2272 SetConfig(
2273 EgressPathConfigValue {
2274 name: "connection_limit",
2275 value: HashableTomlValue {
2276 value: Integer(
2277 1,
2278 ),
2279 },
2280 },
2281 ),
2282 ],
2283 trigger: Immediate,
2284 duration: 5400s,
2285 match_internal: false,
2286 },
2287 ],
2288}
2289"#
2290 );
2291
2292 let example_com = shaping
2293 .get_egress_path_config("example.com", "invalid.source", "invalid.site")
2294 .await
2295 .finish()
2296 .unwrap();
2297 k9::snapshot!(
2298 example_com,
2299 r#"
2300MergedEntry {
2301 params: EgressPathConfig {
2302 connection_limit: 3,
2303 additional_connection_limits: {},
2304 enable_tls: Opportunistic,
2305 tls_mode: StartTls,
2306 enable_mta_sts: true,
2307 enable_dane: false,
2308 enable_pipelining: true,
2309 enable_rset: true,
2310 tls_prefer_openssl: false,
2311 tls_certificate: None,
2312 tls_private_key: None,
2313 openssl_cipher_list: None,
2314 openssl_cipher_suites: None,
2315 openssl_options: None,
2316 rustls_cipher_suites: [],
2317 client_timeouts: SmtpClientTimeouts {
2318 connect_timeout: 60s,
2319 banner_timeout: 60s,
2320 ehlo_timeout: 300s,
2321 mail_from_timeout: 300s,
2322 rcpt_to_timeout: 300s,
2323 data_timeout: 30s,
2324 data_dot_timeout: 60s,
2325 rset_timeout: 5s,
2326 idle_timeout: 60s,
2327 starttls_timeout: 5s,
2328 auth_timeout: 60s,
2329 },
2330 system_shutdown_timeout: None,
2331 max_ready: 1024,
2332 consecutive_connection_failures_before_delay: 100,
2333 smtp_port: 25,
2334 smtp_auth_plain_username: None,
2335 smtp_auth_plain_password: None,
2336 allow_smtp_auth_plain_without_tls: false,
2337 allow_smtp_auth_plain_without_valid_certificate: false,
2338 max_message_rate: Some(
2339 100/s,
2340 ),
2341 additional_message_rate_throttles: {},
2342 source_selection_rate: None,
2343 additional_source_selection_rates: {},
2344 max_connection_rate: Some(
2345 100/m,
2346 ),
2347 max_deliveries_per_connection: 100,
2348 max_recipients_per_batch: 100,
2349 prohibited_hosts: {
2350 "0.0.0.0",
2351 "127.0.0.0/8",
2352 "::/127",
2353 },
2354 skip_hosts: {},
2355 ip_lookup_strategy: Ipv4AndIpv6,
2356 max_mx_plan_size: 50,
2357 max_mx_addresses_per_host: 10,
2358 ehlo_domain: None,
2359 aggressive_connection_opening: false,
2360 refresh_interval: 60s,
2361 refresh_strategy: Ttl,
2362 dispatcher_wakeup_strategy: Aggressive,
2363 maintainer_wakeup_strategy: Aggressive,
2364 provider_name: None,
2365 remember_broken_tls: None,
2366 opportunistic_tls_reconnect_on_failed_handshake: false,
2367 use_lmtp: false,
2368 reconnect_strategy: ConnectNextHost,
2369 readyq_pool_name: None,
2370 low_memory_reduction_policy: ShrinkDataAndMeta,
2371 no_memory_reduction_policy: ShrinkDataAndMeta,
2372 try_next_host_on_transport_error: false,
2373 ignore_8bit_checks: false,
2374 dispatcher_progress_watchdog_timeout: None,
2375 },
2376 sources: {
2377 "my source name": EgressPathConfig {
2378 connection_limit: 5,
2379 additional_connection_limits: {},
2380 enable_tls: Opportunistic,
2381 tls_mode: StartTls,
2382 enable_mta_sts: true,
2383 enable_dane: false,
2384 enable_pipelining: true,
2385 enable_rset: true,
2386 tls_prefer_openssl: false,
2387 tls_certificate: None,
2388 tls_private_key: None,
2389 openssl_cipher_list: None,
2390 openssl_cipher_suites: None,
2391 openssl_options: None,
2392 rustls_cipher_suites: [],
2393 client_timeouts: SmtpClientTimeouts {
2394 connect_timeout: 60s,
2395 banner_timeout: 60s,
2396 ehlo_timeout: 300s,
2397 mail_from_timeout: 300s,
2398 rcpt_to_timeout: 300s,
2399 data_timeout: 300s,
2400 data_dot_timeout: 300s,
2401 rset_timeout: 5s,
2402 idle_timeout: 5s,
2403 starttls_timeout: 5s,
2404 auth_timeout: 60s,
2405 },
2406 system_shutdown_timeout: None,
2407 max_ready: 1024,
2408 consecutive_connection_failures_before_delay: 100,
2409 smtp_port: 25,
2410 smtp_auth_plain_username: None,
2411 smtp_auth_plain_password: None,
2412 allow_smtp_auth_plain_without_tls: false,
2413 allow_smtp_auth_plain_without_valid_certificate: false,
2414 max_message_rate: None,
2415 additional_message_rate_throttles: {},
2416 source_selection_rate: None,
2417 additional_source_selection_rates: {},
2418 max_connection_rate: None,
2419 max_deliveries_per_connection: 1024,
2420 max_recipients_per_batch: 100,
2421 prohibited_hosts: {
2422 "0.0.0.0",
2423 "127.0.0.0/8",
2424 "::/127",
2425 },
2426 skip_hosts: {},
2427 ip_lookup_strategy: Ipv4AndIpv6,
2428 max_mx_plan_size: 50,
2429 max_mx_addresses_per_host: 10,
2430 ehlo_domain: None,
2431 aggressive_connection_opening: false,
2432 refresh_interval: 60s,
2433 refresh_strategy: Ttl,
2434 dispatcher_wakeup_strategy: Aggressive,
2435 maintainer_wakeup_strategy: Aggressive,
2436 provider_name: None,
2437 remember_broken_tls: None,
2438 opportunistic_tls_reconnect_on_failed_handshake: false,
2439 use_lmtp: false,
2440 reconnect_strategy: ConnectNextHost,
2441 readyq_pool_name: None,
2442 low_memory_reduction_policy: ShrinkDataAndMeta,
2443 no_memory_reduction_policy: ShrinkDataAndMeta,
2444 try_next_host_on_transport_error: false,
2445 ignore_8bit_checks: false,
2446 dispatcher_progress_watchdog_timeout: None,
2447 },
2448 },
2449 automation: [
2450 Rule {
2451 regex: [
2452 Regex(
2453 /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2454 ),
2455 Regex(
2456 /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2457 ),
2458 Regex(
2459 /has been temporarily rate limited due to IP reputation/,
2460 ),
2461 Regex(
2462 /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2463 ),
2464 Regex(
2465 /Server busy\. Please try again later from/,
2466 ),
2467 ],
2468 action: [
2469 SetConfig(
2470 EgressPathConfigValue {
2471 name: "max_message_rate",
2472 value: HashableTomlValue {
2473 value: String(
2474 "1/minute",
2475 ),
2476 },
2477 },
2478 ),
2479 SetConfig(
2480 EgressPathConfigValue {
2481 name: "connection_limit",
2482 value: HashableTomlValue {
2483 value: Integer(
2484 1,
2485 ),
2486 },
2487 },
2488 ),
2489 ],
2490 trigger: Immediate,
2491 duration: 5400s,
2492 match_internal: false,
2493 },
2494 ],
2495}
2496"#
2497 );
2498
2499 let yahoo_com = shaping
2502 .get_egress_path_config(
2503 "yahoo.com",
2504 "invalid.source",
2505 "(mta5|mta6|mta7).am0.yahoodns.net",
2506 )
2507 .await
2508 .finish()
2509 .unwrap();
2510 k9::snapshot!(
2511 yahoo_com,
2512 r#"
2513MergedEntry {
2514 params: EgressPathConfig {
2515 connection_limit: 10,
2516 additional_connection_limits: {},
2517 enable_tls: Opportunistic,
2518 tls_mode: StartTls,
2519 enable_mta_sts: true,
2520 enable_dane: false,
2521 enable_pipelining: true,
2522 enable_rset: true,
2523 tls_prefer_openssl: false,
2524 tls_certificate: None,
2525 tls_private_key: None,
2526 openssl_cipher_list: None,
2527 openssl_cipher_suites: None,
2528 openssl_options: None,
2529 rustls_cipher_suites: [],
2530 client_timeouts: SmtpClientTimeouts {
2531 connect_timeout: 60s,
2532 banner_timeout: 60s,
2533 ehlo_timeout: 300s,
2534 mail_from_timeout: 300s,
2535 rcpt_to_timeout: 300s,
2536 data_timeout: 30s,
2537 data_dot_timeout: 60s,
2538 rset_timeout: 5s,
2539 idle_timeout: 60s,
2540 starttls_timeout: 5s,
2541 auth_timeout: 60s,
2542 },
2543 system_shutdown_timeout: None,
2544 max_ready: 1024,
2545 consecutive_connection_failures_before_delay: 100,
2546 smtp_port: 25,
2547 smtp_auth_plain_username: None,
2548 smtp_auth_plain_password: None,
2549 allow_smtp_auth_plain_without_tls: false,
2550 allow_smtp_auth_plain_without_valid_certificate: false,
2551 max_message_rate: Some(
2552 100/s,
2553 ),
2554 additional_message_rate_throttles: {},
2555 source_selection_rate: None,
2556 additional_source_selection_rates: {},
2557 max_connection_rate: Some(
2558 100/m,
2559 ),
2560 max_deliveries_per_connection: 20,
2561 max_recipients_per_batch: 100,
2562 prohibited_hosts: {
2563 "0.0.0.0",
2564 "127.0.0.0/8",
2565 "::/127",
2566 },
2567 skip_hosts: {},
2568 ip_lookup_strategy: Ipv4AndIpv6,
2569 max_mx_plan_size: 50,
2570 max_mx_addresses_per_host: 10,
2571 ehlo_domain: None,
2572 aggressive_connection_opening: false,
2573 refresh_interval: 60s,
2574 refresh_strategy: Ttl,
2575 dispatcher_wakeup_strategy: Aggressive,
2576 maintainer_wakeup_strategy: Aggressive,
2577 provider_name: None,
2578 remember_broken_tls: None,
2579 opportunistic_tls_reconnect_on_failed_handshake: false,
2580 use_lmtp: false,
2581 reconnect_strategy: ConnectNextHost,
2582 readyq_pool_name: None,
2583 low_memory_reduction_policy: ShrinkDataAndMeta,
2584 no_memory_reduction_policy: ShrinkDataAndMeta,
2585 try_next_host_on_transport_error: false,
2586 ignore_8bit_checks: false,
2587 dispatcher_progress_watchdog_timeout: None,
2588 },
2589 sources: {},
2590 automation: [
2591 Rule {
2592 regex: [
2593 Regex(
2594 /Messages from \d+\.\d+\.\d+\.\d+ temporarily deferred/,
2595 ),
2596 Regex(
2597 /All messages from \d+\.\d+\.\d+\.\d+ will be permanently deferred/,
2598 ),
2599 Regex(
2600 /has been temporarily rate limited due to IP reputation/,
2601 ),
2602 Regex(
2603 /Unfortunately, messages from \d+\.\d+\.\d+\.\d+ weren.t sent/,
2604 ),
2605 Regex(
2606 /Server busy\. Please try again later from/,
2607 ),
2608 ],
2609 action: [
2610 SetConfig(
2611 EgressPathConfigValue {
2612 name: "max_message_rate",
2613 value: HashableTomlValue {
2614 value: String(
2615 "1/minute",
2616 ),
2617 },
2618 },
2619 ),
2620 SetConfig(
2621 EgressPathConfigValue {
2622 name: "connection_limit",
2623 value: HashableTomlValue {
2624 value: Integer(
2625 1,
2626 ),
2627 },
2628 },
2629 ),
2630 ],
2631 trigger: Immediate,
2632 duration: 5400s,
2633 match_internal: false,
2634 },
2635 Rule {
2636 regex: [
2637 Regex(
2638 \[TS04\],
2639 ),
2640 ],
2641 action: [
2642 Suspend,
2643 ],
2644 trigger: Immediate,
2645 duration: 7200s,
2646 match_internal: false,
2647 },
2648 ],
2649}
2650"#
2651 );
2652 }
2653
2654 #[tokio::test]
2655 async fn test_load_default_shaping_toml() {
2656 Shaping::merge_files(
2657 &["../../assets/policy-extras/shaping.toml".into()],
2658 &ShapingMergeOptions::default(),
2659 )
2660 .await
2661 .unwrap();
2662 }
2663}