kumo_log_types/
rfc5965.rs

1//! ARF reports
2use crate::rfc3464::{content_type, RemoteMta};
3use anyhow::anyhow;
4use bstr::{BStr, BString, ByteSlice};
5use chrono::{DateTime, Utc};
6use mailparsing::{BStringUtf8, Header, HeaderParseResult, MimePart};
7use rfc5321::parser::EnvelopeAddress;
8use serde::{Deserialize, Serialize};
9use serde_with::serde_as;
10use std::collections::BTreeMap;
11use std::str::FromStr;
12
13#[serde_as]
14#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
15pub struct ARFReport {
16    pub feedback_type: String,
17    pub user_agent: String,
18    pub version: String,
19
20    #[serde(default)]
21    pub arrival_date: Option<DateTime<Utc>>,
22    #[serde(default)]
23    pub incidents: Option<u32>,
24    #[serde(default)]
25    pub original_envelope_id: Option<String>,
26    #[serde(default)]
27    pub original_mail_from: Option<String>,
28    #[serde(default)]
29    pub reporting_mta: Option<RemoteMta>,
30    #[serde(default)]
31    pub source_ip: Option<String>,
32
33    #[serde(default)]
34    pub authentication_results: Vec<String>,
35    #[serde(default)]
36    pub original_rcpto_to: Vec<String>,
37    #[serde(default)]
38    pub reported_domain: Vec<String>,
39    #[serde(default)]
40    pub reported_uri: Vec<String>,
41
42    #[serde_as(as = "BTreeMap<_, Vec<BStringUtf8>>")]
43    pub extensions: BTreeMap<String, Vec<BString>>,
44
45    #[serde_as(as = "Option<BStringUtf8>")]
46    pub original_message: Option<BString>,
47    pub supplemental_trace: Option<serde_json::Value>,
48}
49
50impl ARFReport {
51    pub fn parse(input: &[u8]) -> anyhow::Result<Option<Self>> {
52        let mail = MimePart::parse(input)?;
53        let ct = mail.headers().content_type()?;
54        let ct = match ct {
55            None => return Ok(None),
56            Some(ct) => ct,
57        };
58
59        if ct.value != "multipart/report" {
60            return Ok(None);
61        }
62
63        if ct.get("report-type").as_ref().map(|b| b.as_bstr()) != Some(BStr::new("feedback-report"))
64        {
65            return Ok(None);
66        }
67
68        let mut original_message = None;
69        let mut supplemental_trace = None;
70
71        for part in mail.child_parts() {
72            let ct = content_type(part);
73            let ct = ct.as_ref().map(|b| b.as_bstr());
74            if ct == Some(BStr::new("message/rfc822"))
75                || ct == Some(BStr::new("text/rfc822-headers"))
76            {
77                if let Ok(HeaderParseResult { headers, .. }) =
78                    Header::parse_headers(part.raw_body())
79                {
80                    // Look for x-headers that might be our supplemental trace headers
81                    for hdr in headers.iter() {
82                        if !(hdr.get_name().starts_with_str("X-")
83                            || hdr.get_name().starts_with_str("x-"))
84                        {
85                            continue;
86                        }
87                        // The header value may be folded across continuation
88                        // lines; the folding whitespace is not part of the
89                        // base64 payload, so remove it before decoding.
90                        let encoded: Vec<u8> = hdr
91                            .get_raw_value()
92                            .iter()
93                            .copied()
94                            .filter(|b| !b.is_ascii_whitespace())
95                            .collect();
96                        if let Ok(decoded) = data_encoding::BASE64.decode(&encoded) {
97                            #[derive(Deserialize)]
98                            struct Wrap {
99                                #[serde(rename = "_@_")]
100                                marker: String,
101                                #[serde(flatten)]
102                                payload: serde_json::Value,
103                            }
104                            if let Ok(obj) = serde_json::from_slice::<Wrap>(&decoded) {
105                                // Sanity check that it is our encoded data, rather than
106                                // some other random header that may have been inserted
107                                // somewhere along the way
108                                if obj.marker == "\\_/" {
109                                    supplemental_trace.replace(obj.payload);
110                                    break;
111                                }
112                            }
113                        }
114                    }
115                }
116
117                original_message = Some(BString::new(
118                    part.raw_body().as_bytes().replace("\r\n", "\n"),
119                ));
120            }
121        }
122
123        for part in mail.child_parts() {
124            let ct = content_type(part);
125            let ct = ct.as_ref().map(|b| b.as_bstr());
126            if ct == Some(BStr::new("message/feedback-report")) {
127                return Ok(Some(Self::parse_inner(
128                    part,
129                    original_message,
130                    supplemental_trace,
131                )?));
132            }
133        }
134
135        anyhow::bail!("feedback-report part missing");
136    }
137
138    fn parse_inner(
139        part: &MimePart,
140        original_message: Option<BString>,
141        supplemental_trace: Option<serde_json::Value>,
142    ) -> anyhow::Result<Self> {
143        let body = part.raw_body();
144        let mut extensions = extract_headers(body.as_bytes())?;
145
146        let feedback_type = extract_single_req("feedback-type", &mut extensions)?;
147        let user_agent = extract_single_req("user-agent", &mut extensions)?;
148        let version = extract_single_req("version", &mut extensions)?;
149        let arrival_date = extract_single_conv_fallback::<DateTimeRfc2822, DateTime<Utc>>(
150            "arrival-date",
151            "received-date",
152            &mut extensions,
153        );
154        let incidents = extract_single("incidents", &mut extensions)?;
155        let original_envelope_id = extract_single("original-envelope-id", &mut extensions)?;
156        let original_mail_from =
157            extract_single::<EnvelopeAddress>("original-mail-from", &mut extensions)?
158                .map(|a| a.to_string());
159        let reporting_mta = extract_single("reporting-mta", &mut extensions)?;
160        let source_ip = extract_single("source-ip", &mut extensions)?;
161        let authentication_results = extract_multiple("authentication-results", &mut extensions)?;
162        let original_rcpto_to =
163            extract_multiple::<EnvelopeAddress>("original-rcpt-to", &mut extensions)?
164                .into_iter()
165                .map(|a| a.to_string())
166                .collect();
167        let reported_domain = extract_multiple("reported-domain", &mut extensions)?;
168        let reported_uri = extract_multiple("reported-uri", &mut extensions)?;
169
170        Ok(Self {
171            feedback_type,
172            user_agent,
173            version,
174            arrival_date,
175            incidents,
176            original_envelope_id,
177            original_mail_from,
178            reporting_mta,
179            source_ip,
180            authentication_results,
181            original_rcpto_to,
182            reported_domain,
183            reported_uri,
184            extensions,
185            original_message,
186            supplemental_trace,
187        })
188    }
189}
190
191pub(crate) fn extract_headers(part: &[u8]) -> anyhow::Result<BTreeMap<String, Vec<BString>>> {
192    let HeaderParseResult { headers, .. } = Header::parse_headers(part)?;
193
194    let mut extensions = BTreeMap::new();
195
196    for hdr in headers.iter() {
197        let name = String::from_utf8_lossy(&hdr.get_name()).to_ascii_lowercase();
198        extensions
199            .entry(name)
200            .or_insert_with(std::vec::Vec::new)
201            .push(hdr.as_unstructured()?);
202    }
203    Ok(extensions)
204}
205
206#[derive(Debug)]
207pub(crate) struct DateTimeRfc2822(pub DateTime<Utc>);
208
209impl FromStr for DateTimeRfc2822 {
210    type Err = anyhow::Error;
211    fn from_str(input: &str) -> anyhow::Result<Self> {
212        let date = mailparsing::parse_rfc2822_date(input)?;
213        Ok(Self(date.into()))
214    }
215}
216
217impl From<DateTimeRfc2822> for DateTime<Utc> {
218    fn from(val: DateTimeRfc2822) -> Self {
219        val.0
220    }
221}
222
223pub(crate) fn extract_single_req<R>(
224    name: &str,
225    extensions: &mut BTreeMap<String, Vec<BString>>,
226) -> anyhow::Result<R>
227where
228    R: FromStr,
229    <R as FromStr>::Err: std::fmt::Display,
230{
231    extract_single(name, extensions)?
232        .ok_or_else(|| anyhow!("required header {name} is not present"))
233}
234
235pub(crate) fn extract_single<R>(
236    name: &str,
237    extensions: &mut BTreeMap<String, Vec<BString>>,
238) -> anyhow::Result<Option<R>>
239where
240    R: FromStr,
241    <R as FromStr>::Err: std::fmt::Display,
242{
243    match extensions.remove(name) {
244        Some(mut hdrs) if hdrs.len() == 1 => {
245            let value = hdrs.remove(0);
246            let value = value
247                .to_str()
248                .map_err(|err| anyhow!("{value} could not be converted to UTF-8: {err:#}"))?;
249            let converted = value
250                .parse::<R>()
251                .map_err(|err| anyhow!("failed to convert '{value}': {err:#}"))?;
252            Ok(Some(converted))
253        }
254        Some(_) => anyhow::bail!("header {name} should have only a single value"),
255        None => Ok(None),
256    }
257}
258
259pub(crate) fn extract_single_conv<R, T>(
260    name: &str,
261    extensions: &mut BTreeMap<String, Vec<BString>>,
262) -> anyhow::Result<Option<T>>
263where
264    R: FromStr,
265    <R as FromStr>::Err: std::fmt::Display,
266    R: Into<T>,
267{
268    Ok(extract_single::<R>(name, extensions)?.map(|v| v.into()))
269}
270
271pub(crate) fn extract_single_conv_fallback<R, T>(
272    name: &str,
273    fallback: &str,
274    extensions: &mut BTreeMap<String, Vec<BString>>,
275) -> Option<T>
276where
277    R: FromStr,
278    <R as FromStr>::Err: std::fmt::Display,
279    R: Into<T>,
280{
281    let maybe = extract_single::<R>(name, extensions).ok()?;
282    match maybe {
283        Some(value) => Some(value.into()),
284        None => extract_single::<R>(fallback, extensions)
285            .ok()?
286            .map(Into::into),
287    }
288}
289
290pub(crate) fn extract_multiple<R>(
291    name: &str,
292    extensions: &mut BTreeMap<String, Vec<BString>>,
293) -> anyhow::Result<Vec<R>>
294where
295    R: FromStr,
296    <R as FromStr>::Err: std::fmt::Display,
297{
298    match extensions.remove(name) {
299        Some(hdrs) => {
300            let mut results = vec![];
301            for h in hdrs {
302                let value = h
303                    .to_str()
304                    .map_err(|err| anyhow!("{h} could not be converted to UTF-8: {err:#}"))?;
305                let converted = value
306                    .parse::<R>()
307                    .map_err(|err| anyhow!("failed to convert {h}: {err:#}"))?;
308                results.push(converted);
309            }
310            Ok(results)
311        }
312        None => Ok(vec![]),
313    }
314}
315
316#[cfg(test)]
317mod test {
318    use super::*;
319    use chrono::TimeZone;
320
321    #[test]
322    fn datetime_rfc2822_obsolete_utc_zone() {
323        // Amazon SES emits dates with the non-standard alphabetic zone "UTC".
324        let expected = Utc.with_ymd_and_hms(2026, 7, 2, 18, 55, 38).unwrap();
325
326        let parsed: DateTime<Utc> = "Thu, 02 Jul 26 18:55:38 UTC"
327            .parse::<DateTimeRfc2822>()
328            .expect("SES obsolete UTC zone should parse")
329            .into();
330        k9::assert_equal!(parsed, expected);
331
332        // The canonical numeric form parses to the same instant.
333        let canonical: DateTime<Utc> = "Thu, 02 Jul 2026 18:55:38 +0000"
334            .parse::<DateTimeRfc2822>()
335            .unwrap()
336            .into();
337        k9::assert_equal!(canonical, expected);
338
339        // A genuinely unparseable value still errors.
340        "not a date".parse::<DateTimeRfc2822>().unwrap_err();
341    }
342
343    #[test]
344    fn rfc5965_1() {
345        let result = ARFReport::parse(include_bytes!("../data/rfc5965/1.eml")).unwrap();
346        k9::snapshot!(
347            result,
348            r#"
349Some(
350    ARFReport {
351        feedback_type: "abuse",
352        user_agent: "SomeGenerator/1.0",
353        version: "1",
354        arrival_date: None,
355        incidents: None,
356        original_envelope_id: None,
357        original_mail_from: None,
358        reporting_mta: None,
359        source_ip: None,
360        authentication_results: [],
361        original_rcpto_to: [],
362        reported_domain: [],
363        reported_uri: [],
364        extensions: {},
365        original_message: Some(
366            "Received: from mailserver.example.net
367    (mailserver.example.net [192.0.2.1])
368    by example.com with ESMTP id M63d4137594e46;
369    Thu, 08 Mar 2005 14:00:00 -0400
370From: <somespammer@example.net>
371To: <Undisclosed Recipients>
372Subject: Earn money
373MIME-Version: 1.0
374Content-type: text/plain
375Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net
376Date: Thu, 02 Sep 2004 12:31:03 -0500
377
378Spam Spam Spam
379Spam Spam Spam
380Spam Spam Spam
381Spam Spam Spam
382",
383        ),
384        supplemental_trace: None,
385    },
386)
387"#
388        );
389    }
390
391    #[test]
392    fn rfc5965_2() {
393        let result = ARFReport::parse(include_bytes!("../data/rfc5965/2.eml")).unwrap();
394        k9::snapshot!(
395            result,
396            r#"
397Some(
398    ARFReport {
399        feedback_type: "abuse",
400        user_agent: "SomeGenerator/1.0",
401        version: "1",
402        arrival_date: None,
403        incidents: None,
404        original_envelope_id: None,
405        original_mail_from: Some(
406            "somespammer@example.net",
407        ),
408        reporting_mta: Some(
409            RemoteMta {
410                mta_type: "dns",
411                name: "mail.example.com",
412            },
413        ),
414        source_ip: Some(
415            "192.0.2.1",
416        ),
417        authentication_results: [
418            "mail.example.com; spf=fail smtp.mail=somespammer@example.com",
419        ],
420        original_rcpto_to: [
421            "user@example.com",
422        ],
423        reported_domain: [
424            "example.net",
425        ],
426        reported_uri: [
427            "http://example.net/earn_money.html",
428            "mailto:user@example.com",
429        ],
430        extensions: {
431            "removal-recipient": [
432                "user@example.com",
433            ],
434        },
435        original_message: Some(
436            "From: <somespammer@example.net>
437Received: from mailserver.example.net (mailserver.example.net
438    [192.0.2.1]) by example.com with ESMTP id M63d4137594e46;
439    Tue, 08 Mar 2005 14:00:00 -0400
440X-KumoRef: eyJfQF8iOiJcXF8vIiwicmVjaXBpZW50IjoidGVzdEBleGFtcGxlLmNvbSJ9
441To: <Undisclosed Recipients>
442Subject: Earn money
443MIME-Version: 1.0
444Content-type: text/plain
445Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net
446Date: Thu, 02 Sep 2004 12:31:03 -0500
447
448Spam Spam Spam
449Spam Spam Spam
450Spam Spam Spam
451Spam Spam Spam
452",
453        ),
454        supplemental_trace: Some(
455            Object {
456                "recipient": String("test@example.com"),
457            },
458        ),
459    },
460)
461"#
462        );
463    }
464
465    /// A supplemental trace header folded across continuation lines still
466    /// decodes back to its metadata.
467    #[test]
468    fn supplemental_trace_folded_header() {
469        let report = concat!(
470            "Content-Type: multipart/report; report-type=feedback-report;\r\n",
471            "    boundary=\"b\"\r\n",
472            "\r\n",
473            "--b\r\n",
474            "Content-Type: message/feedback-report\r\n",
475            "\r\n",
476            "Feedback-Type: abuse\r\n",
477            "User-Agent: SomeGenerator/1.0\r\n",
478            "Version: 1\r\n",
479            "\r\n",
480            "--b\r\n",
481            "Content-Type: message/rfc822\r\n",
482            "\r\n",
483            "From: <somespammer@example.net>\r\n",
484            // The base64 value is split with a CRLF + TAB fold.
485            "X-KumoRef: eyJfQF8iOiJcXF8vIiwicmVjaXBpZW50Ijoi\r\n",
486            "\tdGVzdEBleGFtcGxlLmNvbSJ9\r\n",
487            "Subject: Earn money\r\n",
488            "\r\n",
489            "Spam\r\n",
490            "--b--\r\n",
491        );
492
493        let result = ARFReport::parse(report.as_bytes()).unwrap().unwrap();
494        k9::assert_equal!(
495            result.supplemental_trace,
496            Some(serde_json::json!({ "recipient": "test@example.com" }))
497        );
498    }
499
500    #[test]
501    fn rfc5965_3() {
502        let result = ARFReport::parse(include_bytes!("../data/rfc5965/3.eml")).unwrap();
503        k9::snapshot!(
504            result,
505            r#"
506Some(
507    ARFReport {
508        feedback_type: "abuse",
509        user_agent: "Yahoo!-Mail-Feedback/2.0",
510        version: "0.1",
511        arrival_date: Some(
512            2023-12-14T16:16:15Z,
513        ),
514        incidents: None,
515        original_envelope_id: None,
516        original_mail_from: Some(
517            "test1@example.com",
518        ),
519        reporting_mta: None,
520        source_ip: None,
521        authentication_results: [
522            "authentication result string is not available",
523        ],
524        original_rcpto_to: [
525            "user@example.com",
526        ],
527        reported_domain: [
528            "bounce.kumo.example.com",
529        ],
530        reported_uri: [],
531        extensions: {},
532        original_message: Some(
533            "Date: Thu, 14 Dec 2023 16:16:14 +0000
534To: user@example.com
535Subject: test Thu, 14 Dec 2023 16:16:14 +0000
536
537This is a test mailing
538
539",
540        ),
541        supplemental_trace: None,
542    },
543)
544"#
545        );
546    }
547
548    #[test]
549    fn rfc5965_4() {
550        let result = ARFReport::parse(include_bytes!("../data/rfc5965/4.eml")).unwrap();
551        k9::snapshot!(
552            result,
553            r#"
554Some(
555    ARFReport {
556        feedback_type: "abuse",
557        user_agent: "ReturnPathFBL/2.0",
558        version: "1",
559        arrival_date: Some(
560            2023-12-13T19:03:30Z,
561        ),
562        incidents: None,
563        original_envelope_id: None,
564        original_mail_from: Some(
565            "foo@bounce.example.com",
566        ),
567        reporting_mta: None,
568        source_ip: Some(
569            "x.x.x.x",
570        ),
571        authentication_results: [],
572        original_rcpto_to: [
573            "cb4a01a48251d4765f489076aa81e2a4@comcast.net",
574        ],
575        reported_domain: [
576            "bounce.example.com",
577        ],
578        reported_uri: [],
579        extensions: {
580            "abuse-type": [
581                "complaint",
582            ],
583            "source": [
584                "Comcast",
585            ],
586            "subscription-link": [
587                "https://fbl.returnpath.net/manage/subscriptions/xxxx",
588            ],
589        },
590        original_message: Some(
591            "Date: Thu, 14 Dec 2023 16:16:14 +0000
592To: user@example.com
593Subject: test Thu, 14 Dec 2023 16:16:14 +0000
594
595This is a test mailing
596
597",
598        ),
599        supplemental_trace: None,
600    },
601)
602"#
603        );
604    }
605}
kumo_log_types/rfc5965.rs

kumo_log_types/
rfc5965.rs
