kumo_log_types/
rfc5965.rs

1//! ARF reports
2use crate::rfc3464::{content_type, decode_returned_content, RemoteMta};
3use anyhow::anyhow;
4use bstr::{BStr, BString, ByteSlice};
5use chrono::{DateTime, Utc};
6use mailparsing::{BStringUtf8, Header, HeaderParseResult, MimePart};
7use rfc5321::parser::EnvelopeAddress;
8use serde::{Deserialize, Serialize};
9use serde_with::serde_as;
10use std::collections::BTreeMap;
11use std::str::FromStr;
12
13#[serde_as]
14#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
15pub struct ARFReport {
16    pub feedback_type: String,
17    pub user_agent: String,
18    pub version: String,
19
20    #[serde(default)]
21    pub arrival_date: Option<DateTime<Utc>>,
22    #[serde(default)]
23    pub incidents: Option<u32>,
24    #[serde(default)]
25    pub original_envelope_id: Option<String>,
26    #[serde(default)]
27    pub original_mail_from: Option<String>,
28    #[serde(default)]
29    pub reporting_mta: Option<RemoteMta>,
30    #[serde(default)]
31    pub source_ip: Option<String>,
32
33    #[serde(default)]
34    pub authentication_results: Vec<String>,
35    #[serde(default)]
36    pub original_rcpto_to: Vec<String>,
37    #[serde(default)]
38    pub reported_domain: Vec<String>,
39    #[serde(default)]
40    pub reported_uri: Vec<String>,
41
42    #[serde_as(as = "BTreeMap<_, Vec<BStringUtf8>>")]
43    pub extensions: BTreeMap<String, Vec<BString>>,
44
45    #[serde_as(as = "Option<BStringUtf8>")]
46    pub original_message: Option<BString>,
47    pub supplemental_trace: Option<serde_json::Value>,
48}
49
50impl ARFReport {
51    pub fn parse(input: &[u8]) -> anyhow::Result<Option<Self>> {
52        let mail = MimePart::parse(input)?;
53        let ct = mail.headers().content_type()?;
54        let ct = match ct {
55            None => return Ok(None),
56            Some(ct) => ct,
57        };
58
59        if ct.value != "multipart/report" {
60            return Ok(None);
61        }
62
63        if ct.get("report-type").as_ref().map(|b| b.as_bstr()) != Some(BStr::new("feedback-report"))
64        {
65            return Ok(None);
66        }
67
68        let mut original_message = None;
69        let mut supplemental_trace = None;
70
71        for part in mail.child_parts() {
72            let ct = content_type(part);
73            let ct = ct.as_ref().map(|b| b.as_bstr());
74            // A returned-content part may be quoted-printable or base64
75            // encoded. Decode before inspecting or storing it, and skip parts
76            // that are not returned content or that cannot be decoded.
77            let Some(returned) = decode_returned_content(part, ct) else {
78                continue;
79            };
80
81            if let Ok(HeaderParseResult { headers, .. }) =
82                Header::parse_headers(returned.as_slice())
83            {
84                // Look for x-headers that might be our supplemental trace headers
85                for hdr in headers.iter() {
86                    if !(hdr.get_name().starts_with_str("X-")
87                        || hdr.get_name().starts_with_str("x-"))
88                    {
89                        continue;
90                    }
91                    // The header value may be folded across continuation lines.
92                    // Remove the folding whitespace before decoding because it
93                    // is not part of the base64 payload.
94                    let encoded: Vec<u8> = hdr
95                        .get_raw_value()
96                        .iter()
97                        .copied()
98                        .filter(|b| !b.is_ascii_whitespace())
99                        .collect();
100                    if let Ok(decoded) = data_encoding::BASE64.decode(&encoded) {
101                        #[derive(Deserialize)]
102                        struct Wrap {
103                            #[serde(rename = "_@_")]
104                            marker: String,
105                            #[serde(flatten)]
106                            payload: serde_json::Value,
107                        }
108                        if let Ok(obj) = serde_json::from_slice::<Wrap>(&decoded) {
109                            // Sanity check that it is our encoded data, rather
110                            // than some other random header that may have been
111                            // inserted earlier in the stream
112                            if obj.marker == "\\_/" {
113                                supplemental_trace.replace(obj.payload);
114                                break;
115                            }
116                        }
117                    }
118                }
119            }
120
121            original_message = Some(BString::new(returned.replace(b"\r\n", b"\n")));
122        }
123
124        for part in mail.child_parts() {
125            let ct = content_type(part);
126            let ct = ct.as_ref().map(|b| b.as_bstr());
127            if ct == Some(BStr::new("message/feedback-report")) {
128                return Ok(Some(Self::parse_inner(
129                    part,
130                    original_message,
131                    supplemental_trace,
132                )?));
133            }
134        }
135
136        anyhow::bail!("feedback-report part missing");
137    }
138
139    fn parse_inner(
140        part: &MimePart,
141        original_message: Option<BString>,
142        supplemental_trace: Option<serde_json::Value>,
143    ) -> anyhow::Result<Self> {
144        let body = part.raw_body();
145        let mut extensions = extract_headers(body.as_bytes())?;
146
147        let feedback_type = extract_single_req("feedback-type", &mut extensions)?;
148        let user_agent = extract_single_req("user-agent", &mut extensions)?;
149        let version = extract_single_req("version", &mut extensions)?;
150        let arrival_date = extract_single_conv_fallback::<DateTimeRfc2822, DateTime<Utc>>(
151            "arrival-date",
152            "received-date",
153            &mut extensions,
154        );
155        let incidents = extract_single("incidents", &mut extensions)?;
156        let original_envelope_id = extract_single("original-envelope-id", &mut extensions)?;
157        let original_mail_from =
158            extract_single::<EnvelopeAddress>("original-mail-from", &mut extensions)?
159                .map(|a| a.to_string());
160        let reporting_mta = extract_single("reporting-mta", &mut extensions)?;
161        let source_ip = extract_single("source-ip", &mut extensions)?;
162        let authentication_results = extract_multiple("authentication-results", &mut extensions)?;
163        let original_rcpto_to =
164            extract_multiple::<EnvelopeAddress>("original-rcpt-to", &mut extensions)?
165                .into_iter()
166                .map(|a| a.to_string())
167                .collect();
168        let reported_domain = extract_multiple("reported-domain", &mut extensions)?;
169        let reported_uri = extract_multiple("reported-uri", &mut extensions)?;
170
171        Ok(Self {
172            feedback_type,
173            user_agent,
174            version,
175            arrival_date,
176            incidents,
177            original_envelope_id,
178            original_mail_from,
179            reporting_mta,
180            source_ip,
181            authentication_results,
182            original_rcpto_to,
183            reported_domain,
184            reported_uri,
185            extensions,
186            original_message,
187            supplemental_trace,
188        })
189    }
190}
191
192pub(crate) fn extract_headers(part: &[u8]) -> anyhow::Result<BTreeMap<String, Vec<BString>>> {
193    let HeaderParseResult { headers, .. } = Header::parse_headers(part)?;
194
195    let mut extensions = BTreeMap::new();
196
197    for hdr in headers.iter() {
198        let name = String::from_utf8_lossy(&hdr.get_name()).to_ascii_lowercase();
199        extensions
200            .entry(name)
201            .or_insert_with(std::vec::Vec::new)
202            .push(hdr.as_unstructured()?);
203    }
204    Ok(extensions)
205}
206
207#[derive(Debug)]
208pub(crate) struct DateTimeRfc2822(pub DateTime<Utc>);
209
210impl FromStr for DateTimeRfc2822 {
211    type Err = anyhow::Error;
212    fn from_str(input: &str) -> anyhow::Result<Self> {
213        let date = mailparsing::parse_rfc2822_date(input)?;
214        Ok(Self(date.into()))
215    }
216}
217
218impl From<DateTimeRfc2822> for DateTime<Utc> {
219    fn from(val: DateTimeRfc2822) -> Self {
220        val.0
221    }
222}
223
224pub(crate) fn extract_single_req<R>(
225    name: &str,
226    extensions: &mut BTreeMap<String, Vec<BString>>,
227) -> anyhow::Result<R>
228where
229    R: FromStr,
230    <R as FromStr>::Err: std::fmt::Display,
231{
232    extract_single(name, extensions)?
233        .ok_or_else(|| anyhow!("required header {name} is not present"))
234}
235
236pub(crate) fn extract_single<R>(
237    name: &str,
238    extensions: &mut BTreeMap<String, Vec<BString>>,
239) -> anyhow::Result<Option<R>>
240where
241    R: FromStr,
242    <R as FromStr>::Err: std::fmt::Display,
243{
244    match extensions.remove(name) {
245        Some(mut hdrs) if hdrs.len() == 1 => {
246            let value = hdrs.remove(0);
247            let value = value
248                .to_str()
249                .map_err(|err| anyhow!("{value} could not be converted to UTF-8: {err:#}"))?;
250            let converted = value
251                .parse::<R>()
252                .map_err(|err| anyhow!("failed to convert '{value}': {err:#}"))?;
253            Ok(Some(converted))
254        }
255        Some(_) => anyhow::bail!("header {name} should have only a single value"),
256        None => Ok(None),
257    }
258}
259
260pub(crate) fn extract_single_conv<R, T>(
261    name: &str,
262    extensions: &mut BTreeMap<String, Vec<BString>>,
263) -> anyhow::Result<Option<T>>
264where
265    R: FromStr,
266    <R as FromStr>::Err: std::fmt::Display,
267    R: Into<T>,
268{
269    Ok(extract_single::<R>(name, extensions)?.map(|v| v.into()))
270}
271
272pub(crate) fn extract_single_conv_fallback<R, T>(
273    name: &str,
274    fallback: &str,
275    extensions: &mut BTreeMap<String, Vec<BString>>,
276) -> Option<T>
277where
278    R: FromStr,
279    <R as FromStr>::Err: std::fmt::Display,
280    R: Into<T>,
281{
282    let maybe = extract_single::<R>(name, extensions).ok()?;
283    match maybe {
284        Some(value) => Some(value.into()),
285        None => extract_single::<R>(fallback, extensions)
286            .ok()?
287            .map(Into::into),
288    }
289}
290
291pub(crate) fn extract_multiple<R>(
292    name: &str,
293    extensions: &mut BTreeMap<String, Vec<BString>>,
294) -> anyhow::Result<Vec<R>>
295where
296    R: FromStr,
297    <R as FromStr>::Err: std::fmt::Display,
298{
299    match extensions.remove(name) {
300        Some(hdrs) => {
301            let mut results = vec![];
302            for h in hdrs {
303                let value = h
304                    .to_str()
305                    .map_err(|err| anyhow!("{h} could not be converted to UTF-8: {err:#}"))?;
306                let converted = value
307                    .parse::<R>()
308                    .map_err(|err| anyhow!("failed to convert {h}: {err:#}"))?;
309                results.push(converted);
310            }
311            Ok(results)
312        }
313        None => Ok(vec![]),
314    }
315}
316
317#[cfg(test)]
318mod test {
319    use super::*;
320    use chrono::TimeZone;
321
322    #[test]
323    fn datetime_rfc2822_obsolete_utc_zone() {
324        // Amazon SES emits dates with the non-standard alphabetic zone "UTC".
325        let expected = Utc.with_ymd_and_hms(2026, 7, 2, 18, 55, 38).unwrap();
326
327        let parsed: DateTime<Utc> = "Thu, 02 Jul 26 18:55:38 UTC"
328            .parse::<DateTimeRfc2822>()
329            .expect("SES obsolete UTC zone should parse")
330            .into();
331        k9::assert_equal!(parsed, expected);
332
333        // The canonical numeric form parses to the same instant.
334        let canonical: DateTime<Utc> = "Thu, 02 Jul 2026 18:55:38 +0000"
335            .parse::<DateTimeRfc2822>()
336            .unwrap()
337            .into();
338        k9::assert_equal!(canonical, expected);
339
340        // A genuinely unparseable value still errors.
341        "not a date".parse::<DateTimeRfc2822>().unwrap_err();
342    }
343
344    #[test]
345    fn rfc5965_1() {
346        let result = ARFReport::parse(include_bytes!("../data/rfc5965/1.eml")).unwrap();
347        k9::snapshot!(
348            result,
349            r#"
350Some(
351    ARFReport {
352        feedback_type: "abuse",
353        user_agent: "SomeGenerator/1.0",
354        version: "1",
355        arrival_date: None,
356        incidents: None,
357        original_envelope_id: None,
358        original_mail_from: None,
359        reporting_mta: None,
360        source_ip: None,
361        authentication_results: [],
362        original_rcpto_to: [],
363        reported_domain: [],
364        reported_uri: [],
365        extensions: {},
366        original_message: Some(
367            "Received: from mailserver.example.net
368    (mailserver.example.net [192.0.2.1])
369    by example.com with ESMTP id M63d4137594e46;
370    Thu, 08 Mar 2005 14:00:00 -0400
371From: <somespammer@example.net>
372To: <Undisclosed Recipients>
373Subject: Earn money
374MIME-Version: 1.0
375Content-type: text/plain
376Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net
377Date: Thu, 02 Sep 2004 12:31:03 -0500
378
379Spam Spam Spam
380Spam Spam Spam
381Spam Spam Spam
382Spam Spam Spam
383",
384        ),
385        supplemental_trace: None,
386    },
387)
388"#
389        );
390    }
391
392    #[test]
393    fn rfc5965_2() {
394        let result = ARFReport::parse(include_bytes!("../data/rfc5965/2.eml")).unwrap();
395        k9::snapshot!(
396            result,
397            r#"
398Some(
399    ARFReport {
400        feedback_type: "abuse",
401        user_agent: "SomeGenerator/1.0",
402        version: "1",
403        arrival_date: None,
404        incidents: None,
405        original_envelope_id: None,
406        original_mail_from: Some(
407            "somespammer@example.net",
408        ),
409        reporting_mta: Some(
410            RemoteMta {
411                mta_type: "dns",
412                name: "mail.example.com",
413            },
414        ),
415        source_ip: Some(
416            "192.0.2.1",
417        ),
418        authentication_results: [
419            "mail.example.com; spf=fail smtp.mail=somespammer@example.com",
420        ],
421        original_rcpto_to: [
422            "user@example.com",
423        ],
424        reported_domain: [
425            "example.net",
426        ],
427        reported_uri: [
428            "http://example.net/earn_money.html",
429            "mailto:user@example.com",
430        ],
431        extensions: {
432            "removal-recipient": [
433                "user@example.com",
434            ],
435        },
436        original_message: Some(
437            "From: <somespammer@example.net>
438Received: from mailserver.example.net (mailserver.example.net
439    [192.0.2.1]) by example.com with ESMTP id M63d4137594e46;
440    Tue, 08 Mar 2005 14:00:00 -0400
441X-KumoRef: eyJfQF8iOiJcXF8vIiwicmVjaXBpZW50IjoidGVzdEBleGFtcGxlLmNvbSJ9
442To: <Undisclosed Recipients>
443Subject: Earn money
444MIME-Version: 1.0
445Content-type: text/plain
446Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net
447Date: Thu, 02 Sep 2004 12:31:03 -0500
448
449Spam Spam Spam
450Spam Spam Spam
451Spam Spam Spam
452Spam Spam Spam
453",
454        ),
455        supplemental_trace: Some(
456            Object {
457                "recipient": String("test@example.com"),
458            },
459        ),
460    },
461)
462"#
463        );
464    }
465
466    /// A supplemental trace header folded across continuation lines still
467    /// decodes back to its metadata.
468    #[test]
469    fn supplemental_trace_folded_header() {
470        let report = concat!(
471            "Content-Type: multipart/report; report-type=feedback-report;\r\n",
472            "    boundary=\"b\"\r\n",
473            "\r\n",
474            "--b\r\n",
475            "Content-Type: message/feedback-report\r\n",
476            "\r\n",
477            "Feedback-Type: abuse\r\n",
478            "User-Agent: SomeGenerator/1.0\r\n",
479            "Version: 1\r\n",
480            "\r\n",
481            "--b\r\n",
482            "Content-Type: message/rfc822\r\n",
483            "\r\n",
484            "From: <somespammer@example.net>\r\n",
485            // The base64 value is split with a CRLF + TAB fold.
486            "X-KumoRef: eyJfQF8iOiJcXF8vIiwicmVjaXBpZW50Ijoi\r\n",
487            "\tdGVzdEBleGFtcGxlLmNvbSJ9\r\n",
488            "Subject: Earn money\r\n",
489            "\r\n",
490            "Spam\r\n",
491            "--b--\r\n",
492        );
493
494        let result = ARFReport::parse(report.as_bytes()).unwrap().unwrap();
495        k9::assert_equal!(
496            result.supplemental_trace,
497            Some(serde_json::json!({ "recipient": "test@example.com" }))
498        );
499    }
500
501    #[test]
502    fn quoted_printable_rfc822_headers_are_decoded() {
503        // A quoted-printable text/rfc822-headers part must be transfer decoded
504        // while preserving its raw octets, here a Subject holding shift_jis bytes.
505        let report = concat!(
506            "Content-Type: multipart/report; report-type=feedback-report;\r\n",
507            "    boundary=\"b\"\r\n",
508            "\r\n",
509            "--b\r\n",
510            "Content-Type: message/feedback-report\r\n",
511            "\r\n",
512            "Feedback-Type: abuse\r\n",
513            "User-Agent: SomeGenerator/1.0\r\n",
514            "Version: 1\r\n",
515            "\r\n",
516            "--b\r\n",
517            "Content-Type: text/rfc822-headers; charset=\"shift_jis\"\r\n",
518            "Content-Transfer-Encoding: quoted-printable\r\n",
519            "\r\n",
520            "Subject: =93=FA=96=7B=8C=EA\r\n",
521            "--b--\r\n",
522        );
523
524        let result = ARFReport::parse(report.as_bytes()).unwrap().unwrap();
525        k9::assert_equal!(
526            result.original_message.unwrap(),
527            BString::from(b"Subject: \x93\xFA\x96\x7B\x8C\xEA\n")
528        );
529    }
530
531    #[test]
532    fn rfc5965_3() {
533        let result = ARFReport::parse(include_bytes!("../data/rfc5965/3.eml")).unwrap();
534        k9::snapshot!(
535            result,
536            r#"
537Some(
538    ARFReport {
539        feedback_type: "abuse",
540        user_agent: "Yahoo!-Mail-Feedback/2.0",
541        version: "0.1",
542        arrival_date: Some(
543            2023-12-14T16:16:15Z,
544        ),
545        incidents: None,
546        original_envelope_id: None,
547        original_mail_from: Some(
548            "test1@example.com",
549        ),
550        reporting_mta: None,
551        source_ip: None,
552        authentication_results: [
553            "authentication result string is not available",
554        ],
555        original_rcpto_to: [
556            "user@example.com",
557        ],
558        reported_domain: [
559            "bounce.kumo.example.com",
560        ],
561        reported_uri: [],
562        extensions: {},
563        original_message: Some(
564            "Date: Thu, 14 Dec 2023 16:16:14 +0000
565To: user@example.com
566Subject: test Thu, 14 Dec 2023 16:16:14 +0000
567
568This is a test mailing
569
570",
571        ),
572        supplemental_trace: None,
573    },
574)
575"#
576        );
577    }
578
579    #[test]
580    fn rfc5965_4() {
581        let result = ARFReport::parse(include_bytes!("../data/rfc5965/4.eml")).unwrap();
582        k9::snapshot!(
583            result,
584            r#"
585Some(
586    ARFReport {
587        feedback_type: "abuse",
588        user_agent: "ReturnPathFBL/2.0",
589        version: "1",
590        arrival_date: Some(
591            2023-12-13T19:03:30Z,
592        ),
593        incidents: None,
594        original_envelope_id: None,
595        original_mail_from: Some(
596            "foo@bounce.example.com",
597        ),
598        reporting_mta: None,
599        source_ip: Some(
600            "x.x.x.x",
601        ),
602        authentication_results: [],
603        original_rcpto_to: [
604            "cb4a01a48251d4765f489076aa81e2a4@comcast.net",
605        ],
606        reported_domain: [
607            "bounce.example.com",
608        ],
609        reported_uri: [],
610        extensions: {
611            "abuse-type": [
612                "complaint",
613            ],
614            "source": [
615                "Comcast",
616            ],
617            "subscription-link": [
618                "https://fbl.returnpath.net/manage/subscriptions/xxxx",
619            ],
620        },
621        original_message: Some(
622            "Date: Thu, 14 Dec 2023 16:16:14 +0000
623To: user@example.com
624Subject: test Thu, 14 Dec 2023 16:16:14 +0000
625
626This is a test mailing
627
628",
629        ),
630        supplemental_trace: None,
631    },
632)
633"#
634        );
635    }
636}
kumo_log_types/rfc5965.rs

kumo_log_types/
rfc5965.rs
