kumo_prometheus/
lib.rs

1pub use crate::counter::*;
2use crate::labels::MetricLabel;
3use crate::registry::StreamingCollector;
4use async_stream::stream;
5use futures::stream::BoxStream;
6use futures::StreamExt;
7use parking_lot::{RwLock, RwLockUpgradableReadGuard};
8pub use pastey as paste;
9use serde::Serialize;
10use std::borrow::Borrow;
11use std::collections::HashMap;
12use std::hash::Hash;
13use std::sync::{Arc, LazyLock};
14
15mod counter;
16pub mod counter_bundle;
17
18#[macro_use]
19pub mod labels;
20pub mod parser;
21pub mod registry;
22pub use prometheus;
23
24struct CounterRegistryInner<K, V: AtomicCounterEntry> {
25    map: RwLock<HashMap<K, V>>,
26    name: &'static str,
27    help: String,
28    metric_type: MetricType,
29}
30
31/// Keep up to 4k at a time of pending text or json data
32/// when streaming out the serialized counter values
33const CHUNK_SIZE: usize = 4 * 1024;
34
35impl<K: Clone + MetricLabel + Send + Sync, V: AtomicCounterEntry> StreamingCollector
36    for CounterRegistryInner<K, V>
37{
38    fn stream_text(&'_ self, prefix: &Option<String>) -> BoxStream<'_, String> {
39        /*
40        # HELP tokio_total_overflow_count The number of times worker threads saturated their local queues.
41        # TYPE tokio_total_overflow_count counter
42        tokio_total_overflow_count 0
43        total_connection_count{service="smtp_client:source2->loopback.dummy-mx.wezfurlong.org@smtp_client"} 25
44        */
45
46        let mut buffer = String::with_capacity(CHUNK_SIZE);
47        buffer.push_str("# HELP ");
48        let prefix = prefix.as_deref().unwrap_or("").to_string();
49        buffer.push_str(&prefix);
50        buffer.push_str(self.name);
51        buffer.push(' ');
52        buffer.push_str(&self.help);
53        buffer.push_str("\n# TYPE ");
54        buffer.push_str(&prefix);
55        buffer.push_str(self.name);
56        buffer.push(' ');
57        buffer.push_str(self.metric_type.label());
58        buffer.push('\n');
59
60        let mut buffer = Some(buffer);
61
62        let counters = {
63            let map = self.map.read();
64            let mut pairs = Vec::with_capacity(map.len());
65            for (key, weak) in map.iter() {
66                if let Some(strong) = weak.resolve() {
67                    pairs.push((key.clone(), strong));
68                }
69            }
70            pairs
71        };
72
73        stream! {
74            for (key, counter) in counters {
75                let Some(buf) = buffer.as_mut() else {break;};
76
77                buf.push_str(&prefix);
78                buf.push_str(self.name);
79                key.emit_text_value(buf, &counter.get().to_string());
80                buf.push('\n');
81
82                let need_flush = buf.len() >= CHUNK_SIZE;
83
84                if need_flush {
85                    yield buffer.take().expect("always have buffer");
86                    buffer.replace(String::with_capacity(CHUNK_SIZE));
87                }
88            }
89
90            if let Some(buf) = buffer.take() {
91                if !buf.is_empty(){
92                    yield buf;
93                }
94            }
95
96        }
97        .boxed()
98    }
99
100    fn stream_json(&'_ self) -> BoxStream<'_, String> {
101        let mut target = String::with_capacity(CHUNK_SIZE);
102        target.push_str(",\n\"");
103        target.push_str(self.name);
104        target.push_str("\":{");
105        if !self.help.is_empty() {
106            target.push_str("\"help\":\"");
107            target.push_str(&self.help);
108            target.push_str("\",");
109        }
110        target.push_str("\"type\":\"");
111        target.push_str(self.metric_type.label());
112        target.push_str("\",\"value\":");
113
114        let counters = {
115            let map = self.map.read();
116            let mut pairs = Vec::with_capacity(map.len());
117            for (key, weak) in map.iter() {
118                if let Some(strong) = weak.resolve() {
119                    pairs.push((key.clone(), strong));
120                }
121            }
122            pairs
123        };
124
125        stream! {
126            if counters.is_empty() {
127                target.push_str("null}");
128                yield target;
129                return;
130            }
131
132            let labels = K::label_names();
133
134            if labels.len() == 1 {
135                target.push_str("{\"");
136                target.push_str(labels[0]);
137                target.push_str("\":{");
138            } else {
139                target.push('[');
140            }
141
142            let mut buffer = Some(target);
143
144            for (i, (key, counter)) in counters.iter().enumerate() {
145                let Some(target) = buffer.as_mut() else {break;};
146                if i > 0 {
147                    target.push_str(",\n");
148                }
149
150                let value = counter.get().to_string();
151                key.emit_json_value(target,&value);
152
153                let need_flush = target.len() >= CHUNK_SIZE;
154
155                if need_flush {
156                    yield buffer.take().expect("always have buffer");
157                    buffer.replace(String::with_capacity(CHUNK_SIZE));
158                }
159            }
160
161            let Some(mut target) = buffer.take() else {return;};
162            if labels.len() == 1 {
163                target.push_str("}}}");
164            } else {
165                target.push_str("]}");
166            }
167
168            yield target;
169        }
170        .boxed()
171    }
172
173    fn prune(&self) {
174        if !V::needs_pruning() {
175            return;
176        }
177
178        let mut map = self.map.write();
179        map.retain(|_key, entry| entry.resolve().is_some());
180    }
181}
182
183/// Either a Counter or Gauge with a specific name, where there can
184/// be multiple labelled counter instances.
185///
186/// CounterRegistry has a PruningCounterRegistry variant which will
187/// drop unreferenced counter instances when they fall out of scope.
188///
189/// The key type K must be created via the label_key! macro provided
190/// by this crate. It allows making type-safe keys and resolving
191/// counter instances without making extraneous copies of the keys.
192///
193/// CounterRegistry implements the StreamingCollector trait which
194/// allows for efficient streaming serialization of its set of
195/// counters in either text or json format.
196pub struct CounterRegistry<K, V: AtomicCounterEntry = AtomicCounter> {
197    inner: Arc<CounterRegistryInner<K, V>>,
198}
199
200impl<K, V: AtomicCounterEntry> CounterRegistry<K, V> {
201    pub fn metric_type(&self) -> MetricType {
202        self.inner.metric_type
203    }
204}
205
206pub type PruningCounterRegistry<K> = CounterRegistry<K, WeakAtomicCounter>;
207
208#[derive(Serialize, Clone, Copy)]
209pub enum MetricType {
210    Counter,
211    Gauge,
212    Histogram,
213}
214
215impl MetricType {
216    pub fn label(&self) -> &'static str {
217        match self {
218            Self::Counter => "counter",
219            Self::Gauge => "gauge",
220            Self::Histogram => "histogram",
221        }
222    }
223}
224
225#[derive(Serialize, Clone, Copy)]
226pub enum MetricPrune {
227    Pruning,
228    NonPruning,
229}
230
231#[derive(Serialize, Clone)]
232pub struct CounterDescription {
233    /// The name of the counter, as it appears in the metric export
234    pub name: String,
235    /// Help description, preserving the author's line wrapping.
236    pub help: String,
237    /// Any documentation paragraphs following the first, preserved verbatim.
238    pub doc: Option<String>,
239    /// What sort of metric this is
240    pub metric_type: MetricType,
241    /// If the metric has labels, this lists them out
242    pub label_names: Vec<String>,
243    /// If the metric is a histogram, this holds the bucket thresholds
244    pub buckets: Vec<f64>,
245    /// True if the metric is subject to pruning
246    pub pruning: MetricPrune,
247}
248
249/// Accumulates metric metadata
250#[linkme::distributed_slice]
251pub static COUNTER_METADATA: [fn() -> CounterDescription];
252
253fn compute_metadata() -> Vec<CounterDescription> {
254    let mut metadata = vec![];
255
256    for func in COUNTER_METADATA.iter() {
257        let desc = (func)();
258        metadata.push(desc);
259    }
260
261    metadata.sort_by(|a, b| a.name.cmp(&b.name));
262    metadata
263}
264
265static METADATA: LazyLock<Vec<CounterDescription>> = LazyLock::new(compute_metadata);
266
267pub fn export_metadata() -> Vec<CounterDescription> {
268    // We're generally called early in a process lifecycle, and Registry
269    // requires a running tokio environment otherwise it will panic,
270    // so we make a little one just for this call.
271    let rt = tokio::runtime::Builder::new_current_thread()
272        .build()
273        .expect("failed to make single thread runtime for export_metadata call");
274    let _guard = rt.enter();
275    METADATA.clone()
276}
277
278/// This macro matches a series of doc comment attributes.
279/// Multi-line doc comments appear as a sequence of doc
280/// comment attributes, so we need to be able to match
281/// both the individual case and the sequence, and map
282/// them back to a single string.
283///
284/// While this macro accepts the no-doc-comment case,
285/// we require that every metric have a doc comment,
286/// so we will emit a compile error if none were present.
287#[macro_export]
288macro_rules! mandatory_doc {
289    ($doc:expr) => {
290        $doc
291    };
292    ($($doc:expr)+) => {
293        // Join the sequence into a multi-line string
294        concat!($($doc, "\n",)+)
295    };
296    () => {
297        compile_error!("doc comments are mandatory")
298    };
299}
300
301/// Collapse a multi-line string into a space-separated line.
302pub fn one_line(s: &str) -> String {
303    s.replace("\n", " ").trim().to_string()
304}
305
306/// Splits doc comment metadata. Look for two successive line breaks. If they
307/// are present, they denote the break between the first paragraph and a longer
308/// descriptive exposition. Returns that first paragraph, preserving the
309/// author's line wrapping, and the optional exposition.
310pub fn split_help(help: &str) -> (String, Option<String>) {
311    // The input will be a series of lines each with a space
312    // at the start because "/// something" -> " something".
313    // Normalize those away.
314    let normalized = help.trim().replace("\n ", "\n");
315
316    match normalized.split_once("\n\n") {
317        Some((a, b)) => (a.to_string(), Some(b.to_string())),
318        None => (normalized, None),
319    }
320}
321
322#[doc(hidden)]
323#[macro_export]
324macro_rules! __histogram_buckets {
325    () => {
326        $crate::prometheus::DEFAULT_BUCKETS.to_vec()
327    };
328    ($buckets:expr) => {
329        $buckets
330    };
331}
332
333#[doc(hidden)]
334#[macro_export]
335macro_rules! __register_metric {
336    (
337        $sym:ident,
338        $name:expr,
339        $($doc:expr)+,
340        $metric:expr,
341        $labels:expr,
342        $pruning:expr
343    ) => {
344        $crate::__register_metric!($sym, $name, $($doc)+, $metric, $labels, $pruning, vec![]);
345    };
346
347    (
348        $sym:ident,
349        $name:expr,
350        $($doc:expr)+,
351        $metric:expr,
352        $labels:expr,
353        $pruning:expr,
354        $buckets:expr
355    ) => {
356        // Link into COUNTER_METADATA
357        $crate::paste::paste! {
358            #[linkme::distributed_slice($crate::COUNTER_METADATA)]
359            static [<VIVIFY_ $sym>]: fn() -> $crate::CounterDescription = || {
360                ::std::sync::LazyLock::force(&$sym);
361                let (help, doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
362                let labels : &[&str] = $labels;
363                $crate::CounterDescription {
364                    name: $name.to_string(),
365                    help,
366                    doc,
367                    metric_type: $metric,
368                    label_names: labels.iter().map(|s| s.to_string()).collect(),
369                    buckets: $buckets,
370                    pruning: $pruning,
371                }
372            };
373        }
374    }
375}
376
377/// This macro aids in declaring metrics.  Usage looks like:
378///
379/// ```rust
380/// declare_metric! {
381/// /// The number of active outgoing connections in the system,
382/// /// keyed by the service name.
383/// pub static CONN_GAUGE: PruningGaugeRegistry<ServiceKey>("connection_count");
384/// }
385/// ```
386///
387/// This will wrap the declaration of the global into a LazyLock as
388/// well as capture metadata about the metric in a way that allows
389/// it to be retrieved via the `export_metadata()` function.
390///
391/// Doc comments are required for every metric declared by this
392/// macro.
393///
394/// Different types of metric collector are supported, not just
395/// the `PruningGaugeRegistry` shown above.
396///
397/// `PruningGaugeRegistry` is not actually a real type, it is
398/// some sugar allowed here to enable setting up a Gauge
399/// rather than a Counter.
400#[macro_export]
401macro_rules! declare_metric {
402    (
403        $(#[doc = $doc:expr])*
404        $vis:vis
405        static $sym:ident:
406        CounterRegistry<$key:ty>(
407            $name:expr $(,)?
408        );
409    ) => {
410        $(#[doc = $doc])*
411        $vis static $sym: ::std::sync::LazyLock<$crate::CounterRegistry<$key>> =
412            ::std::sync::LazyLock::new(
413                || {
414                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
415                    let help = $crate::one_line(&help);
416                    $crate::CounterRegistry::register($name, help)
417                });
418
419        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Counter,
420            <$key as $crate::labels::MetricLabel>::label_names(),
421            $crate::MetricPrune::NonPruning
422        );
423    };
424
425    (
426        $(#[doc = $doc:expr])*
427        $vis:vis
428        static $sym:ident:
429        PruningCounterRegistry<$key:ty>(
430            $name:expr $(,)?
431        );
432    ) => {
433        $(#[doc = $doc])*
434        $vis static $sym: ::std::sync::LazyLock<$crate::PruningCounterRegistry<$key>> =
435            ::std::sync::LazyLock::new(
436                || {
437                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
438                    let help = $crate::one_line(&help);
439                    $crate::PruningCounterRegistry::register($name, help)
440                });
441
442        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Counter,
443            <$key as $crate::labels::MetricLabel>::label_names(),
444            $crate::MetricPrune::Pruning
445        );
446    };
447
448    (
449        $(#[doc = $doc:expr])*
450        $vis:vis
451        static $sym:ident:
452        PruningGaugeRegistry<$key:ty>(
453            $name:expr $(,)?
454        );
455    ) => {
456        $(#[doc = $doc])*
457        $vis static $sym: ::std::sync::LazyLock<$crate::PruningCounterRegistry<$key>> =
458            ::std::sync::LazyLock::new(
459                || {
460                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
461                    let help = $crate::one_line(&help);
462                    $crate::PruningCounterRegistry::register_gauge($name, help)
463                });
464
465        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Gauge,
466            <$key as $crate::labels::MetricLabel>::label_names(),
467            $crate::MetricPrune::Pruning
468        );
469    };
470
471
472    (
473        $(#[doc = $doc:expr])*
474        $vis:vis
475        static $sym:ident:
476        IntGaugeVec(
477            $name:expr,
478            $labels:expr $(,)*
479        );
480    ) => {
481        $(#[doc = $doc])*
482        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::IntGaugeVec> =
483            ::std::sync::LazyLock::new(
484                || {
485                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
486                    let help = $crate::one_line(&help);
487
488                    $crate::prometheus::register_int_gauge_vec!(
489                        $name,
490                        help,
491                        $labels
492                    ).unwrap()
493                });
494
495        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Gauge,
496            $labels, $crate::MetricPrune::NonPruning);
497    };
498    (
499        $(#[doc = $doc:expr])*
500        $vis:vis
501        static $sym:ident:
502        IntCounterVec(
503            $name:expr,
504            $labels:expr $(,)*
505        );
506    ) => {
507        $(#[doc = $doc])*
508        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::IntCounterVec> =
509            ::std::sync::LazyLock::new(
510                || {
511                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
512                    let help = $crate::one_line(&help);
513
514                    $crate::prometheus::register_int_counter_vec!(
515                        $name,
516                        help,
517                        $labels
518                    ).unwrap()
519                });
520
521        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Counter,
522            &$labels[..], $crate::MetricPrune::NonPruning);
523    };
524    (
525        $(#[doc = $doc:expr])*
526        $vis:vis
527        static $sym:ident:
528        CounterVec(
529            $name:expr,
530            $labels:expr $(,)*
531        );
532    ) => {
533        $(#[doc = $doc])*
534        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::CounterVec> =
535            ::std::sync::LazyLock::new(
536                || {
537                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
538                    let help = $crate::one_line(&help);
539
540                    $crate::prometheus::register_counter_vec!(
541                        $name,
542                        help,
543                        $labels
544                    ).unwrap()
545                });
546
547        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Counter,
548            &$labels[..], $crate::MetricPrune::NonPruning);
549    };
550    (
551        $(#[doc = $doc:expr])*
552        $vis:vis
553        static $sym:ident:
554        HistogramVec(
555            $name:expr,
556            $labels:expr
557            $(, $buckets:expr)?
558            $(,)*
559        );
560    ) => {
561        $(#[doc = $doc])*
562        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::HistogramVec> =
563            ::std::sync::LazyLock::new(
564                || {
565                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
566                    let help = $crate::one_line(&help);
567
568                    $crate::prometheus::register_histogram_vec!(
569                        $name,
570                        help,
571                        $labels
572                        $(,$buckets)?
573                    ).unwrap()
574                });
575
576        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Histogram,
577            &$labels[..],
578            $crate::MetricPrune::NonPruning,
579            $crate::__histogram_buckets!($($buckets)?)
580        );
581    };
582
583    (
584        $(#[doc = $doc:expr])*
585        $vis:vis
586        static $sym:ident:
587        Histogram($name:expr $(, $buckets:expr)?);
588    ) => {
589        $(#[doc = $doc])*
590        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::Histogram> =
591            ::std::sync::LazyLock::new(
592                || {
593                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
594                    let help = $crate::one_line(&help);
595
596                    $crate::prometheus::register_histogram!(
597                        $name,
598                        help,
599                        $($buckets)?
600                    ).unwrap()
601                });
602
603        $crate::__register_metric!($sym, $name, $($doc)*, $crate::MetricType::Histogram, &[],
604            $crate::MetricPrune::NonPruning,
605            $crate::__histogram_buckets!($($buckets)?)
606        );
607    };
608
609    (
610        $(#[doc = $doc:expr])*
611        $vis:vis
612        static $sym:ident:
613        IntCounter($name:expr);
614    ) => {
615        $(#[doc = $doc])*
616        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::IntCounter> =
617            ::std::sync::LazyLock::new(
618                || {
619                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
620                    let help = $crate::one_line(&help);
621
622                    $crate::prometheus::register_int_counter!(
623                        $name,
624                        help,
625                    ).unwrap()
626                });
627
628        $crate::__register_metric!($sym, $name, $($doc)*,
629            $crate::MetricType::Counter, &[], $crate::MetricPrune::NonPruning);
630    };
631
632    (
633        $(#[doc = $doc:expr])*
634        $vis:vis
635        static $sym:ident:
636        IntGauge($name:expr);
637    ) => {
638        $(#[doc = $doc])*
639        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::IntGauge> =
640            ::std::sync::LazyLock::new(
641                || {
642                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
643                    let help = $crate::one_line(&help);
644
645                    $crate::prometheus::register_int_gauge!(
646                        $name,
647                        help,
648                    ).unwrap()
649                });
650
651        $crate::__register_metric!($sym, $name, $($doc)*,
652            $crate::MetricType::Gauge, &[], $crate::MetricPrune::NonPruning);
653    };
654    (
655        $(#[doc = $doc:expr])*
656        $vis:vis
657        static $sym:ident:
658        Gauge($name:expr);
659    ) => {
660        $(#[doc = $doc])*
661        $vis static $sym: ::std::sync::LazyLock<$crate::prometheus::Gauge> =
662            ::std::sync::LazyLock::new(
663                || {
664                    let (help, _doc) = $crate::split_help($crate::mandatory_doc!($($doc)*));
665                    let help = $crate::one_line(&help);
666
667                    $crate::prometheus::register_gauge!(
668                        $name,
669                        help,
670                    ).unwrap()
671                });
672
673        $crate::__register_metric!($sym, $name, $($doc)*,
674            $crate::MetricType::Gauge, &[], $crate::MetricPrune::NonPruning);
675    };
676}
677
678impl<K, V: AtomicCounterEntry> Clone for CounterRegistry<K, V> {
679    fn clone(&self) -> Self {
680        Self {
681            inner: Arc::clone(&self.inner),
682        }
683    }
684}
685
686impl<K: Clone + Send + Sync + MetricLabel + 'static, V: AtomicCounterEntry + 'static>
687    CounterRegistry<K, V>
688{
689    /// Register a set of Counters, values that are only allowed
690    /// to increment.
691    pub fn register(name: &'static str, help: String) -> Self {
692        Self::register_impl(name, help, MetricType::Counter)
693    }
694
695    /// Register a set of Gauges, values that are allowed to increase and decrease.
696    pub fn register_gauge(name: &'static str, help: String) -> Self {
697        Self::register_impl(name, help, MetricType::Gauge)
698    }
699
700    fn register_impl(name: &'static str, help: String, metric_type: MetricType) -> Self {
701        let me = Self {
702            inner: Arc::new(CounterRegistryInner {
703                map: Default::default(),
704                name,
705                help,
706                metric_type,
707            }),
708        };
709
710        crate::registry::Registry::register(me.inner.clone());
711
712        me
713    }
714}
715
716impl<K, V> CounterRegistry<K, V>
717where
718    V: AtomicCounterEntry,
719    K: Eq + Hash + MetricLabel,
720{
721    /// Resolve an already-existing counter for the given key, or None
722    /// if there either has never been such a value, or if it was pruned.
723    pub fn get<Q: ?Sized>(&self, key: &Q) -> Option<AtomicCounter>
724    where
725        K: Borrow<Q>,
726        Q: Hash + Eq,
727    {
728        let map = self.inner.map.read();
729        map.get(key).and_then(|weak| weak.resolve())
730    }
731
732    /// Resolve an already-existing counter for the given key, creating
733    /// a new one if it didn't already exist, or was previously pruned.
734    pub fn get_or_create<'a, Q: ?Sized>(&self, key: &'a Q) -> AtomicCounter
735    where
736        K: Borrow<Q> + From<&'a Q>,
737        Q: Hash + Eq,
738    {
739        let map = self.inner.map.upgradable_read();
740        if let Some(weak) = map.get(key) {
741            if let Some(strong) = weak.resolve() {
742                return strong;
743            }
744        }
745
746        let mut map = RwLockUpgradableReadGuard::upgrade(map);
747
748        // Check again, as we may have lost a race
749        if let Some(weak) = map.get(key) {
750            if let Some(strong) = weak.resolve() {
751                return strong;
752            }
753        }
754
755        let result = AtomicCounter::new();
756        map.insert(key.into(), V::make_storable(&result));
757
758        result
759    }
760}
761
762#[cfg(test)]
763mod tests {
764    use super::*;
765
766    #[test]
767    fn split_help_single_line() {
768        let (help, doc) = split_help(" A single line of help.");
769        k9::assert_equal!(help, "A single line of help.");
770        k9::assert_equal!(doc, None);
771    }
772
773    #[test]
774    fn split_help_multi_line_first_paragraph() {
775        let (help, doc) = split_help(" First line,\n second line.");
776        k9::assert_equal!(help, "First line,\nsecond line.");
777        k9::assert_equal!(doc, None);
778        k9::assert_equal!(one_line(&help), "First line, second line.");
779    }
780
781    #[test]
782    fn split_help_with_exposition() {
783        let (help, doc) = split_help(
784            " First line,\n second line.\n\n Exposition line one.\n Exposition line two.",
785        );
786        k9::assert_equal!(help, "First line,\nsecond line.");
787        k9::assert_equal!(
788            doc,
789            Some("Exposition line one.\nExposition line two.".to_string())
790        );
791    }
792
793    #[test]
794    fn split_help_preserves_internal_blank_lines_in_doc() {
795        let (help, doc) = split_help(" Help.\n\n Para one.\n\n Para two.");
796        k9::assert_equal!(help, "Help.");
797        k9::assert_equal!(doc, Some("Para one.\n\nPara two.".to_string()));
798    }
799}