mod_serde/
lib.rs

1use anyhow::Context;
2use config::{
3    any_err, get_or_create_module, get_or_create_sub_module, materialize_to_lua_value,
4    serialize_options,
5};
6use mlua::{Lua, LuaSerdeExt, Value as LuaValue};
7use serde_json::Value as JValue;
8
9pub fn register(lua: &Lua) -> anyhow::Result<()> {
10    let serde_mod = get_or_create_sub_module(lua, "serde")?;
11    let kumo_mod = get_or_create_module(lua, "kumo")?;
12
13    serde_mod.set("json_load", lua.create_async_function(json_load)?)?;
14    serde_mod.set("json_parse", lua.create_function(json_parse)?)?;
15    serde_mod.set("json_encode", lua.create_function(json_encode)?)?;
16    serde_mod.set(
17        "json_encode_pretty",
18        lua.create_function(json_encode_pretty)?,
19    )?;
20
21    serde_mod.set("toml_load", lua.create_async_function(toml_load)?)?;
22    serde_mod.set("toml_parse", lua.create_function(toml_parse)?)?;
23    serde_mod.set("toml_encode", lua.create_function(toml_encode)?)?;
24    serde_mod.set(
25        "toml_encode_pretty",
26        lua.create_function(toml_encode_pretty)?,
27    )?;
28    serde_mod.set(
29        "toml_encode_pretty_compact",
30        lua.create_function(toml_encode_pretty_compact_lua)?,
31    )?;
32
33    serde_mod.set("load_files", lua.create_async_function(load_files)?)?;
34
35    serde_mod.set("yaml_load", lua.create_async_function(yaml_load)?)?;
36    serde_mod.set("yaml_parse", lua.create_function(yaml_parse)?)?;
37    serde_mod.set("yaml_encode", lua.create_function(yaml_encode)?)?;
38    // Note there is no pretty encoder for yaml, because the default one is pretty already.
39    // See https://github.com/dtolnay/serde-yaml/issues/226
40
41    // Backwards compatibility
42    kumo_mod.set("json_load", lua.create_async_function(json_load)?)?;
43    kumo_mod.set("json_parse", lua.create_function(json_parse)?)?;
44    kumo_mod.set("json_encode", lua.create_function(json_encode)?)?;
45    kumo_mod.set(
46        "json_encode_pretty",
47        lua.create_function(json_encode_pretty)?,
48    )?;
49
50    kumo_mod.set("toml_load", lua.create_async_function(toml_load)?)?;
51    kumo_mod.set("toml_parse", lua.create_function(toml_parse)?)?;
52    kumo_mod.set("toml_encode", lua.create_function(toml_encode)?)?;
53    kumo_mod.set(
54        "toml_encode_pretty",
55        lua.create_function(toml_encode_pretty)?,
56    )?;
57
58    Ok(())
59}
60
61fn parse_json(file_name: &str, data: &[u8]) -> anyhow::Result<JValue> {
62    let stripped = json_comments::StripComments::new(data);
63    serde_json::from_reader(stripped).with_context(|| format!("parsing {file_name} as json"))
64}
65
66fn parse_toml(file_name: &str, data: &str) -> anyhow::Result<toml::Value> {
67    toml::from_str(data).with_context(|| format!("parsing {file_name} as toml"))
68}
69
70async fn json_load(lua: Lua, file_name: String) -> mlua::Result<LuaValue> {
71    let data = tokio::fs::read(&file_name)
72        .await
73        .with_context(|| format!("reading file {file_name}"))
74        .map_err(any_err)?;
75
76    let obj = parse_json(&file_name, &data).map_err(any_err)?;
77    lua.to_value_with(&obj, serialize_options())
78}
79
80/// The contents of a file parsed according to its name: `.toml` as TOML,
81/// anything else as JSON.
82#[derive(Debug)]
83enum Parsed {
84    Json(JValue),
85    Toml(toml::Value),
86}
87
88fn read_and_parse(file_name: &str) -> anyhow::Result<Parsed> {
89    let data = std::fs::read(file_name).with_context(|| format!("reading file {file_name}"))?;
90    if file_name.ends_with(".toml") {
91        let text =
92            std::str::from_utf8(&data).with_context(|| format!("decoding {file_name} as utf-8"))?;
93        Ok(Parsed::Toml(parse_toml(file_name, text)?))
94    } else {
95        Ok(Parsed::Json(parse_json(file_name, &data)?))
96    }
97}
98
99fn read_and_parse_all(file_names: &[String]) -> anyhow::Result<Vec<Parsed>> {
100    file_names
101        .iter()
102        .map(|file_name| read_and_parse(file_name))
103        .collect()
104}
105
106async fn load_files(lua: Lua, file_names: Vec<String>) -> mlua::Result<Vec<LuaValue>> {
107    let parsed = tokio::task::spawn_blocking(move || read_and_parse_all(&file_names))
108        .await
109        .map_err(any_err)?
110        .map_err(any_err)?;
111
112    parsed
113        .into_iter()
114        .map(|parsed| match parsed {
115            Parsed::Json(value) => lua.to_value_with(&value, serialize_options()),
116            Parsed::Toml(value) => lua.to_value_with(&value, serialize_options()),
117        })
118        .collect()
119}
120
121fn json_parse(lua: &Lua, text: String) -> mlua::Result<LuaValue> {
122    let stripped = json_comments::StripComments::new(text.as_bytes());
123    let obj: serde_json::Value = serde_json::from_reader(stripped)
124        .with_context(|| format!("parsing {text} as json"))
125        .map_err(any_err)?;
126    lua.to_value_with(&obj, serialize_options())
127}
128
129fn json_encode(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
130    let value = materialize_to_lua_value(lua, value)?;
131    serde_json::to_string(&value).map_err(any_err)
132}
133
134fn json_encode_pretty(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
135    let value = materialize_to_lua_value(lua, value)?;
136    // This additional conversion to a json value causes any object types
137    // to become backed by a BTreeMap rather than lua's randomly ordered
138    // table type, so that the pretty print will show the keys in sorted order.
139    // We only do that for pretty printing because we don't usually want
140    // to bother with that additional overhead
141    let value = serde_json::to_value(&value).map_err(any_err)?;
142    serde_json::to_string_pretty(&value).map_err(any_err)
143}
144
145async fn toml_load(lua: Lua, file_name: String) -> mlua::Result<LuaValue> {
146    let data = tokio::fs::read_to_string(&file_name)
147        .await
148        .with_context(|| format!("reading file {file_name}"))
149        .map_err(any_err)?;
150
151    let obj = parse_toml(&file_name, &data).map_err(any_err)?;
152    lua.to_value_with(&obj, serialize_options())
153}
154
155fn toml_parse(lua: &Lua, toml: String) -> mlua::Result<LuaValue> {
156    let obj: toml::Value = toml::from_str(&toml)
157        .with_context(|| format!("parsing {toml} as toml"))
158        .map_err(any_err)?;
159    lua.to_value_with(&obj, serialize_options())
160}
161
162fn toml_encode(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
163    let value = materialize_to_lua_value(lua, value)?;
164    toml::to_string(&value).map_err(any_err)
165}
166
167fn toml_encode_pretty(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
168    let value = materialize_to_lua_value(lua, value)?;
169    toml::to_string_pretty(&value).map_err(any_err)
170}
171
172/// Render a value as pretty TOML with:
173///
174/// - Keys sorted alphabetically at every nesting level.
175/// - Empty tables emitted inline as `{}` rather than as a `[name]`
176///   header. Empty arrays already render inline as `[]` via the
177///   default toml serializer, so no special handling is needed.
178///
179/// Useful for human-readable diagnostic output where stable
180/// scan-friendly ordering matters and empty-table headers would
181/// add visual noise.
182pub fn toml_encode_pretty_compact<T: serde::Serialize>(value: &T) -> anyhow::Result<String> {
183    let initial = toml::to_string(value).context("serializing value as TOML")?;
184    let mut doc: toml_edit::DocumentMut = initial
185        .parse()
186        .context("re-parsing TOML for format normalization")?;
187    fn normalize(table: &mut toml_edit::Table) {
188        // Collect keys of empty sub-tables and re-insert them as
189        // inline empty tables via `insert()`, which establishes the
190        // standard `key = {}` decor. Mutating the Item in place
191        // would leave the original table-header decor on the key
192        // (no `=` separator).
193        let empty_keys: Vec<String> = table
194            .iter()
195            .filter_map(|(k, v)| match v {
196                toml_edit::Item::Table(t) if t.is_empty() => Some(k.to_string()),
197                _ => None,
198            })
199            .collect();
200        for key in empty_keys {
201            table.insert(
202                &key,
203                toml_edit::Item::Value(
204                    toml_edit::Value::InlineTable(toml_edit::InlineTable::new()),
205                ),
206            );
207        }
208
209        // Recurse before sorting; sub-tables that remain (non-empty)
210        // get their own inner sort.
211        for (_, item) in table.iter_mut() {
212            match item {
213                toml_edit::Item::Table(t) => normalize(t),
214                toml_edit::Item::ArrayOfTables(arr) => {
215                    for t in arr.iter_mut() {
216                        normalize(t);
217                    }
218                }
219                _ => {}
220            }
221        }
222
223        // Sort the value entries alphabetically. TOML grammar
224        // requires sub-tables to follow value entries, so sub-tables
225        // naturally trail the sorted values at this level.
226        table.sort_values();
227    }
228    normalize(doc.as_table_mut());
229    Ok(doc.to_string())
230}
231
232fn toml_encode_pretty_compact_lua(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
233    let value = materialize_to_lua_value(lua, value)?;
234    toml_encode_pretty_compact(&value).map_err(any_err)
235}
236
237async fn yaml_load(lua: Lua, file_name: String) -> mlua::Result<LuaValue> {
238    let data = tokio::fs::read(&file_name)
239        .await
240        .with_context(|| format!("reading file {file_name}"))
241        .map_err(any_err)?;
242
243    let value: JValue = serde_yaml::from_slice(&data)
244        .with_context(|| format!("parsing {file_name} as yaml"))
245        .map_err(any_err)?;
246    lua.to_value_with(&value, serialize_options())
247}
248
249fn yaml_parse(lua: &Lua, text: String) -> mlua::Result<LuaValue> {
250    let value: JValue = serde_yaml::from_str(&text)
251        .with_context(|| format!("parsing {text} as yaml"))
252        .map_err(any_err)?;
253    lua.to_value_with(&value, serialize_options())
254}
255
256fn yaml_encode(lua: &Lua, value: LuaValue) -> mlua::Result<String> {
257    let value = materialize_to_lua_value(lua, value)?;
258    serde_yaml::to_string(&value).map_err(any_err)
259}
260
261#[cfg(test)]
262mod test {
263    use super::*;
264
265    fn write(dir: &tempfile::TempDir, name: &str, contents: &str) -> String {
266        let path = dir.path().join(name);
267        std::fs::write(&path, contents).unwrap();
268        path.to_str().unwrap().to_string()
269    }
270
271    #[test]
272    fn load_files_preserves_order_and_dispatches_by_extension() {
273        let dir = tempfile::tempdir().unwrap();
274        let a = write(&dir, "a.toml", "n = 1");
275        let b = write(&dir, "b.json", "{\"n\": 2}");
276        let c = write(&dir, "c.toml", "n = 3");
277
278        let parsed = read_and_parse_all(&[a, b, c]).unwrap();
279        let ns: Vec<i64> = parsed
280            .iter()
281            .map(|p| match p {
282                Parsed::Toml(v) => v["n"].as_integer().unwrap(),
283                Parsed::Json(v) => v["n"].as_i64().unwrap(),
284            })
285            .collect();
286        k9::assert_equal!(ns, vec![1, 2, 3]);
287        assert!(matches!(parsed[0], Parsed::Toml(_)));
288        assert!(matches!(parsed[1], Parsed::Json(_)));
289        assert!(matches!(parsed[2], Parsed::Toml(_)));
290    }
291
292    #[test]
293    fn load_files_names_the_file_that_failed() {
294        let dir = tempfile::tempdir().unwrap();
295        let ok = write(&dir, "ok.toml", "n = 1");
296        let bad = write(&dir, "bad.toml", "not = = toml");
297        let missing = dir.path().join("missing.json");
298        let missing = missing.to_str().unwrap().to_string();
299
300        let err = read_and_parse_all(&[ok.clone(), bad.clone()]).unwrap_err();
301        k9::assert_equal!(err.to_string(), format!("parsing {bad} as toml"));
302
303        let err = read_and_parse_all(&[ok, missing.clone()]).unwrap_err();
304        k9::assert_equal!(err.to_string(), format!("reading file {missing}"));
305    }
306}