ICU instead of unholy combination of langtag and isolang

Author: Porges

build/Cargo.lock

@@ -9,13 +9,13 @@ celes = "2.5.0"
 clap = { version = "4.5.23", features = ["derive"] }
 dunce = "1.0.5"
 eyre = "0.6.12"
-indexmap = "2.7.0"
+icu = { version = "1.5.0", features = ["experimental", "serde"] }
+icu_provider = { version = "1.5.0", features = ["sync"] }
+indexmap = "2.7.1"
 isbn = { version = "0.4.0" }
-isolang = { version = "2.4.0", features = ["serde", "local_names"] }
 itertools = "0.14.0"
-langtag = { version = "0.4.0", features = ["serde"] }
 markdown = "1.0.0-alpha"
-maud = "0.26.0"
+maud = "0.27.0"
 notify = "8.0.0"
 notify-debouncer-full = "0.5.0"
 num-format = "0.4.4"

build/Cargo.toml

@@ -10,6 +10,7 @@ use crate::{
         LString, MagazineArticle, NewspaperArticle, NumberOrString, Pagination, Periodical, Person,
         Reference, Thesis, WebPage,
     },
+    intl::INTL,
     nvec::OneOrMore,
 };
 
@@ -288,11 +289,11 @@ fn render_title(r: &Reference) -> Markup {
     };
 
     let additional_suffix = r.common().language.as_ref().map(|lang| {
-            html! {
-                "text in " (lang.to_name())
-                meta itemprop="inLanguage" content=(lang.to_639_1().unwrap_or_else(|| lang.to_639_3())) ;
-            }
-        });
+        html! {
+            "text in " (INTL.english_name(lang.language).unwrap())
+            meta itemprop="inLanguage" content=(lang);
+        }
+    });
 
     let rest = render_lstr_alt(
         &r.common().title,
@@ -422,7 +423,7 @@ fn render_lstr_just_span(
     item_prop: Option<&'static str>,
 ) -> Markup {
     html! {
-        span class=[class] itemprop=[item_prop] lang=[lstr.lang.as_deref()] {
+        span class=[class] itemprop=[item_prop] lang=[lstr.lang.as_ref()] {
             (maud::PreEscaped(&lstr.value))
         }
     }
@@ -469,7 +470,7 @@ fn render_lstr_just_cite(
     item_prop: Option<&'static str>,
 ) -> Markup {
     html! {
-        cite class=[class] itemprop=[item_prop] lang=[lstr.lang.as_deref()] {
+        cite class=[class] itemprop=[item_prop] lang=[lstr.lang.as_ref()] {
             (maud::PreEscaped(&lstr.value))
         }
     }

build/src/bib_render.rs

@@ -1,7 +1,6 @@
 use std::{collections::BTreeMap, convert::Infallible, str::FromStr};
 
-use isolang::Language;
-use langtag::LangTagBuf;
+use icu::locid::LanguageIdentifier;
 use num_format::ToFormattedString;
 use serde::{
     de::{self, Visitor},
@@ -137,7 +136,7 @@ pub struct Common {
     #[serde(rename = "archive-URL")]
     pub archive_url: Option<String>,
 
-    pub language: Option<Language>,
+    pub language: Option<LanguageIdentifier>,
 
     pub notes: Option<String>,
     pub warnings: Option<String>,
@@ -508,7 +507,7 @@ impl Date {
 #[derive(Debug, Clone)]
 pub struct LString {
     pub value: String,
-    pub lang: Option<LangTagBuf>,
+    pub lang: Option<LanguageIdentifier>,
     pub alt: Option<String>,
 }
 
@@ -557,7 +556,7 @@ impl<'de> Deserialize<'de> for LString {
             where
                 A: de::MapAccess<'de>,
             {
-                let mut value = None;
+                let mut value: Option<String> = None;
                 let mut lang = None;
                 let mut alt = None;
 
@@ -575,7 +574,7 @@ impl<'de> Deserialize<'de> for LString {
                                 return Err(de::Error::duplicate_field("lang"));
                             }
 
-                            lang = match LangTagBuf::new(v) {
+                            lang = match LanguageIdentifier::try_from_bytes(v.as_bytes()) {
                                 Ok(l) => Some(l),
                                 Err(e) => return Err(de::Error::custom(e)),
                             };

build/src/bibliography.rs

@@ -0,0 +1,81 @@
+use std::{
+    collections::BTreeMap,
+    sync::{LazyLock, Mutex},
+};
+
+use icu::{
+    experimental::displaynames::LanguageDisplayNames,
+    locid::{
+        locale,
+        subtags::{language, Language},
+        Locale,
+    },
+};
+
+pub struct Intl {
+    english_display_names: LanguageDisplayNames,
+    other_display_names: Mutex<BTreeMap<Language, Option<String>>>,
+}
+
+impl Intl {
+    pub fn new() -> Self {
+        let locale = locale!("en-GB").into();
+        let options = Default::default();
+        let english_display_names = LanguageDisplayNames::try_new(&locale, options).unwrap();
+
+        let mut other_display_names = BTreeMap::new();
+        // backfill missing values
+        other_display_names.insert(language!("cmn"), Some("官话".into()));
+        other_display_names.insert(language!("ewe"), Some("Eʋegbe".into()));
+        other_display_names.insert(language!("gez"), Some("ግዕዝ".into()));
+        other_display_names.insert(language!("kxd"), Some("Bahasa Melayu Brunei".into()));
+        other_display_names.insert(language!("mak"), Some("ᨅᨔ ᨆᨀᨔᨑ".into()));
+        other_display_names.insert(language!("mcm"), Some("Papia Kristang".into()));
+        other_display_names.insert(language!("nan"), Some("閩南語".into()));
+        other_display_names.insert(language!("rng"), Some("XiRonga".into()));
+        other_display_names.insert(language!("tws"), Some("潮州話".into()));
+        other_display_names.insert(language!("wuu"), Some("吳語".into()));
+
+        Self {
+            english_display_names,
+            other_display_names: Mutex::new(other_display_names),
+        }
+    }
+
+    pub fn english_name(&self, language: Language) -> Option<&str> {
+        self.english_display_names.of(language).or_else(|| {
+            match language.as_str() {
+                "cmn" => "Mandarin Chinese",
+                "kew" => "Kewa",
+                "kxd" => "Brunei Malay",
+                "mbw" => "Maring",
+                "mcm" => "Kristang",
+                "mnk" => "Mandinka",
+                "mnr" => "Mono",
+                "rng" => "Ronga",
+                "stv" => "Siltʼe",
+                "tws" => "Teochew",
+                "urh" => "Urhobo",
+                _ => return None,
+            }
+            .into()
+        })
+    }
+
+    pub fn autonym(&self, language: Language) -> Option<String> {
+        self.other_display_names
+            .lock()
+            .unwrap()
+            .entry(language)
+            .or_insert_with_key(|key| {
+                let locale = Locale::from(*key).into();
+                let options = Default::default();
+                LanguageDisplayNames::try_new(&locale, options)
+                    .ok()
+                    .and_then(|display_names| display_names.of(*key).map(str::to_string))
+            })
+            .clone()
+    }
+}
+
+pub static INTL: LazyLock<Intl> = LazyLock::new(Intl::new);

build/src/intl.rs

@@ -15,8 +15,8 @@ use std::{
 use bib_render::RenderedBibliography;
 use clap::Parser;
 use eyre::{bail, eyre, Context, ContextCompat, OptionExt, Result};
+use icu::locid::LanguageIdentifier;
 use itertools::Itertools;
-use langtag::LangTagBuf;
 use markdown::{mdast, Constructs, ParseOptions};
 use maud::Markup;
 use notify::{
@@ -36,6 +36,7 @@ use walkdir::WalkDir;
 mod bib_render;
 mod bib_to_csl;
 mod bibliography;
+mod intl;
 mod mdast_to_html;
 mod nvec;
 mod templates;
@@ -338,7 +339,7 @@ impl ImageManifestEntry {
 }
 
 pub struct Aka {
-    pub language: LangTagBuf,
+    pub lang_id: LanguageIdentifier,
     pub word: Markup,
     pub url_path: String,
 }
@@ -671,7 +672,7 @@ impl Builder {
         article: &File<T>,
         url_lookup: &BTreeMap<String, Option<&str>>,
         article_tree: Option<&ArticleNode>,
-        aka_handler: impl Fn(LangTagBuf, Markup),
+        aka_handler: impl Fn(LanguageIdentifier, Markup),
     ) -> Result<OutputFile>
     where
         File<T>: ArticleMetadata + BaseMetadata,
@@ -758,10 +759,10 @@ impl Builder {
             .par_iter()
             .filter(|a| !a.is_draft() || self.output_drafts)
             .map(|article| {
-                let push_aka = |language: LangTagBuf, word: Markup| {
+                let push_aka = |language: LanguageIdentifier, word: Markup| {
                     if self.output_drafts || !article.is_draft() {
                         let aka = Aka {
-                            language,
+                            lang_id: language,
                             word,
                             url_path: article.url_path.to_string(),
                         };
@@ -777,10 +778,10 @@ impl Builder {
             .par_iter()
             .filter(|a| !a.is_draft() || self.output_drafts)
             .map(|game| {
-                let push_aka = |language: LangTagBuf, word: Markup| {
+                let push_aka = |language: LanguageIdentifier, word: Markup| {
                     if self.output_drafts || !game.is_draft() {
                         let aka = Aka {
-                            language,
+                            lang_id: language,
                             word,
                             url_path: game.url_path.to_string(),
                         };

build/src/main.rs

@@ -1,9 +1,9 @@
 use std::{borrow::Cow, collections::BTreeMap, path::Path, sync::LazyLock};
 
 use eyre::{bail, eyre, Context, OptionExt, Result};
+use icu::locid::{langid, LanguageIdentifier};
 use indexmap::IndexMap;
 use itertools::Itertools;
-use langtag::LangTagBuf;
 use markdown::mdast::{
     AttributeContent, AttributeValue, Blockquote, MdxJsxFlowElement, MdxJsxTextElement, Node, Text,
     Yaml,
@@ -57,7 +57,7 @@ pub fn to_html(
     bibliography: &RenderedBibliography,
     images: &ImageManifest,
     url_lookup: &BTreeMap<String, Option<&str>>,
-    aka_handler: impl Fn(LangTagBuf, Markup),
+    aka_handler: impl Fn(LanguageIdentifier, Markup),
 ) -> Result<Markup> {
     let (fndefs, linkdefs) = locate_defs(node);
     Converter {
@@ -87,7 +87,7 @@ struct Converter<'a> {
     cite_count: usize,
     header_stack: Vec<usize>,
     url_lookup: &'a BTreeMap<String, Option<&'a str>>,
-    aka_handler: &'a dyn Fn(LangTagBuf, Markup),
+    aka_handler: &'a dyn Fn(LanguageIdentifier, Markup),
 }
 
 fn index_to_string(mut index: u32) -> String {
@@ -706,10 +706,10 @@ impl Converter<'_> {
                         .any(|(name, value)| *name == "class" && value.contains("aka"))
                 {
                     let lang_attr = find_attribute(&text.attributes, "lang")
-                        .map(|n| LangTagBuf::new(n.to_string()))
+                        .map(|n| LanguageIdentifier::try_from_bytes(n.as_bytes()))
                         .transpose()
-                        .wrap_err("invalid lang tag")?
-                        .unwrap_or_else(|| LangTagBuf::new("en".to_string()).unwrap());
+                        .map_err(|e| eyre!("invalid lang tag {e}"))?
+                        .unwrap_or_else(|| langid!("en"));
 
                     let markup = self.expand(&text.children)?;
                     (self.aka_handler)(lang_attr, markup);