init

migration/Cargo.toml

@@ -10,6 +10,7 @@ path = "src/lib.rs"
 
 [dependencies]
 async-std = { version = "1", features = ["attributes", "tokio1"] }
+uuid = { version = "1", features = ["v5"] }
 
 [dependencies.sea-orm-migration]
 version = "2.0.2"

migration/src/m20220101_000001_create_table.rs

@@ -1,4 +1,9 @@
 use sea_orm_migration::prelude::*;
+use std::collections::HashSet;
+use uuid::Uuid;
+
+const UNICODE_EMOJI_VERSION: &str = "17.0";
+const UNICODE_EMOJI_DATA: &str = include_str!("../assets/emoji-rgi-17.0.txt");
 
 #[derive(DeriveMigrationName)]
 pub struct Migration;
@@ -698,6 +703,8 @@ impl MigrationTrait for Migration {
             )
             .await?;
 
+        seed_unicode_emojis(manager).await?;
+
         manager
             .create_table(
                 Table::create()
@@ -746,6 +753,8 @@ impl MigrationTrait for Migration {
             )
             .await?;
 
+        seed_unicode_aliases(manager).await?;
+
         manager
             .create_table(
                 Table::create()
@@ -1044,3 +1053,187 @@ impl MigrationTrait for Migration {
         Ok(())
     }
 }
+
+/// Seeds the global RGI emoji set from the pinned Unicode 17.0 data file.
+///
+/// The inserts are intentionally chunked because SQLite commonly limits the
+/// number of bind parameters in a single statement to 999.
+async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
+    const CHUNK_SIZE: usize = 250;
+    let rows = unicode_rows()?;
+
+    for chunk in rows.chunks(CHUNK_SIZE) {
+        let mut insert = Query::insert();
+        insert.into_table(Alias::new("emoji")).columns([
+            Alias::new("id"),
+            Alias::new("server_id"),
+            Alias::new("emoji_type"),
+            Alias::new("unicode_sequence"),
+        ]);
+
+        for (id, sequence, _) in chunk {
+            insert.values_panic([
+                Expr::val(*id).into(),
+                Expr::val(Option::<Uuid>::None).into(),
+                Expr::val("unicode").into(),
+                Expr::val(sequence.as_str()).into(),
+            ]);
+        }
+
+        manager.exec_stmt(insert).await?;
+    }
+
+    Ok(())
+}
+
+async fn seed_unicode_aliases(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
+    const CHUNK_SIZE: usize = 250;
+    let rows = unicode_rows()?;
+    let mut aliases = Vec::with_capacity(rows.len());
+    let mut used = HashSet::with_capacity(rows.len());
+
+    for (_, sequence, name) in rows {
+        let base = normalize_unicode_alias(&name);
+        let mut alias = base.clone();
+        if !used.insert(alias.clone()) {
+            alias = format!("{base}_u{}", sequence.replace(' ', ""));
+            if alias.len() > 64 {
+                alias.truncate(64);
+            }
+            if !used.insert(alias.clone()) {
+                return Err(DbErr::Custom(format!(
+                    "duplicate generated emoji alias: {alias}"
+                )));
+            }
+        }
+
+        let emoji_id = Uuid::new_v5(
+            &Uuid::NAMESPACE_URL,
+            format!("https://oxspeak.local/unicode/{UNICODE_EMOJI_VERSION}/{sequence}").as_bytes(),
+        );
+        let id = Uuid::new_v5(
+            &Uuid::NAMESPACE_URL,
+            format!("https://oxspeak.local/unicode-alias/{UNICODE_EMOJI_VERSION}/{sequence}")
+                .as_bytes(),
+        );
+        aliases.push((id, emoji_id, alias));
+    }
+
+    for chunk in aliases.chunks(CHUNK_SIZE) {
+        let mut insert = Query::insert();
+        insert.into_table(Alias::new("emoji_alias")).columns([
+            Alias::new("id"),
+            Alias::new("emoji_id"),
+            Alias::new("alias"),
+        ]);
+
+        for (id, emoji_id, alias) in chunk {
+            insert.values_panic([
+                Expr::val(*id).into(),
+                Expr::val(*emoji_id).into(),
+                Expr::val(alias.as_str()).into(),
+            ]);
+        }
+
+        manager.exec_stmt(insert).await?;
+    }
+
+    Ok(())
+}
+
+fn unicode_rows() -> Result<Vec<(Uuid, String, String)>, DbErr> {
+    UNICODE_EMOJI_DATA
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| {
+            let (codepoints, name) = line.split_once('\t').ok_or_else(|| {
+                DbErr::Custom(format!("invalid Unicode emoji data line: {line:?}"))
+            })?;
+            let sequence = parse_unicode_sequence(codepoints)?;
+            let id = Uuid::new_v5(
+                &Uuid::NAMESPACE_URL,
+                format!("https://oxspeak.local/unicode/{UNICODE_EMOJI_VERSION}/{sequence}")
+                    .as_bytes(),
+            );
+            Ok((id, sequence, name.to_string()))
+        })
+        .collect()
+}
+
+fn normalize_unicode_alias(name: &str) -> String {
+    let mut alias = String::with_capacity(name.len());
+    let mut separator = false;
+
+    for character in name.chars() {
+        if character.is_ascii_alphanumeric() {
+            if separator && !alias.is_empty() {
+                alias.push('_');
+            }
+            alias.push(character.to_ascii_lowercase());
+            separator = false;
+        } else {
+            separator = true;
+        }
+    }
+
+    alias.truncate(64);
+    alias.trim_end_matches('_').to_string()
+}
+
+fn parse_unicode_sequence(codepoints: &str) -> Result<String, DbErr> {
+    codepoints
+        .split_whitespace()
+        .map(|codepoint| {
+            let value = u32::from_str_radix(codepoint, 16).map_err(|error| {
+                DbErr::Custom(format!("invalid Unicode codepoint {codepoint:?}: {error}"))
+            })?;
+            char::from_u32(value).ok_or_else(|| {
+                DbErr::Custom(format!("invalid Unicode scalar value: {codepoint:?}"))
+            })
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unicode_dataset_contains_the_pinned_rgi_set() {
+        let entries: Vec<_> = unicode_rows().unwrap();
+
+        assert_eq!(entries.len(), 3944);
+        assert!(entries.iter().any(|(_, sequence, name)| {
+            sequence == "🇺🇸" && name == "flag: United States"
+        }));
+        assert!(entries.iter().any(|(_, sequence, name)| {
+            sequence == "👍🏽" && name == "thumbs up: medium skin tone"
+        }));
+        assert!(entries.iter().any(|(_, sequence, name)| {
+            sequence == "👩‍💻" && name == "woman technologist"
+        }));
+    }
+
+    #[test]
+    fn unicode_sequence_parser_handles_sequences_and_rejects_invalid_values() {
+        assert_eq!(parse_unicode_sequence("1F1FA 1F1F8").unwrap(), "🇺🇸");
+        assert_eq!(parse_unicode_sequence("1F44D 1F3FD").unwrap(), "👍🏽");
+        assert!(parse_unicode_sequence("not-a-codepoint").is_err());
+        assert!(parse_unicode_sequence("110000").is_err());
+    }
+
+    #[test]
+    fn unicode_aliases_are_normalized_for_the_existing_api_constraints() {
+        assert_eq!(
+            normalize_unicode_alias("flag: United States"),
+            "flag_united_states"
+        );
+        assert_eq!(
+            normalize_unicode_alias("thumbs up: medium skin tone"),
+            "thumbs_up_medium_skin_tone"
+        );
+        assert!(normalize_unicode_alias("woman technologist")
+            .chars()
+            .all(|character| character.is_ascii_alphanumeric() || character == '_'));
+    }
+}