init

migration/src/m20220101_000001_create_table.rs

@@ -659,6 +659,12 @@ impl MigrationTrait for Migration {
                     )
                     .col(ColumnDef::new(Alias::new("server_id")).uuid().null())
                     .col(ColumnDef::new(Alias::new("name")).string().not_null())
+                    .col(
+                        ColumnDef::new(Alias::new("supports_skin_tone"))
+                            .boolean()
+                            .not_null()
+                            .default(false),
+                    )
                     .col(ColumnDef::new(Alias::new("emoji_type")).string().not_null())
                     .col(ColumnDef::new(Alias::new("unicode_sequence")).text().null())
                     .col(ColumnDef::new(Alias::new("file_path")).text().null())
@@ -1010,10 +1016,17 @@ impl MigrationTrait for Migration {
 /// number of bind parameters in a single statement to 999.
 async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
     const CHUNK_SIZE: usize = 250;
+    let all_rows = unicode_rows()?;
+    let skin_tone_bases: HashSet<String> = all_rows
+        .iter()
+        .filter(|(_, _, _, codepoints)| contains_skin_tone_modifier(codepoints))
+        .map(|(_, _, _, codepoints)| strip_skin_tone_modifiers(codepoints))
+        .collect();
     let mut used_names = HashSet::with_capacity(3944);
-    let rows = unicode_rows()?
+    let rows = all_rows
         .into_iter()
-        .map(|(id, sequence, raw_name)| {
+        .filter(|(_, _, _, codepoints)| !contains_skin_tone_modifier(codepoints))
+        .map(|(id, sequence, raw_name, codepoints)| {
             let base = normalize_unicode_name(&raw_name);
             let mut name = base.clone();
             if !used_names.insert(name.clone()) {
@@ -1027,7 +1040,7 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
                     )));
                 }
             }
-            Ok((id, sequence, name))
+            Ok((id, sequence, name, skin_tone_bases.contains(&codepoints)))
         })
         .collect::<Result<Vec<_>, DbErr>>()?;
 
@@ -1037,15 +1050,17 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
             Alias::new("id"),
             Alias::new("server_id"),
             Alias::new("name"),
+            Alias::new("supports_skin_tone"),
             Alias::new("emoji_type"),
             Alias::new("unicode_sequence"),
         ]);
 
-        for (id, sequence, name) in chunk {
+        for (id, sequence, name, supports_skin_tone) in chunk {
             insert.values_panic([
                 Expr::val(*id).into(),
                 Expr::val(Option::<Uuid>::None).into(),
                 Expr::val(name.as_str()).into(),
+                Expr::val(*supports_skin_tone).into(),
                 Expr::val("unicode").into(),
                 Expr::val(sequence.as_str()).into(),
             ]);
@@ -1057,7 +1072,7 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
     Ok(())
 }
 
-fn unicode_rows() -> Result<Vec<(Uuid, String, String)>, DbErr> {
+fn unicode_rows() -> Result<Vec<(Uuid, String, String, String)>, DbErr> {
     UNICODE_EMOJI_DATA
         .lines()
         .filter(|line| !line.trim().is_empty())
@@ -1071,11 +1086,25 @@ fn unicode_rows() -> Result<Vec<(Uuid, String, String)>, DbErr> {
                 format!("https://oxspeak.local/unicode/{UNICODE_EMOJI_VERSION}/{sequence}")
                     .as_bytes(),
             );
-            Ok((id, sequence, name.to_string()))
+            Ok((id, sequence, name.to_string(), codepoints.to_string()))
         })
         .collect()
 }
 
+fn contains_skin_tone_modifier(codepoints: &str) -> bool {
+    codepoints
+        .split_whitespace()
+        .any(|codepoint| matches!(u32::from_str_radix(codepoint, 16), Ok(0x1F3FB..=0x1F3FF)))
+}
+
+fn strip_skin_tone_modifiers(codepoints: &str) -> String {
+    codepoints
+        .split_whitespace()
+        .filter(|codepoint| !matches!(u32::from_str_radix(codepoint, 16), Ok(0x1F3FB..=0x1F3FF)))
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
 fn normalize_unicode_name(name: &str) -> String {
     let mut alias = String::with_capacity(name.len());
     let mut separator = false;
@@ -1119,13 +1148,13 @@ mod tests {
         let entries: Vec<_> = unicode_rows().unwrap();
 
         assert_eq!(entries.len(), 3944);
-        assert!(entries.iter().any(|(_, sequence, name)| {
+        assert!(entries.iter().any(|(_, sequence, name, _)| {
             sequence == "🇺🇸" && name == "flag: United States"
         }));
-        assert!(entries.iter().any(|(_, sequence, name)| {
+        assert!(entries.iter().any(|(_, sequence, name, _)| {
             sequence == "👍🏽" && name == "thumbs up: medium skin tone"
         }));
-        assert!(entries.iter().any(|(_, sequence, name)| {
+        assert!(entries.iter().any(|(_, sequence, name, _)| {
             sequence == "👩‍💻" && name == "woman technologist"
         }));
     }
@@ -1152,4 +1181,23 @@ mod tests {
             .chars()
             .all(|character| character.is_ascii_alphanumeric() || character == '_'));
     }
+
+    #[test]
+    fn skin_tone_variants_are_grouped_under_their_base_sequence() {
+        let rows = unicode_rows().unwrap();
+        let thumbs_up: Vec<_> = rows
+            .iter()
+            .filter(|(_, _, name, _)| name.starts_with("thumbs up"))
+            .collect();
+
+        assert_eq!(thumbs_up.len(), 6);
+        assert_eq!(
+            thumbs_up
+                .iter()
+                .filter(|(_, _, _, codepoints)| !contains_skin_tone_modifier(codepoints))
+                .count(),
+            1
+        );
+        assert_eq!(strip_skin_tone_modifiers("1F44D 1F3FD"), "1F44D");
+    }
 }