diff --git a/migration/src/m20220101_000001_create_table.rs b/migration/src/m20220101_000001_create_table.rs index 99261eb..21d72b0 100644 --- a/migration/src/m20220101_000001_create_table.rs +++ b/migration/src/m20220101_000001_create_table.rs @@ -659,6 +659,12 @@ impl MigrationTrait for Migration { ) .col(ColumnDef::new(Alias::new("server_id")).uuid().null()) .col(ColumnDef::new(Alias::new("name")).string().not_null()) + .col( + ColumnDef::new(Alias::new("supports_skin_tone")) + .boolean() + .not_null() + .default(false), + ) .col(ColumnDef::new(Alias::new("emoji_type")).string().not_null()) .col(ColumnDef::new(Alias::new("unicode_sequence")).text().null()) .col(ColumnDef::new(Alias::new("file_path")).text().null()) @@ -1010,10 +1016,17 @@ impl MigrationTrait for Migration { /// number of bind parameters in a single statement to 999. async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> { const CHUNK_SIZE: usize = 250; + let all_rows = unicode_rows()?; + let skin_tone_bases: HashSet = all_rows + .iter() + .filter(|(_, _, _, codepoints)| contains_skin_tone_modifier(codepoints)) + .map(|(_, _, _, codepoints)| strip_skin_tone_modifiers(codepoints)) + .collect(); let mut used_names = HashSet::with_capacity(3944); - let rows = unicode_rows()? + let rows = all_rows .into_iter() - .map(|(id, sequence, raw_name)| { + .filter(|(_, _, _, codepoints)| !contains_skin_tone_modifier(codepoints)) + .map(|(id, sequence, raw_name, codepoints)| { let base = normalize_unicode_name(&raw_name); let mut name = base.clone(); if !used_names.insert(name.clone()) { @@ -1027,7 +1040,7 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> { ))); } } - Ok((id, sequence, name)) + Ok((id, sequence, name, skin_tone_bases.contains(&codepoints))) }) .collect::, DbErr>>()?; @@ -1037,15 +1050,17 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> { Alias::new("id"), Alias::new("server_id"), Alias::new("name"), + Alias::new("supports_skin_tone"), Alias::new("emoji_type"), Alias::new("unicode_sequence"), ]); - for (id, sequence, name) in chunk { + for (id, sequence, name, supports_skin_tone) in chunk { insert.values_panic([ Expr::val(*id).into(), Expr::val(Option::::None).into(), Expr::val(name.as_str()).into(), + Expr::val(*supports_skin_tone).into(), Expr::val("unicode").into(), Expr::val(sequence.as_str()).into(), ]); @@ -1057,7 +1072,7 @@ async fn seed_unicode_emojis(manager: &SchemaManager<'_>) -> Result<(), DbErr> { Ok(()) } -fn unicode_rows() -> Result, DbErr> { +fn unicode_rows() -> Result, DbErr> { UNICODE_EMOJI_DATA .lines() .filter(|line| !line.trim().is_empty()) @@ -1071,11 +1086,25 @@ fn unicode_rows() -> Result, DbErr> { format!("https://oxspeak.local/unicode/{UNICODE_EMOJI_VERSION}/{sequence}") .as_bytes(), ); - Ok((id, sequence, name.to_string())) + Ok((id, sequence, name.to_string(), codepoints.to_string())) }) .collect() } +fn contains_skin_tone_modifier(codepoints: &str) -> bool { + codepoints + .split_whitespace() + .any(|codepoint| matches!(u32::from_str_radix(codepoint, 16), Ok(0x1F3FB..=0x1F3FF))) +} + +fn strip_skin_tone_modifiers(codepoints: &str) -> String { + codepoints + .split_whitespace() + .filter(|codepoint| !matches!(u32::from_str_radix(codepoint, 16), Ok(0x1F3FB..=0x1F3FF))) + .collect::>() + .join(" ") +} + fn normalize_unicode_name(name: &str) -> String { let mut alias = String::with_capacity(name.len()); let mut separator = false; @@ -1119,13 +1148,13 @@ mod tests { let entries: Vec<_> = unicode_rows().unwrap(); assert_eq!(entries.len(), 3944); - assert!(entries.iter().any(|(_, sequence, name)| { + assert!(entries.iter().any(|(_, sequence, name, _)| { sequence == "πŸ‡ΊπŸ‡Έ" && name == "flag: United States" })); - assert!(entries.iter().any(|(_, sequence, name)| { + assert!(entries.iter().any(|(_, sequence, name, _)| { sequence == "πŸ‘πŸ½" && name == "thumbs up: medium skin tone" })); - assert!(entries.iter().any(|(_, sequence, name)| { + assert!(entries.iter().any(|(_, sequence, name, _)| { sequence == "πŸ‘©β€πŸ’»" && name == "woman technologist" })); } @@ -1152,4 +1181,23 @@ mod tests { .chars() .all(|character| character.is_ascii_alphanumeric() || character == '_')); } + + #[test] + fn skin_tone_variants_are_grouped_under_their_base_sequence() { + let rows = unicode_rows().unwrap(); + let thumbs_up: Vec<_> = rows + .iter() + .filter(|(_, _, name, _)| name.starts_with("thumbs up")) + .collect(); + + assert_eq!(thumbs_up.len(), 6); + assert_eq!( + thumbs_up + .iter() + .filter(|(_, _, _, codepoints)| !contains_skin_tone_modifier(codepoints)) + .count(), + 1 + ); + assert_eq!(strip_skin_tone_modifiers("1F44D 1F3FD"), "1F44D"); + } } diff --git a/src/domain/dto/emoji.rs b/src/domain/dto/emoji.rs index 949ea53..f78b8b2 100644 --- a/src/domain/dto/emoji.rs +++ b/src/domain/dto/emoji.rs @@ -32,6 +32,7 @@ pub struct EmojiResponse { pub emoji_type: String, pub unicode_sequence: Option, pub name: String, + pub supports_skin_tone: bool, pub asset_url: Option, pub mime_type: Option, pub file_size: Option, diff --git a/src/models/emoji.rs b/src/models/emoji.rs index 04271e8..391e181 100644 --- a/src/models/emoji.rs +++ b/src/models/emoji.rs @@ -10,6 +10,7 @@ pub struct Model { pub id: Uuid, pub server_id: Option, pub name: String, + pub supports_skin_tone: bool, pub emoji_type: String, pub unicode_sequence: Option, pub file_path: Option, diff --git a/src/routes/emoji/handlers.rs b/src/routes/emoji/handlers.rs index f63262b..581a389 100644 --- a/src/routes/emoji/handlers.rs +++ b/src/routes/emoji/handlers.rs @@ -137,6 +137,7 @@ pub async fn create( id: Set(id), server_id: Set(server_id), name: Set(String::new()), + supports_skin_tone: Set(false), emoji_type: Set(emoji_type), unicode_sequence: Set(unicode_sequence), file_path: Set(path), diff --git a/src/routes/emoji/mapper.rs b/src/routes/emoji/mapper.rs index fa6213d..2260efc 100644 --- a/src/routes/emoji/mapper.rs +++ b/src/routes/emoji/mapper.rs @@ -7,6 +7,7 @@ pub fn response(model: emoji::Model) -> EmojiResponse { emoji_type: model.emoji_type, unicode_sequence: model.unicode_sequence, name: model.name, + supports_skin_tone: model.supports_skin_tone, asset_url: model .file_path .as_ref()