Implement emoji properties.

crlf0710 · crlf0710 · commit dfb02a6c3371 · 2022-10-16T11:01:53.000+08:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,4 +3,8 @@ name = "unicode-properties"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+emoji = []
+default = ["emoji"]
+
 [dependencies]
diff --git a/scripts/unicode.py b/scripts/unicode.py
@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+#
+# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This script uses the following Unicode UCD data:
+# - emoji/emoji-data.txt
+#
+# Since this should not require frequent updates, we just store this
+# out-of-line and check the tables.rs file into git.
+
+import fileinput, re, os, sys, operator
+
+preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
+
+#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
+'''
+
+UNICODE_VERSION = (15, 0, 0)
+
+UNICODE_VERSION_NUMBER = "%s.%s.%s" %UNICODE_VERSION
+
+# Download a UCD table file
+def fetch_unidata(f):
+    if not os.path.exists(os.path.basename(f)):
+        os.system("curl -O https://www.unicode.org/Public/%s/ucd/%s"
+                  % (UNICODE_VERSION_NUMBER, f))
+
+    if not os.path.exists(os.path.basename(f)):
+        sys.stderr.write("cannot load %s" % f)
+        exit(1)
+
+# Loads code point data from emoji-data.txt
+# Implementation from unicode-segmentation
+def load_emoji_properties(f):
+    fetch_unidata(f)
+    kinds = {}
+    re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)")
+    re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+) *#")
+
+    for line in fileinput.input(os.path.basename(f), openhook=fileinput.hook_encoded("utf-8")):
+        kind = None
+        d_lo = 0
+        d_hi = 0
+        m = re1.match(line)
+        if m:
+            d_lo = m.group(1)
+            d_hi = m.group(1)
+            kind = m.group(2).strip()
+        else:
+            m = re2.match(line)
+            if m:
+                d_lo = m.group(1)
+                d_hi = m.group(2)
+                kind = m.group(3).strip()
+            else:
+                continue
+        d_lo = int(d_lo, 16)
+        d_hi = int(d_hi, 16)
+        if kind not in kinds:
+            kinds[kind] = []
+        kinds[kind].append((d_lo, d_hi))
+
+    return kinds
+
+def format_table_content(f, content, indent):
+    line = " "*indent
+    first = True
+    for chunk in content.split(","):
+        if len(line) + len(chunk) < 98:
+            if first:
+                line += chunk
+            else:
+                line += ", " + chunk
+            first = False
+        else:
+            f.write(line + ",\n")
+            line = " "*indent + chunk
+    f.write(line)
+
+def escape_char(c):
+    if c == 'multi':
+        return "\"<multiple code points>\""
+    return "'\\u{%x}'" % c
+
+def escape_char_list(l):
+    line = "["
+    first = True
+    for c in l:
+        if first:
+            line += escape_char(c)
+        else:
+            line += ", " + escape_char(c)
+        first = False
+    line += "]"
+    return line
+
+def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
+        pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])), is_const=True):
+    pub_string = "const"
+    if not is_const:
+        pub_string = "let"
+    if is_pub:
+        pub_string = "pub " + pub_string
+    f.write("    %s %s: %s = &[\n" % (pub_string, name, t_type))
+    data = ""
+    first = True
+    for dat in t_data:
+        if not first:
+            data += ","
+        first = False
+        data += pfun(dat)
+    format_table_content(f, data, 8)
+    f.write("\n    ];\n\n")
+
+def emit_emoji_module(f):
+    f.write("""#[cfg(feature = \"emoji\")]
+pub mod emoji {""")
+    f.write("""
+
+    #[derive(Copy, Clone, Hash, Eq, PartialEq, Ord, PartialOrd, Debug)]
+    #[allow(non_camel_case_types)]
+    #[non_exhaustive]
+    pub enum EmojiStatus {
+        NonEmoji,
+        NonEmojiButEmojiComponent,
+        EmojiPresentation,
+        EmojiModifierBase,
+        EmojiPresentationAndModifierBase,
+        EmojiOther,
+        EmojiPresentationAndEmojiComponent,
+        EmojiPresentationAndModifierAndEmojiComponent,
+        EmojiOtherAndEmojiComponent,
+    }
+    #[inline]
+    pub(crate) fn emoji_status(c: char) -> EmojiStatus {
+        // FIXME: do we want to special case ASCII here?
+        match c as usize {
+            _ => super::util::bsearch_range_value_table(c, EMOJI_STATUS).unwrap()
+        }
+    }
+    #[inline]
+    pub(crate) fn is_emoji_status_for_emoji_char(s: EmojiStatus) -> bool {
+        !matches!(s, EmojiStatus::NonEmoji | EmojiStatus::NonEmojiButEmojiComponent)
+    }
+    #[inline]
+    pub(crate) fn is_emoji_status_for_emoji_component(s: EmojiStatus) -> bool {
+        matches!(s, EmojiStatus::EmojiPresentationAndEmojiComponent |
+            EmojiStatus::EmojiPresentationAndModifierAndEmojiComponent |
+            EmojiStatus::EmojiOtherAndEmojiComponent)
+    }
+""")
+
+    f.write("    // Emoji status table:\n")
+    emoji_status_table = load_emoji_properties("emoji/emoji-data.txt")
+    # we combine things together here.
+    
+    # `Extended_Pictographic`` is only for future proof usages, we ignore it here.
+    # emoji_prop_list = ["Emoji", "Emoji_Presentation", "Emoji_Modifier", "Emoji_Modifier_Base", "Emoji_Component", "Extended_Pictographic"]
+    emoji_prop_list = ["Emoji", "Emoji_Presentation", "Emoji_Modifier", "Emoji_Modifier_Base", "Emoji_Component"]
+
+    # need to skip surrogates because they're not representable by rust `char`s
+    emoji_status_table["Surrogate"] = [(0xD800, 0xDFFF)]
+    emoji_prop_list.append("Surrogate")
+    
+    emoji_prop_list_len = [len(emoji_status_table[x]) for x in emoji_prop_list]
+    emoji_prop_count = len(emoji_prop_list)
+    code_point_first = 0
+    code_point_last = 0x10FFFF
+    emoji_prop_list_pos = [0 for _ in emoji_prop_list]
+    cur_group_first = code_point_first
+    emoji_table = []
+    def group_text(s):
+        if s == "Surrogate":
+            return "<Surrogate>"
+        elif s == "":
+            return "EmojiStatus::NonEmoji"
+        elif s == "Emoji_Component":
+            return "EmojiStatus::NonEmojiButEmojiComponent"
+        elif s == "Emoji;Emoji_Presentation":
+            return "EmojiStatus::EmojiPresentation"
+        elif s == "Emoji;Emoji_Presentation;Emoji_Modifier_Base":
+            return "EmojiStatus::EmojiPresentationAndModifierBase"
+        elif s == "Emoji;Emoji_Modifier_Base":
+            return "EmojiStatus::EmojiModifierBase"
+        elif s == "Emoji":
+            return "EmojiStatus::EmojiOther"
+        elif s == "Emoji;Emoji_Presentation;Emoji_Component":
+            return "EmojiStatus::EmojiPresentationAndEmojiComponent"
+        elif s == "Emoji;Emoji_Presentation;Emoji_Modifier;Emoji_Component":
+            return "EmojiStatus::EmojiPresentationAndModifierAndEmojiComponent"
+        elif s == "Emoji;Emoji_Component":
+            return "EmojiStatus::EmojiOtherAndEmojiComponent"
+        else:
+            return "EmojiStatus::NewCombination(\"" + s + "\")"
+    while cur_group_first <= code_point_last:
+        cur_group_props = []
+        cur_group_last = code_point_last
+        for prop_list_idx in range(emoji_prop_count):
+            if emoji_prop_list_pos[prop_list_idx] >= emoji_prop_list_len[prop_list_idx]:
+                continue
+            elif emoji_status_table[emoji_prop_list[prop_list_idx]][emoji_prop_list_pos[prop_list_idx]][0] > cur_group_first:
+                cur_group_last = min(cur_group_last, emoji_status_table[emoji_prop_list[prop_list_idx]][emoji_prop_list_pos[prop_list_idx]][0] - 1)
+            else:
+                cur_group_props.append(emoji_prop_list[prop_list_idx])
+                cur_group_last = min(cur_group_last, emoji_status_table[emoji_prop_list[prop_list_idx]][emoji_prop_list_pos[prop_list_idx]][1])
+        cur_group_text = group_text(";".join(cur_group_props))
+        if cur_group_text != "<Surrogate>":
+            emoji_table.append((cur_group_first, cur_group_last, cur_group_text))
+        for prop_list_idx in range(emoji_prop_count):
+            if emoji_prop_list_pos[prop_list_idx] >= emoji_prop_list_len[prop_list_idx]:
+                continue
+            elif emoji_status_table[emoji_prop_list[prop_list_idx]][emoji_prop_list_pos[prop_list_idx]][0] > cur_group_first:
+                continue
+            else:
+                if cur_group_last == emoji_status_table[emoji_prop_list[prop_list_idx]][emoji_prop_list_pos[prop_list_idx]][1]:
+                    emoji_prop_list_pos[prop_list_idx] += 1
+        cur_group_first = cur_group_last + 1
+
+    emit_table(f, "EMOJI_STATUS", emoji_table, "&'static [(char, char, EmojiStatus)]", is_pub=False,
+            pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), x[2]))
+    f.write("}\n\n")
+
+def emit_util_mod(f):
+    f.write("""
+#[allow(dead_code)]
+pub mod util {
+    use core::result::Result::{Ok, Err};
+
+    #[inline]
+    pub fn bsearch_table(c: char, r: &'static [char]) -> bool {
+        r.binary_search(&c).is_ok()
+    }
+
+    #[inline]
+    pub fn bsearch_value_table<T: Copy>(c: char, r: &'static [(char, T)]) -> Option<T> {
+        match r.binary_search_by_key(&c, |&(k, _)| k) {
+            Ok(idx) => {
+                let (_, v) = r[idx];
+                Some(v)
+            }
+            Err(_) => None
+        }
+    }
+
+    #[inline]
+    pub fn bsearch_range_table(c: char, r: &'static [(char,char)]) -> bool {
+        use core::cmp::Ordering::{Equal, Less, Greater};
+        r.binary_search_by(|&(lo,hi)| {
+            if lo <= c && c <= hi { Equal }
+            else if hi < c { Less }
+            else { Greater }
+        }).is_ok()
+    }
+
+    pub fn bsearch_range_value_table<T: Copy>(c: char, r: &'static [(char, char, T)]) -> Option<T> {
+        use core::cmp::Ordering::{Equal, Less, Greater};
+        match r.binary_search_by(|&(lo, hi, _)| {
+            if lo <= c && c <= hi { Equal }
+            else if hi < c { Less }
+            else { Greater }
+        }) {
+            Ok(idx) => {
+                let (_, _, cat) = r[idx];
+                Some(cat)
+            }
+            Err(_) => None
+        }
+    }
+
+}
+
+""")
+
+if __name__ == "__main__":
+    r = "tables.rs"
+    if os.path.exists(r):
+        os.remove(r)
+    with open(r, "w") as rf:
+        # write the file's preamble
+        rf.write(preamble)
+
+        rf.write("""
+/// The version of [Unicode](http://www.unicode.org/)
+/// that this version of unicode-security is based on.
+pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
+
+""" % UNICODE_VERSION)
+
+        emit_util_mod(rf)
+        ### emoji module
+        emit_emoji_module(rf)
diff --git a/src/lib.rs b/src/lib.rs
@@ -1,8 +1,33 @@
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn it_works() {
-        let result = 2 + 2;
-        assert_eq!(result, 4);
+#[rustfmt::skip]
+mod tables;
+
+#[cfg(feature = "emoji")]
+mod emoji {
+    pub use crate::tables::emoji::EmojiStatus;
+
+    pub trait UnicodeEmoji: Sized {
+        fn emoji_status(self) -> EmojiStatus;
+
+        fn is_emoji_char(self) -> bool {
+            crate::tables::emoji::is_emoji_status_for_emoji_char(self.emoji_status())
+        }
+
+        fn is_emoji_component(self) -> bool {
+            crate::tables::emoji::is_emoji_status_for_emoji_component(self.emoji_status())
+        }
+    }
+
+    impl UnicodeEmoji for char {
+        fn emoji_status(self) -> EmojiStatus {
+            crate::tables::emoji::emoji_status(self)
+        }
     }
 }
+
+pub use tables::UNICODE_VERSION;
+
+#[cfg(feature = "emoji")]
+pub use emoji::UnicodeEmoji;
+
+#[cfg(feature = "emoji")]
+pub use emoji::EmojiStatus;
diff --git a/src/tables.rs b/src/tables.rs
