Merge branch 'master' into patch-1

altendky · web-flow · commit b726540d005b · 2024-11-13T09:39:05.000-05:00
diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml
@@ -23,5 +23,11 @@ path = "fuzz_targets/streaming.rs"
 test = false
 doc = false
 
+[[bin]]
+name = "process"
+path = "fuzz_targets/process.rs"
+test = false
+doc = false
+
 # Work around https://github.com/rust-lang/cargo/issues/8338
 [workspace]
diff --git a/fuzz/fuzz_targets/process.rs b/fuzz/fuzz_targets/process.rs
@@ -0,0 +1,78 @@
+// The fuzzing harness fuzz test some of the the
+// unicode string normalization processing
+
+#![no_main]
+
+#[macro_use]
+extern crate libfuzzer_sys;
+extern crate unicode_normalization;
+
+use unicode_normalization::{
+    char::{
+        canonical_combining_class, compose, decompose_canonical, decompose_compatible,
+        is_combining_mark,
+    },
+    UnicodeNormalization,
+};
+
+fuzz_target!(|data: (u8, String)| {
+    let (function_index, string_data) = data;
+
+    // Create an iterator for characters
+    let mut chars = string_data.chars();
+
+    // Randomly fuzz a target function
+    match function_index % 10 {
+        0 => {
+            // Fuzz compose with two distinct characters
+            if let (Some(c1), Some(c2)) = (chars.next(), chars.next()) {
+                let _ = compose(c1, c2);
+            }
+        }
+        1 => {
+            // Fuzz canonical_combining_class
+            if let Some(c) = chars.next() {
+                let _ = canonical_combining_class(c);
+            }
+        }
+        2 => {
+            // Fuzz is_combining_mark
+            if let Some(c) = chars.next() {
+                let _ = is_combining_mark(c);
+            }
+        }
+        3 => {
+            // Fuzz NFC
+            let _ = string_data.nfc().collect::<String>();
+        }
+        4 => {
+            // Fuzz NFKD
+            let _ = string_data.nfkd().collect::<String>();
+        }
+        5 => {
+            // Fuzz NFD
+            let _ = string_data.nfd().collect::<String>();
+        }
+        6 => {
+            // Fuzz NFKC
+            let _ = string_data.nfkc().collect::<String>();
+        }
+        7 => {
+            // Fuzz stream_safe
+            let _ = string_data.stream_safe().collect::<String>();
+        }
+        8 => {
+            // Fuzz decompose_canonical
+            if let Some(c) = chars.next() {
+                decompose_canonical(c, |_| {});
+            }
+        }
+        9 => {
+            // Fuzz decompose_compatible
+            if let Some(c) = chars.next() {
+                decompose_compatible(c, |_| {});
+            }
+        }
+        _ => {}
+    }
+});
diff --git a/src/decompose.rs b/src/decompose.rs
@@ -53,7 +53,7 @@ impl<I: Iterator<Item = char>> Decompositions<I> {
 
     /// Create a new decomposition iterator for compatability decompositions (NFkD)
     ///
-    /// Note that this iterator can also be obtained by directly calling [`.nfd()`](crate::UnicodeNormalization::nfd)
+    /// Note that this iterator can also be obtained by directly calling [`.nfkd()`](crate::UnicodeNormalization::nfkd)
     /// on the iterator.
     #[inline]
     pub fn new_compatible(iter: I) -> Decompositions<I> {
diff --git a/src/lib.rs b/src/lib.rs
@@ -113,8 +113,8 @@ pub trait UnicodeNormalization<I: Iterator<Item = char>> {
     /// (compatibility decomposition followed by canonical composition).
     fn nfkc(self) -> Recompositions<I>;
 
-    /// A transformation which replaces CJK Compatibility Ideograph codepoints
-    /// with normal forms using Standardized Variation Sequences. This is not
+    /// A transformation which replaces [CJK Compatibility Ideograph] codepoints
+    /// with normal forms using [Standardized Variation Sequences]. This is not
     /// part of the canonical or compatibility decomposition algorithms, but
     /// performing it before those algorithms produces normalized output which
     /// better preserves the intent of the original text.
@@ -123,10 +123,15 @@ pub trait UnicodeNormalization<I: Iterator<Item = char>> {
     /// may not immediately help text display as intended, but they at
     /// least preserve the information in a standardized form, giving
     /// implementations the option to recognize them.
+    ///
+    /// [CJK Compatibility Ideograph]: https://www.unicode.org/glossary/#compatibility_ideograph
+    /// [Standardized Variation Sequences]: https://www.unicode.org/glossary/#standardized_variation_sequence
     fn cjk_compat_variants(self) -> Replacements<I>;
 
     /// An Iterator over the string with Conjoining Grapheme Joiner characters
-    /// inserted according to the Stream-Safe Text Process (UAX15-D4)
+    /// inserted according to the Stream-Safe Text Process ([UAX15-D4]).
+    ///
+    /// [UAX15-D4]: https://www.unicode.org/reports/tr15/#UAX15-D4
     fn stream_safe(self) -> StreamSafe<I>;
 }
 
@@ -153,7 +158,7 @@ impl<'a> UnicodeNormalization<Chars<'a>> for &'a str {
 
     #[inline]
     fn cjk_compat_variants(self) -> Replacements<Chars<'a>> {
-        replace::new_cjk_compat_variants(self.chars())
+        Replacements::new_cjk_compat_variants(self.chars())
     }
 
     #[inline]
@@ -185,7 +190,7 @@ impl UnicodeNormalization<option::IntoIter<char>> for char {
 
     #[inline]
     fn cjk_compat_variants(self) -> Replacements<option::IntoIter<char>> {
-        replace::new_cjk_compat_variants(Some(self).into_iter())
+        Replacements::new_cjk_compat_variants(Some(self).into_iter())
     }
 
     #[inline]
@@ -217,7 +222,7 @@ impl<I: Iterator<Item = char>> UnicodeNormalization<I> for I {
 
     #[inline]
     fn cjk_compat_variants(self) -> Replacements<I> {
-        replace::new_cjk_compat_variants(self)
+        Replacements::new_cjk_compat_variants(self)
     }
 
     #[inline]
diff --git a/src/normalize.rs b/src/normalize.rs
@@ -41,11 +41,12 @@ pub fn decompose_compatible<F: FnMut(char)>(c: char, emit_char: F) {
 ///
 /// [Standardized Variation Sequences] are used instead of the standard canonical
 /// decompositions, notably for CJK codepoints with singleton canonical decompositions,
-/// to avoid losing information. See the
-/// [Unicode Variation Sequence FAQ](http://unicode.org/faq/vs.html) and the
-/// "Other Enhancements" section of the
-/// [Unicode 6.3 Release Summary](https://www.unicode.org/versions/Unicode6.3.0/#Summary)
-/// for more information.
+/// to avoid losing information. See the [Unicode Variation Sequence FAQ] and the
+/// "Other Enhancements" section of the [Unicode 6.3 Release Summary] for more information.
+///
+/// [Standardized Variation Sequences]: https://www.unicode.org/glossary/#standardized_variation_sequence
+/// [Unicode Variation Sequence FAQ]: http://unicode.org/faq/vs.html
+/// [Unicode 6.3 Release Summary]: https://www.unicode.org/versions/Unicode6.3.0/#Summary
 #[inline]
 pub fn decompose_cjk_compat_variants<F>(c: char, mut emit_char: F)
 where
diff --git a/src/replace.rs b/src/replace.rs
@@ -22,9 +22,18 @@ pub struct Replacements<I> {
     buffer: Option<char>,
 }
 
-#[inline]
-pub fn new_cjk_compat_variants<I: Iterator<Item = char>>(iter: I) -> Replacements<I> {
-    Replacements { iter, buffer: None }
+impl<I: Iterator<Item = char>> Replacements<I> {
+    /// Create a new iterator that replaces [CJK Compatibility Ideograph] codepoints with normal forms using [Standardized Variation Sequences].
+    ///
+    /// Note that this iterator can also be obtained by directly calling [`.cjk_compat_variants()`] on the iterator.
+    ///
+    /// [CJK Compatibility Ideograph]: https://www.unicode.org/glossary/#compatibility_ideograph
+    /// [Standardized Variation Sequences]: https://www.unicode.org/glossary/#standardized_variation_sequence
+    /// [`.cjk_compat_variants()`]: crate::UnicodeNormalization::cjk_compat_variants
+    #[inline]
+    pub fn new_cjk_compat_variants(iter: I) -> Replacements<I> {
+        Replacements { iter, buffer: None }
+    }
 }
 
 impl<I: Iterator<Item = char>> Iterator for Replacements<I> {
diff --git a/src/stream_safe.rs b/src/stream_safe.rs
@@ -10,17 +10,24 @@ use crate::tables::stream_safe_leading_nonstarters;
 pub(crate) const MAX_NONSTARTERS: usize = 30;
 const COMBINING_GRAPHEME_JOINER: char = '\u{034F}';
 
-/// UAX15-D4: This iterator keeps track of how many non-starters there have been
+/// [UAX15-D4]: This iterator keeps track of how many non-starters there have been
 /// since the last starter in *NFKD* and will emit a Combining Grapheme Joiner
 /// (U+034F) if the count exceeds 30.
+///
+/// [UAX15-D4]: https://www.unicode.org/reports/tr15/#UAX15-D4
 pub struct StreamSafe<I> {
     iter: I,
     nonstarter_count: usize,
     buffer: Option<char>,
 }
 
-impl<I> StreamSafe<I> {
-    pub(crate) fn new(iter: I) -> Self {
+impl<I: Iterator<Item = char>> StreamSafe<I> {
+    /// Create a new stream safe iterator.
+    ///
+    /// Note that this iterator can also be obtained by directly calling [`.stream_safe()`](crate::UnicodeNormalization::stream_safe)
+    /// on the iterator.
+    #[inline]
+    pub fn new(iter: I) -> Self {
         Self {
             iter,
             nonstarter_count: 0,
