keyhammer/
text.rs

1// SPDX-License-Identifier: AGPL-3.0-or-later
2// Copyright (C) 2026 Robson Trasel
3
4//! Text normalisation: case folding and diacritic folding with tables of our
5//! own, and the offset utilities needed to report ranges in the caller's text.
6//!
7//! [`Normalizer`] maps a `&str` to the form the index stores. Build the trie
8//! with [`Trie::build_normalized`](crate::trie::Trie::build_normalized) and
9//! search with [`Searcher::search_text`](crate::search::Searcher::search_text)
10//! so that terms and queries go through the same function.
11//!
12//! What is folded, and what is not, is fixed in `docs/design/unicode.md`
13//! (section 2). In short, the default normaliser:
14//!
15//! - drops diacritics of the letters of Latin-1 Supplement and Latin
16//!   Extended-A (U+00C0 to U+017F), expands `æ`, `œ`, `ß`, `ĳ`, `þ` and the
17//!   ligatures U+FB00 to U+FB06, and drops the combining marks U+0300 to U+036F;
18//! - then folds case (Unicode full case folding) in ASCII and those blocks.
19//!
20//! Everything else (other scripts, digits, punctuation, spaces, control
21//! characters, joiners, emoji) is copied unchanged. There is no locale: `I`
22//! always folds to `i`.
23//!
24//! ```
25//! use keyhammer::text::Normalizer;
26//!
27//! let n = Normalizer::new();
28//! assert_eq!(n.normalize("São Paulo"), "sao paulo");
29//! assert_eq!(n.normalize("Straße"), "strasse");
30//! assert_eq!(n.normalize("Œuvre"), "oeuvre");
31//! assert_eq!(n.normalize("ﬁnal"), "final");
32//! // Case only: accents are kept.
33//! let case = Normalizer::new().with_diacritic_folding(false);
34//! assert_eq!(case.normalize("CAFÉ"), "café");
35//! ```
36
37use alloc::string::String;
38use alloc::vec::Vec;
39use core::ops::Range;
40
41mod tables;
42
43use tables::{BASE, CASE_FOLD, FIRST, LAST};
44
45/// The Combining Diacritical Marks block, dropped by diacritic folding.
46const COMBINING: Range<u32> = 0x300..0x370;
47
48/// The letters of a Latin ligature U+FB00 to U+FB06, or `None`.
49fn ligature(cp: u32) -> Option<&'static str> {
50    Some(match cp {
51        0xFB00 => "ff",
52        0xFB01 => "fi",
53        0xFB02 => "fl",
54        0xFB03 => "ffi",
55        0xFB04 => "ffl",
56        0xFB05 | 0xFB06 => "st",
57        _ => return None,
58    })
59}
60
61/// Folds case and diacritics, identically for terms and queries.
62///
63/// Both foldings are on by default ([`Normalizer::new`]). Diacritic folding
64/// runs first, then case folding; the result depends only on the input and
65/// the two flags, and normalising twice gives the same result as normalising
66/// once.
67///
68/// ```
69/// use keyhammer::text::Normalizer;
70///
71/// let n = Normalizer::new();
72/// assert_eq!(n.normalize("Ação"), "acao");
73/// assert_eq!(n.normalize("naïve CAFÉ"), "naive cafe");
74/// // A decomposed accent (e + U+0301) is dropped too.
75/// assert_eq!(n.normalize("cafe\u{301}"), "cafe");
76/// // No Turkish tailoring: every i-like letter becomes i.
77/// assert_eq!(n.normalize("Iıİi"), "iiii");
78/// // Other scripts are copied unchanged.
79/// assert_eq!(n.normalize("Ωmega"), "Ωmega");
80/// ```
81#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
82pub struct Normalizer {
83    case: bool,
84    diacritics: bool,
85}
86
87impl Default for Normalizer {
88    fn default() -> Self {
89        Self::new()
90    }
91}
92
93impl Normalizer {
94    /// Case folding and diacritic folding.
95    pub const fn new() -> Self {
96        Self {
97            case: true,
98            diacritics: true,
99        }
100    }
101
102    /// The same normaliser with case folding on or off.
103    #[must_use]
104    pub const fn with_case_folding(self, on: bool) -> Self {
105        Self { case: on, ..self }
106    }
107
108    /// The same normaliser with diacritic folding on or off.
109    #[must_use]
110    pub const fn with_diacritic_folding(self, on: bool) -> Self {
111        Self {
112            diacritics: on,
113            ..self
114        }
115    }
116
117    /// Whether case is folded.
118    pub const fn folds_case(self) -> bool {
119        self.case
120    }
121
122    /// Whether diacritics are folded.
123    pub const fn folds_diacritics(self) -> bool {
124        self.diacritics
125    }
126
127    /// Calls `emit` with the characters `c` normalises to (none, one or up to
128    /// three), in order.
129    ///
130    /// ```
131    /// use keyhammer::text::Normalizer;
132    /// let mut out = String::new();
133    /// Normalizer::new().normalize_char('ß', |c| out.push(c));
134    /// assert_eq!(out, "ss");
135    /// ```
136    pub fn normalize_char(self, c: char, mut emit: impl FnMut(char)) {
137        let cp = u32::from(c);
138        if self.diacritics {
139            if COMBINING.contains(&cp) {
140                return;
141            }
142            if (FIRST..=LAST).contains(&cp) {
143                let [a, b] = BASE[(cp - FIRST) as usize];
144                if a != 0 {
145                    self.fold_case(char::from(a), &mut emit);
146                    if b != 0 {
147                        self.fold_case(char::from(b), &mut emit);
148                    }
149                    return;
150                }
151            }
152            if let Some(l) = ligature(cp) {
153                l.chars().for_each(emit);
154                return;
155            }
156        }
157        self.fold_case(c, &mut emit);
158    }
159
160    fn fold_case(self, c: char, emit: &mut impl FnMut(char)) {
161        if !self.case {
162            emit(c);
163            return;
164        }
165        let cp = u32::from(c);
166        match cp {
167            0..0x80 => emit(c.to_ascii_lowercase()),
168            0xB5 => emit('\u{3BC}'),
169            0xDF => {
170                emit('s');
171                emit('s');
172            }
173            0x130 => {
174                emit('i');
175                emit('\u{307}');
176            }
177            0x149 => {
178                emit('\u{2BC}');
179                emit('n');
180            }
181            FIRST..=LAST => {
182                let f = u32::from(CASE_FOLD[(cp - FIRST) as usize]);
183                emit(char::from_u32(f).unwrap_or(c));
184            }
185            _ => match ligature(cp) {
186                Some(l) => l.chars().for_each(emit),
187                None => emit(c),
188            },
189        }
190    }
191
192    /// The normalised form of `s`.
193    pub fn normalize(&self, s: &str) -> String {
194        let mut out = String::new();
195        self.normalize_into(s, &mut out);
196        out
197    }
198
199    /// Clears `out` and writes the normalised form of `s` into it, reusing its
200    /// allocation.
201    pub fn normalize_into(&self, s: &str, out: &mut String) {
202        out.clear();
203        if s.is_ascii() {
204            out.push_str(s);
205            if self.case {
206                out.make_ascii_lowercase();
207            }
208            return;
209        }
210        out.reserve(s.len());
211        for c in s.chars() {
212            self.normalize_char(c, |o| out.push(o));
213        }
214    }
215
216    /// The normalised form of `s`, and in `map` where each of its code points
217    /// came from (see [`SourceMap`]). `map` is cleared first.
218    ///
219    /// ```
220    /// use keyhammer::text::{Normalizer, SourceMap};
221    ///
222    /// let mut map = SourceMap::new();
223    /// let n = Normalizer::new().normalize_mapped("Maße", &mut map);
224    /// assert_eq!(n, "masse");
225    /// // "ss" (output code points 2..4) comes from "ß" (input code point 2).
226    /// assert_eq!(map.to_source(2..4), Some(2..3));
227    /// assert_eq!(map.to_source(3..5), Some(2..4));
228    /// ```
229    pub fn normalize_mapped(&self, s: &str, map: &mut SourceMap) -> String {
230        map.from.clear();
231        let mut out = String::with_capacity(s.len());
232        let mut n = 0;
233        for (i, c) in s.chars().enumerate() {
234            self.normalize_char(c, |o| {
235                out.push(o);
236                map.from.push(i);
237            });
238            n = i + 1;
239        }
240        map.source_len = n;
241        out
242    }
243}
244
245/// Where each code point of a normalised string came from in its source.
246///
247/// Filled by [`Normalizer::normalize_mapped`]. All positions are code-point
248/// indices; [`utf8_range`] and [`utf16_range`] convert a range of the source
249/// to bytes or UTF-16 units.
250#[derive(Clone, Debug, Default, PartialEq, Eq)]
251pub struct SourceMap {
252    from: Vec<usize>,
253    source_len: usize,
254}
255
256impl SourceMap {
257    /// An empty map.
258    pub fn new() -> Self {
259        Self::default()
260    }
261
262    /// Number of code points of the normalised string.
263    pub fn len(&self) -> usize {
264        self.from.len()
265    }
266
267    /// Whether the normalised string is empty.
268    pub fn is_empty(&self) -> bool {
269        self.from.is_empty()
270    }
271
272    /// Number of code points of the source.
273    pub fn source_len(&self) -> usize {
274        self.source_len
275    }
276
277    /// The source code point that produced normalised code point `i`.
278    pub fn source_of(&self, i: usize) -> Option<usize> {
279        self.from.get(i).copied()
280    }
281
282    /// The smallest range of source code points that produced the normalised
283    /// code points `range`. It is extended at the end over source characters
284    /// that produced nothing (a dropped combining mark after the last letter
285    /// belongs to that letter). An empty range maps to an empty range at the
286    /// matching source position. `None` if `range` is not within
287    /// `0..=self.len()` or is reversed.
288    pub fn to_source(&self, range: Range<usize>) -> Option<Range<usize>> {
289        let (a, b) = (range.start, range.end);
290        let n = self.from.len();
291        if a > b || b > n {
292            return None;
293        }
294        let at = |i: usize| self.from.get(i).copied().unwrap_or(self.source_len);
295        if a == b {
296            let p = at(a);
297            return Some(p..p);
298        }
299        let start = self.from[a];
300        let last = self.from[b - 1];
301        let next = at(b);
302        let end = if next > last { next } else { last + 1 };
303        Some(start..end)
304    }
305}
306
307/// The byte offset in `s` of code point index `cp`: `s.len()` for
308/// `cp == s.chars().count()`, `None` beyond that.
309///
310/// ```
311/// assert_eq!(keyhammer::text::utf8_offset("aé€b", 3), Some(6));
312/// ```
313pub fn utf8_offset(s: &str, cp: usize) -> Option<usize> {
314    match s.char_indices().nth(cp) {
315        Some((b, _)) => Some(b),
316        None if s.chars().count() == cp => Some(s.len()),
317        None => None,
318    }
319}
320
321/// The UTF-16 offset (in 16-bit units, as JavaScript counts) in `s` of code
322/// point index `cp`; `None` beyond the end.
323///
324/// ```
325/// // The emoji is one code point and two UTF-16 units.
326/// assert_eq!(keyhammer::text::utf16_offset("a😀b", 2), Some(3));
327/// ```
328pub fn utf16_offset(s: &str, cp: usize) -> Option<usize> {
329    let mut units = 0;
330    let mut chars = s.chars();
331    for _ in 0..cp {
332        units += chars.next()?.len_utf16();
333    }
334    Some(units)
335}
336
337/// A range of code points of `s` as a range of bytes; `None` if it is not
338/// within `s` or is reversed.
339pub fn utf8_range(s: &str, range: Range<usize>) -> Option<Range<usize>> {
340    if range.start > range.end {
341        return None;
342    }
343    Some(utf8_offset(s, range.start)?..utf8_offset(s, range.end)?)
344}
345
346/// A range of code points of `s` as a range of UTF-16 units; `None` if it is
347/// not within `s` or is reversed.
348pub fn utf16_range(s: &str, range: Range<usize>) -> Option<Range<usize>> {
349    if range.start > range.end {
350        return None;
351    }
352    Some(utf16_offset(s, range.start)?..utf16_offset(s, range.end)?)
353}
354
355/// First symbol used for bytes that are not valid UTF-8: byte `b` becomes
356/// `INVALID_BYTE + b`, above every scalar value, so it equals no trie label.
357pub(crate) const INVALID_BYTE: u32 = 0x11_0000;
358
359/// Clears `out` and decodes `bytes` into symbols: the code points of the valid
360/// UTF-8 sequences and `INVALID_BYTE + b` for every other byte `b`. At most
361/// `max + 1` symbols are stored; the return value is the total number of
362/// symbols, stored or not.
363pub(crate) fn decode(bytes: &[u8], out: &mut Vec<u32>, max: usize) -> usize {
364    out.clear();
365    let cap = max.saturating_add(1);
366    if bytes.is_ascii() {
367        out.extend(bytes.iter().take(cap).map(|&b| u32::from(b)));
368        return bytes.len();
369    }
370    let mut n = 0usize;
371    let mut push = |s: u32| {
372        if n < cap {
373            out.push(s);
374        }
375        n += 1;
376    };
377    for chunk in bytes.utf8_chunks() {
378        chunk.valid().chars().for_each(|c| push(u32::from(c)));
379        chunk
380            .invalid()
381            .iter()
382            .for_each(|&b| push(INVALID_BYTE + u32::from(b)));
383    }
384    n
385}
386
387#[cfg(test)]
388mod tests {
389    use super::*;
390    use alloc::string::ToString;
391    use alloc::vec;
392
393    const MODES: [Normalizer; 4] = [
394        Normalizer::new(),
395        Normalizer::new().with_diacritic_folding(false),
396        Normalizer::new().with_case_folding(false),
397        Normalizer::new()
398            .with_case_folding(false)
399            .with_diacritic_folding(false),
400    ];
401
402    fn all_chars() -> impl Iterator<Item = char> {
403        // Under Miri (about 100x slower, run weekly with a 60-minute limit)
404        // only a sample: every 9973rd scalar value plus the tables.
405        let step = if cfg!(miri) { 9973 } else { 1 };
406        (0..=0x10_FFFFu32)
407            .step_by(step)
408            .chain(0xB5..0x180)
409            .filter_map(char::from_u32)
410    }
411
412    fn norm(n: Normalizer, c: char) -> String {
413        let mut s = String::new();
414        n.normalize_char(c, |o| s.push(o));
415        s
416    }
417
418    #[test]
419    fn every_mode_is_idempotent_on_every_scalar_value() {
420        for n in MODES {
421            for c in all_chars() {
422                let once = norm(n, c);
423                assert!(once.chars().count() <= 3, "{c:?}");
424                assert_eq!(n.normalize(&once), once, "{n:?} {c:?} -> {once:?}");
425            }
426        }
427    }
428
429    #[test]
430    fn the_identity_mode_changes_nothing() {
431        let n = MODES[3];
432        for c in all_chars() {
433            assert_eq!(norm(n, c), c.to_string());
434        }
435    }
436
437    #[test]
438    fn case_folding_agrees_with_core_lowercase_except_full_foldings() {
439        let n = Normalizer::new().with_diacritic_folding(false);
440        for c in all_chars() {
441            let cp = u32::from(c);
442            let covered = cp < 0x80 || (FIRST..=LAST).contains(&cp);
443            let got = norm(n, c);
444            let want = match cp {
445                0xB5 => "\u{3BC}".to_string(),
446                0xDF => "ss".to_string(),
447                0x149 => "\u{2BC}n".to_string(),
448                0x17F => "s".to_string(),
449                0xFB00..=0xFB06 => ligature(cp).unwrap_or_default().to_string(),
450                _ if covered => c.to_lowercase().collect(),
451                _ => c.to_string(),
452            };
453            assert_eq!(got, want, "{c:?} U+{cp:04X}");
454        }
455    }
456
457    /// A letter and its other case normalise alike whenever case folding is
458    /// on, so the lowercase and uppercase rows of the tables must agree. The
459    /// exceptions are documented policies: `ı` stays apart from `I` without
460    /// diacritic folding (no Turkic tailoring folds `I` to `i`), and `ŉ`
461    /// becomes `'n` by diacritic folding while its uppercase `ʼN` has a
462    /// modifier apostrophe (U+02BC) that no table covers.
463    #[test]
464    fn a_letter_and_its_other_case_normalise_alike() {
465        let mut checked = 0;
466        for n in [MODES[0], MODES[1]] {
467            for cp in FIRST..=LAST {
468                let Some(c) = char::from_u32(cp) else {
469                    continue;
470                };
471                if !c.is_alphabetic()
472                    || (cp == 0x131 && !n.folds_diacritics())
473                    || (cp == 0x149 && n.folds_diacritics())
474                {
475                    continue;
476                }
477                let upper: String = c.to_uppercase().collect();
478                let lower: String = c.to_lowercase().collect();
479                let want = norm(n, c);
480                assert_eq!(n.normalize(&upper), want, "{n:?} {c:?} vs {upper:?}");
481                assert_eq!(n.normalize(&lower), want, "{n:?} {c:?} vs {lower:?}");
482                checked += 1;
483            }
484        }
485        assert!(checked > 370, "{checked}");
486    }
487
488    #[test]
489    fn portuguese_words_fold_in_both_cases() {
490        let n = Normalizer::new();
491        for (lower, upper, want) in [
492            ("coração", "CORAÇÃO", "coracao"),
493            ("não", "NÃO", "nao"),
494            ("pé", "PÉ", "pe"),
495            ("ônibus", "ÔNIBUS", "onibus"),
496            ("ação", "AÇÃO", "acao"),
497            ("são paulo", "SÃO PAULO", "sao paulo"),
498            ("você", "VOCÊ", "voce"),
499            ("põe", "PÕE", "poe"),
500            ("à", "À", "a"),
501        ] {
502            assert_eq!(n.normalize(lower), want, "{lower}");
503            assert_eq!(n.normalize(upper), want, "{upper}");
504        }
505        // Case only: the accents stay, lowercase.
506        let case = Normalizer::new().with_diacritic_folding(false);
507        for (upper, want) in [
508            ("CORAÇÃO", "coração"),
509            ("ÔNIBUS", "ônibus"),
510            ("VOCÊ", "você"),
511            ("PÕE", "põe"),
512            ("SÃO PAULO", "são paulo"),
513        ] {
514            assert_eq!(case.normalize(upper), want, "{upper}");
515            assert_eq!(case.normalize(want), want, "{want}");
516        }
517    }
518
519    #[test]
520    fn diacritic_folding_leaves_only_ascii_for_the_latin_letters() {
521        let n = Normalizer::new();
522        for cp in FIRST..=LAST {
523            let Some(c) = char::from_u32(cp) else {
524                continue;
525            };
526            let out = norm(n, c);
527            if cp == 0xD7 || cp == 0xF7 {
528                assert_eq!(out, c.to_string(), "× and ÷ are not letters");
529            } else {
530                assert!(
531                    !out.is_empty() && out.is_ascii() && out == out.to_lowercase(),
532                    "U+{cp:04X} -> {out:?}"
533                );
534            }
535        }
536    }
537
538    #[test]
539    fn the_documented_policies_hold() {
540        let n = Normalizer::new();
541        for (input, want) in [
542            ("São Paulo", "sao paulo"),
543            ("AÇÃO", "acao"),
544            ("Ç", "c"),
545            ("café", "cafe"),
546            ("naïve", "naive"),
547            ("Straße", "strasse"),
548            ("MÜLLER", "muller"),
549            ("Æsir", "aesir"),
550            ("cœur", "coeur"),
551            ("Œ", "oe"),
552            ("ﬁ ﬂ ﬀ ﬃ ﬄ ﬅ ﬆ", "fi fl ff ffi ffl st st"),
553            ("ĳ", "ij"),
554            ("Þór", "thor"),
555            ("ðø", "do"),
556            ("Łódź", "lodz"),
557            ("ŉ", "'n"),
558            ("ĸ", "q"),
559            ("ŋ", "n"),
560            ("ſ", "s"),
561            ("İstanbul", "istanbul"),
562            ("ıI", "ii"),
563            ("µ", "\u{3BC}"),
564            ("e\u{301}\u{308}", "e"),
565            ("a\u{200D}b\u{200F}\u{0}", "a\u{200D}b\u{200F}\u{0}"),
566            ("😀ÉÀ", "😀ea"),
567            ("ΑΩ", "ΑΩ"),
568        ] {
569            assert_eq!(n.normalize(input), want, "{input:?}");
570        }
571        let case = Normalizer::new().with_diacritic_folding(false);
572        for (input, want) in [
573            ("CAFÉ", "café"),
574            ("Straße", "strasse"),
575            ("İ", "i\u{307}"),
576            ("I", "i"),
577            ("ı", "ı"),
578            ("Æ", "æ"),
579            ("ﬁ", "fi"),
580            ("ŉ", "\u{2BC}n"),
581            ("e\u{301}", "e\u{301}"),
582        ] {
583            assert_eq!(case.normalize(input), want, "{input:?}");
584        }
585        let accents = Normalizer::new().with_case_folding(false);
586        for (input, want) in [("CAFÉ", "CAFE"), ("Æ", "AE"), ("ß", "ss"), ("ﬁ", "fi")] {
587            assert_eq!(accents.normalize(input), want, "{input:?}");
588        }
589    }
590
591    #[test]
592    fn strings_normalise_as_their_characters() {
593        let s = "Ação ß İ ŉ ﬃ e\u{301} ı Ω 😀 \u{0}";
594        for n in MODES {
595            let mut want = String::new();
596            for c in s.chars() {
597                want.push_str(&norm(n, c));
598            }
599            assert_eq!(n.normalize(s), want);
600            let mut buf = String::from("stale");
601            n.normalize_into(s, &mut buf);
602            assert_eq!(buf, want);
603            let mut map = SourceMap::new();
604            assert_eq!(n.normalize_mapped(s, &mut map), want);
605            assert_eq!(map.len(), want.chars().count());
606            assert_eq!(map.source_len(), s.chars().count());
607            assert_eq!(
608                n.normalize("ABC xyz"),
609                n.normalize_mapped("ABC xyz", &mut map)
610            );
611        }
612    }
613
614    #[test]
615    fn source_maps_cover_expansions_and_dropped_marks() {
616        let n = Normalizer::new();
617        let mut map = SourceMap::new();
618        // Source: c a f e U+0301 ß ; output: c a f e s s
619        let out = n.normalize_mapped("cafe\u{301}ß", &mut map);
620        assert_eq!(out, "cafess");
621        assert_eq!(map.source_of(4), Some(5));
622        assert_eq!(map.source_of(6), None);
623        assert_eq!(map.to_source(0..4), Some(0..5)); // "cafe" takes its accent
624        assert_eq!(map.to_source(3..4), Some(3..5));
625        assert_eq!(map.to_source(4..5), Some(5..6)); // half of "ss" is all of ß
626        assert_eq!(map.to_source(5..6), Some(5..6));
627        assert_eq!(map.to_source(0..6), Some(0..6));
628        assert_eq!(map.to_source(2..2), Some(2..2));
629        assert_eq!(map.to_source(6..6), Some(6..6));
630        assert_eq!(map.to_source(0..7), None);
631        assert_eq!(map.to_source(Range { start: 3, end: 2 }), None);
632        // Nothing produced at all.
633        let out = n.normalize_mapped("\u{301}\u{302}", &mut map);
634        assert!(out.is_empty() && map.is_empty());
635        assert_eq!(map.to_source(0..0), Some(2..2));
636    }
637
638    #[test]
639    fn source_maps_agree_with_brute_force() {
640        let s = "ÀÉ ß\u{301}ﬃxĲ\u{300}ŉ😀";
641        for n in MODES {
642            let mut map = SourceMap::new();
643            let out: Vec<char> = n.normalize_mapped(s, &mut map).chars().collect();
644            let src: Vec<char> = s.chars().collect();
645            for a in 0..=out.len() {
646                for b in a..=out.len() {
647                    let r = map.to_source(a..b).unwrap_or(0..0);
648                    // Normalising the source range yields a superstring of the
649                    // output range, and the range is not wider than needed at
650                    // the start.
651                    let part: String = src[r.clone()].iter().collect();
652                    let got: String = n.normalize(&part);
653                    let want: String = out[a..b].iter().collect();
654                    assert!(got.contains(&want), "{n:?} {a}..{b} -> {r:?}");
655                    if a < b {
656                        assert_eq!(Some(r.start), map.source_of(a));
657                    }
658                }
659            }
660        }
661    }
662
663    #[test]
664    fn offsets_in_bytes_and_utf16_units() {
665        let s = "aé€😀b";
666        let bytes = [0, 1, 3, 6, 10, 11];
667        let units = [0, 1, 2, 3, 5, 6];
668        for cp in 0..=5 {
669            assert_eq!(utf8_offset(s, cp), Some(bytes[cp]));
670            assert_eq!(utf16_offset(s, cp), Some(units[cp]));
671        }
672        assert_eq!(utf8_offset(s, 6), None);
673        assert_eq!(utf16_offset(s, 6), None);
674        assert_eq!(utf8_range(s, 1..4), Some(1..10));
675        assert_eq!(utf16_range(s, 1..4), Some(1..5));
676        assert_eq!(utf8_range(s, 4..9), None);
677        assert_eq!(utf16_range(s, Range { start: 2, end: 1 }), None);
678        assert_eq!(utf8_offset("", 0), Some(0));
679        assert_eq!(utf16_offset("", 1), None);
680        // Against the standard encoders.
681        for cp in 0..=5 {
682            let prefix: String = s.chars().take(cp).collect();
683            assert_eq!(utf8_offset(s, cp), Some(prefix.len()));
684            assert_eq!(utf16_offset(s, cp), Some(prefix.encode_utf16().count()));
685        }
686    }
687
688    #[test]
689    fn decoding_maps_invalid_bytes_above_every_scalar_value() {
690        let mut out = vec![7];
691        assert_eq!(decode(b"abc", &mut out, 10), 3);
692        assert_eq!(out, [97, 98, 99]);
693        assert_eq!(decode("é€".as_bytes(), &mut out, 10), 2);
694        assert_eq!(out, [0xE9, 0x20AC]);
695        // A lone continuation byte and a truncated sequence.
696        assert_eq!(decode(b"a\x80b\xE2\x82", &mut out, 10), 5);
697        assert_eq!(
698            out,
699            [
700                97,
701                INVALID_BYTE + 0x80,
702                98,
703                INVALID_BYTE + 0xE2,
704                INVALID_BYTE + 0x82
705            ]
706        );
707        // Counting goes on past the cap, storing stops at max + 1.
708        assert_eq!(decode(b"abcdef", &mut out, 2), 6);
709        assert_eq!(out, [97, 98, 99]);
710        assert_eq!(decode("ééééé".as_bytes(), &mut out, 2), 5);
711        assert_eq!(out.len(), 3);
712        assert_eq!(decode(b"", &mut out, 0), 0);
713        assert!(out.is_empty());
714    }
715}
keyhammer/text.rs

keyhammer/
text.rs
