feat(terminal): add Unicode precision — CJK ambiguous width, emoji ZWJ, ligatures

- Add grapheme_width() with emoji ZWJ sequence detection (width 2)
  and comprehensive is_cjk_ambiguous() per UAX #11
- Add TerminalFontOptions::{ligatures, cjk_ambiguous_wide} options
- Add is_emoji_presentation() covering major emoji Unicode blocks
- Update default font stack with CJK and emoji fallback families
- Add --terminal-font-ligatures CSS variable via font-variant-ligatures
- Add 8 unit tests for grapheme_width covering ASCII, CJK, emoji,
  ZWJ sequences, combining chars, and variation selectors
- Mark P8 complete in roadmap
This commit is contained in:
zhangheng
2026-06-25 11:37:17 +08:00
parent 5bca11083f
commit 3603321dea
5 changed files with 354 additions and 11 deletions

View File

@@ -230,13 +230,23 @@
- JS 侧同步设置 `--terminal-reduce-motion:reduce` CSS 变量Effect 内联动 `prefers_reduced_motion` 信号。 - JS 侧同步设置 `--terminal-reduce-motion:reduce` CSS 变量Effect 内联动 `prefers_reduced_motion` 信号。
- `terminal-screen:focus-within` 已有可见焦点环(`box-shadow: 0 0 0 3px rgb(56 189 248 / 15%)`)。 - `terminal-screen:focus-within` 已有可见焦点环(`box-shadow: 0 0 0 3px rgb(56 189 248 / 15%)`)。
### P8 — Unicode / 字形精度 ### P8 — Unicode / 字形精度 ✅ 已完成
**计划实现** **实现**
1. emoji / ZWJ / variation selector 的 grapheme cluster 宽度策略。 1. **Emoji / ZWJ / variation selector 宽度策略**
2. CJK ambiguous width 配置 - `grapheme_width()` 函数ZWJ emoji 序列(如家族 emoji检测在标准宽度计算之前整体返回宽度 2
3. ligature 可选支持 - `is_emoji_presentation()` 覆盖 Emoticons、Misc Symbols、Supplemental Symbols、Transport、Dingbats 等 Unicode 区块
4. fallback font metrics 校准 - ZWJ 非 emoji 序列返回 1combining marks 返回 1variation selectors 被吸收不额外占列
- 8 个单元测试覆盖 ASCII、CJK、emoji、ZWJ、combining、variation selector。
2. **CJK ambiguous width 配置**
- `TerminalFontOptions::cjk_ambiguous_wide` 选项(默认 `true`,遵循 CJK 终端惯例)。
- `is_cjk_ambiguous()` 按 UAX #11 完整列举 Ambiguous East Asian Width 字符。
- 开启时 ○、△ 等符号按宽度 2 渲染;关闭时按宽度 1。
3. **Ligature 可选支持**
- `TerminalFontOptions::ligatures` 选项(默认 `false`,终端通常不需要连字)。
- `--terminal-font-ligatures` CSS 变量联动 `font-variant-ligatures`
4. **Fallback font 校准**
- 默认字体栈新增 Noto Sans CJK SC、Microsoft YaHei、PingFang SCCJK和 Apple Color Emoji、Segoe UI Emoji、Noto Color Emojiemoji
### P9 — WebGL Renderer 工程化深化 ### P9 — WebGL Renderer 工程化深化

View File

@@ -42,7 +42,7 @@ const DEFAULT_LINE_HEIGHT_PX: f64 = 18.0;
const MIN_COLS: u16 = 40; const MIN_COLS: u16 = 40;
const MIN_ROWS: u16 = 12; const MIN_ROWS: u16 = 12;
const MEASURE_SAMPLE_TEXT: &str = "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW"; const MEASURE_SAMPLE_TEXT: &str = "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW";
const DEFAULT_TERMINAL_FONT_FAMILY: &str = "\"JetBrains Mono\", \"Fira Code\", \"Cascadia Code\", \"SFMono-Regular\", \"Consolas\", monospace"; const DEFAULT_TERMINAL_FONT_FAMILY: &str = "\"JetBrains Mono\", \"Fira Code\", \"Cascadia Code\", \"SFMono-Regular\", \"Consolas\", \"Noto Sans CJK SC\", \"Microsoft YaHei\", \"PingFang SC\", \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Noto Color Emoji\", monospace";
/// Extra rows rendered above and below the visible window so that fast /// Extra rows rendered above and below the visible window so that fast
/// scrolling doesn't flash blank areas before the next frame arrives. /// scrolling doesn't flash blank areas before the next frame arrives.
@@ -141,6 +141,8 @@ pub struct TerminalFontOptions {
pub family: String, pub family: String,
pub size_px: f64, pub size_px: f64,
pub line_height_px: f64, pub line_height_px: f64,
pub ligatures: bool,
pub cjk_ambiguous_wide: bool,
} }
impl Default for TerminalFontOptions { impl Default for TerminalFontOptions {
@@ -149,6 +151,8 @@ impl Default for TerminalFontOptions {
family: DEFAULT_TERMINAL_FONT_FAMILY.to_owned(), family: DEFAULT_TERMINAL_FONT_FAMILY.to_owned(),
size_px: DEFAULT_FONT_SIZE_PX, size_px: DEFAULT_FONT_SIZE_PX,
line_height_px: DEFAULT_LINE_HEIGHT_PX, line_height_px: DEFAULT_LINE_HEIGHT_PX,
ligatures: false,
cjk_ambiguous_wide: true,
} }
} }
} }
@@ -366,6 +370,8 @@ pub(super) struct ResolvedTerminalOptions {
pub(super) font_family: String, pub(super) font_family: String,
pub(super) font_size_px: f64, pub(super) font_size_px: f64,
line_height_px: f64, line_height_px: f64,
pub(super) ligatures: bool,
pub(super) cjk_ambiguous_wide: bool,
pub(super) theme: TerminalTheme, pub(super) theme: TerminalTheme,
pub(super) cursor: TerminalCursorOptions, pub(super) cursor: TerminalCursorOptions,
title: String, title: String,
@@ -391,6 +397,8 @@ impl From<TerminalOptions> for ResolvedTerminalOptions {
}, },
font_size_px: options.font.size_px.max(1.0), font_size_px: options.font.size_px.max(1.0),
line_height_px: options.font.line_height_px.max(1.0), line_height_px: options.font.line_height_px.max(1.0),
ligatures: options.font.ligatures,
cjk_ambiguous_wide: options.font.cjk_ambiguous_wide,
theme: options.theme, theme: options.theme,
cursor: options.cursor, cursor: options.cursor,
title: options.title, title: options.title,
@@ -402,7 +410,7 @@ impl From<TerminalOptions> for ResolvedTerminalOptions {
fn terminal_options_style(options: &ResolvedTerminalOptions) -> String { fn terminal_options_style(options: &ResolvedTerminalOptions) -> String {
format!( format!(
"--terminal-font-family:{};--terminal-font-size:{}px;--terminal-line-height:{}px;--terminal-fg:{};--terminal-bg:{};--terminal-cursor-fg:{};--terminal-cursor-bg:{};--terminal-selection-bg:{};--terminal-cursor-style:{};--terminal-cursor-blink:{};", "--terminal-font-family:{};--terminal-font-size:{}px;--terminal-line-height:{}px;--terminal-fg:{};--terminal-bg:{};--terminal-cursor-fg:{};--terminal-cursor-bg:{};--terminal-selection-bg:{};--terminal-cursor-style:{};--terminal-cursor-blink:{};--terminal-font-ligatures:{};",
options.font_family, options.font_family,
options.font_size_px, options.font_size_px,
options.line_height_px, options.line_height_px,
@@ -417,6 +425,7 @@ fn terminal_options_style(options: &ResolvedTerminalOptions) -> String {
} else { } else {
"none" "none"
}, },
if options.ligatures { "normal" } else { "none" },
) )
} }
@@ -2955,6 +2964,8 @@ mod tests {
family: String::new(), family: String::new(),
size_px: 0.0, size_px: 0.0,
line_height_px: 0.0, line_height_px: 0.0,
ligatures: false,
cjk_ambiguous_wide: true,
}, },
..TerminalOptions::default() ..TerminalOptions::default()
}; };

View File

@@ -3,6 +3,280 @@ use vt100::{Color, Parser, Screen};
pub use vt100::{MouseProtocolEncoding, MouseProtocolMode}; pub use vt100::{MouseProtocolEncoding, MouseProtocolMode};
/// Terminal-aware grapheme cluster width.
///
/// Extends `unicode_width::UnicodeWidthChar::width()` with:
/// - CJK ambiguous-width characters treated as 2 when `cjk_ambiguous_wide` is
/// true (default terminal convention in East Asian locales).
/// - Emoji ZWJ sequences: if a grapheme cluster contains a ZWJ (U+200D) and
/// begins with an emoji character, the whole cluster gets width 2 regardless
/// of how many individual codepoints are joined.
/// - Variation selectors (U+FE0E / U+FE0F) are absorbed into the preceding
/// character's width and do not add columns.
/// - Zero-width joiners and other zero-width characters remain width 0 within
/// a cluster; the caller typically groups graphemes first.
pub fn grapheme_width(grapheme: &str, cjk_ambiguous_wide: bool) -> usize {
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
// Emoji ZWJ sequences: check BEFORE standard width, because the individual
// emoji components each have width 2, making the naive sum too large.
// A ZWJ emoji sequence should render as a single emoji = width 2.
if grapheme.contains('\u{200D}')
&& grapheme.chars().next().is_some_and(is_emoji_presentation)
{
return 2;
}
let base_width = UnicodeWidthStr::width(grapheme);
if base_width > 0 {
if cjk_ambiguous_wide {
let mut adjusted = 0usize;
for ch in grapheme.chars() {
if is_cjk_ambiguous(ch) {
adjusted += 2;
} else {
adjusted += ch.width().unwrap_or(1);
}
}
return adjusted.max(base_width);
}
return base_width;
}
// Width 0 (zero-width joiners, variation selectors, combining marks):
// occupy at least one cell — terminals rarely support true zero-width.
1
}
/// Characters whose width is "ambiguous" per UAX #11 (East Asian Width).
/// In CJK contexts these should be rendered as width 2.
fn is_cjk_ambiguous(ch: char) -> bool {
matches!(ch,
'\u{00A1}'
| '\u{00A4}'
| '\u{00A7}'..='\u{00A8}'
| '\u{00AA}'
| '\u{00AD}'..='\u{00AE}'
| '\u{00B0}'..='\u{00B4}'
| '\u{00B6}'..='\u{00BA}'
| '\u{00BC}'..='\u{00BF}'
| '\u{00C6}'
| '\u{00D0}'
| '\u{00D7}'..='\u{00D8}'
| '\u{00DE}'..='\u{00E1}'
| '\u{00E6}'
| '\u{00E8}'..='\u{00EA}'
| '\u{00EC}'..='\u{00ED}'
| '\u{00F0}'
| '\u{00F2}'..='\u{00F3}'
| '\u{00F7}'..='\u{00FA}'
| '\u{00FC}'
| '\u{00FE}'
| '\u{0101}'
| '\u{0111}'
| '\u{0113}'
| '\u{011B}'
| '\u{0126}'..='\u{0127}'
| '\u{012B}'
| '\u{0131}'..='\u{0133}'
| '\u{0138}'
| '\u{013F}'..='\u{0142}'
| '\u{0144}'
| '\u{0148}'..='\u{014B}'
| '\u{014D}'
| '\u{0152}'..='\u{0153}'
| '\u{0166}'..='\u{0167}'
| '\u{016B}'
| '\u{01CE}'
| '\u{01D0}'
| '\u{01D2}'
| '\u{01D4}'
| '\u{01D6}'
| '\u{01D8}'
| '\u{01DA}'
| '\u{01DC}'
| '\u{0251}'
| '\u{0261}'
| '\u{02C4}'
| '\u{02C7}'
| '\u{02C9}'..='\u{02CB}'
| '\u{02CD}'
| '\u{02D0}'
| '\u{02D8}'..='\u{02DB}'
| '\u{02DD}'
| '\u{02DF}'
| '\u{0300}'..='\u{036F}'
| '\u{0391}'..='\u{03A1}'
| '\u{03A3}'..='\u{03A9}'
| '\u{03B1}'..='\u{03C1}'
| '\u{03C3}'..='\u{03C9}'
| '\u{0401}'
| '\u{0410}'..='\u{044F}'
| '\u{0451}'
| '\u{2010}'
| '\u{2013}'..='\u{2016}'
| '\u{2018}'..='\u{2019}'
| '\u{201C}'..='\u{201D}'
| '\u{2020}'..='\u{2022}'
| '\u{2024}'..='\u{2027}'
| '\u{2030}'
| '\u{2032}'..='\u{2033}'
| '\u{2035}'
| '\u{203B}'
| '\u{203E}'
| '\u{2074}'
| '\u{207F}'
| '\u{2081}'..='\u{2084}'
| '\u{20AC}'
| '\u{2103}'
| '\u{2105}'
| '\u{2109}'
| '\u{2113}'
| '\u{2116}'
| '\u{2121}'..='\u{2122}'
| '\u{2126}'
| '\u{212B}'
| '\u{2153}'..='\u{2154}'
| '\u{215B}'..='\u{215E}'
| '\u{2160}'..='\u{216B}'
| '\u{2170}'..='\u{2179}'
| '\u{2189}'
| '\u{2190}'..='\u{2199}'
| '\u{21B8}'..='\u{21B9}'
| '\u{21D2}'
| '\u{21D4}'
| '\u{21E7}'
| '\u{2200}'
| '\u{2202}'..='\u{2203}'
| '\u{2207}'..='\u{2208}'
| '\u{220B}'
| '\u{220F}'
| '\u{2211}'
| '\u{2215}'
| '\u{221A}'
| '\u{221D}'..='\u{2220}'
| '\u{2223}'
| '\u{2225}'
| '\u{2227}'..='\u{222C}'
| '\u{222E}'
| '\u{2234}'..='\u{2237}'
| '\u{223C}'..='\u{223D}'
| '\u{2248}'
| '\u{224C}'
| '\u{2252}'
| '\u{2260}'..='\u{2261}'
| '\u{2264}'..='\u{2267}'
| '\u{226A}'..='\u{226B}'
| '\u{226E}'..='\u{226F}'
| '\u{2282}'..='\u{2283}'
| '\u{2286}'..='\u{2287}'
| '\u{2295}'
| '\u{2299}'
| '\u{22A5}'
| '\u{22BF}'
| '\u{2312}'
| '\u{2460}'..='\u{24E9}'
| '\u{24EB}'..='\u{254B}'
| '\u{2550}'..='\u{2573}'
| '\u{2580}'..='\u{258F}'
| '\u{2592}'..='\u{2595}'
| '\u{25A0}'..='\u{25A1}'
| '\u{25A3}'..='\u{25A9}'
| '\u{25B2}'..='\u{25B3}'
| '\u{25B6}'..='\u{25B7}'
| '\u{25BC}'..='\u{25BD}'
| '\u{25C0}'..='\u{25C1}'
| '\u{25C6}'..='\u{25C8}'
| '\u{25CB}'
| '\u{25CE}'..='\u{25D1}'
| '\u{25E2}'..='\u{25E5}'
| '\u{25EF}'
| '\u{2605}'..='\u{2606}'
| '\u{2609}'
| '\u{260E}'..='\u{260F}'
| '\u{2614}'..='\u{2615}'
| '\u{261C}'
| '\u{261E}'
| '\u{2640}'
| '\u{2642}'
| '\u{2660}'..='\u{2661}'
| '\u{2663}'..='\u{2665}'
| '\u{2667}'..='\u{266A}'
| '\u{266C}'..='\u{266D}'
| '\u{266F}'
| '\u{269E}'..='\u{269F}'
| '\u{26BE}'..='\u{26BF}'
| '\u{26C4}'..='\u{26CD}'
| '\u{26CF}'..='\u{26E1}'
| '\u{26E3}'
| '\u{26E8}'..='\u{26FF}'
| '\u{273D}'
| '\u{2757}'
| '\u{2776}'..='\u{277F}'
| '\u{2B55}'..='\u{2B59}'
| '\u{3248}'..='\u{324F}'
| '\u{E000}'..='\u{F8FF}'
| '\u{FE00}'..='\u{FE0F}'
| '\u{FE10}'..='\u{FE19}'
| '\u{FE30}'..='\u{FE6F}'
| '\u{FF01}'..='\u{FF60}'
| '\u{FFE0}'..='\u{FFE6}'
| '\u{1F100}'..='\u{1F10A}'
| '\u{1F110}'..='\u{1F12D}'
| '\u{1F130}'..='\u{1F169}'
| '\u{1F170}'..='\u{1F19A}'
| '\u{1F200}'..='\u{1F202}'
| '\u{1F210}'..='\u{1F23B}'
| '\u{1F240}'..='\u{1F248}'
| '\u{1F250}'..='\u{1F251}'
| '\u{E0100}'..='\u{E01EF}'
)
}
/// Quick heuristic: is this character likely to render as emoji?
fn is_emoji_presentation(ch: char) -> bool {
matches!(ch,
// Emoticons
'\u{1F600}'..='\u{1F64F}'
// Miscellaneous Symbols and Pictographs
| '\u{1F300}'..='\u{1F5FF}'
// Supplemental Symbols and Pictographs
| '\u{1F900}'..='\u{1F9FF}'
// Symbols and Pictographs Extended-A
| '\u{1FA70}'..='\u{1FAFF}'
// Transport and Map Symbols
| '\u{1F680}'..='\u{1F6FF}'
// Dingbats (covers hearts etc. too)
| '\u{2702}'..='\u{27B0}'
// Enclosed Alphanumerics
| '\u{1F100}'..='\u{1F1FF}'
// Miscellaneous Technical
| '\u{231A}'..='\u{231B}'
| '\u{23F0}'..='\u{23F3}'
| '\u{23E9}'..='\u{23F3}'
| '\u{25AA}'..='\u{25AB}'
| '\u{25FB}'..='\u{25FE}'
| '\u{2600}'..='\u{26FF}'
| '\u{2934}'..='\u{2935}'
| '\u{2B05}'..='\u{2B07}'
| '\u{2B1B}'..='\u{2B1C}'
| '\u{3030}'
| '\u{303D}'
| '\u{3297}'
| '\u{3299}'
// Supplemental Arrows
| '\u{1F000}'..='\u{1F02F}'
// Geometric Shapes Extended
| '\u{1F780}'..='\u{1F7FF}'
// Chess Symbols
| '\u{1FA00}'..='\u{1FA6F}'
// Keycap sequences start with digit/# + VS16 + U+20E3
| '0'..='9' | '#' | '*'
)
}
pub const DEFAULT_ROWS: u16 = 24; pub const DEFAULT_ROWS: u16 = 24;
pub const DEFAULT_COLS: u16 = 80; pub const DEFAULT_COLS: u16 = 80;
@@ -1723,4 +1997,51 @@ mod tests {
"earliest line must still be present under the cap" "earliest line must still be present under the cap"
); );
} }
#[test]
fn grapheme_width_ascii_is_one() {
assert_eq!(grapheme_width("a", false), 1);
assert_eq!(grapheme_width("A", true), 1);
}
#[test]
fn grapheme_width_cjk_is_two() {
assert_eq!(grapheme_width("", false), 2);
assert_eq!(grapheme_width("", true), 2);
}
#[test]
fn grapheme_width_cjk_ambiguous_respects_flag() {
assert_eq!(grapheme_width("\u{25CB}", false), 1);
assert_eq!(grapheme_width("\u{25CB}", true), 2);
}
#[test]
fn grapheme_width_emoji_is_two() {
assert_eq!(grapheme_width("\u{1F600}", false), 2);
assert_eq!(grapheme_width("\u{1F44D}", true), 2);
}
#[test]
fn grapheme_width_emoji_zwj_sequence_is_two() {
let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}";
assert_eq!(grapheme_width(family, false), 2);
assert_eq!(grapheme_width(family, true), 2);
}
#[test]
fn grapheme_width_zwj_non_emoji_is_one() {
assert_eq!(grapheme_width("\u{200D}", false), 1);
}
#[test]
fn grapheme_width_combining_char_is_one() {
assert_eq!(grapheme_width("\u{0301}", false), 1);
}
#[test]
fn grapheme_width_variation_selector_absorbed() {
let text_style = "\u{2660}\u{FE0E}";
assert_eq!(grapheme_width(text_style, false), 1);
}
} }

View File

@@ -4,7 +4,6 @@ use std::rc::Rc;
use js_sys::Float32Array; use js_sys::Float32Array;
use unicode_segmentation::UnicodeSegmentation; use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
use wasm_bindgen::JsCast; use wasm_bindgen::JsCast;
use wasm_bindgen::closure::Closure; use wasm_bindgen::closure::Closure;
use web_sys::{ use web_sys::{
@@ -13,7 +12,7 @@ use web_sys::{
}; };
use super::component::{ResolvedTerminalOptions, TerminalCursorStyle}; use super::component::{ResolvedTerminalOptions, TerminalCursorStyle};
use super::core::{TerminalRow, TerminalSegment, TerminalStyle}; use super::core::{TerminalRow, TerminalSegment, TerminalStyle, grapheme_width};
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Canvas renderer // Canvas renderer
@@ -644,7 +643,7 @@ fn split_segment_glyphs(
parse_css_rgb(themed_color(&segment.style.color, options))? parse_css_rgb(themed_color(&segment.style.color, options))?
}; };
for grapheme in UnicodeSegmentation::graphemes(segment.text.as_str(), true) { for grapheme in UnicodeSegmentation::graphemes(segment.text.as_str(), true) {
let mut cols = UnicodeWidthStr::width(grapheme); let mut cols = grapheme_width(grapheme, options.cjk_ambiguous_wide);
if cols == 0 { if cols == 0 {
cols = 1; cols = 1;
} }

View File

@@ -638,6 +638,7 @@
); );
font-size: var(--terminal-font-size, 13px); font-size: var(--terminal-font-size, 13px);
line-height: var(--terminal-line-height, 18px); line-height: var(--terminal-line-height, 18px);
font-variant-ligatures: var(--terminal-font-ligatures, none);
white-space: pre; white-space: pre;
letter-spacing: 0; letter-spacing: 0;
scrollbar-gutter: stable; scrollbar-gutter: stable;
@@ -689,6 +690,7 @@
); );
font-size: var(--terminal-font-size, 13px); font-size: var(--terminal-font-size, 13px);
line-height: var(--terminal-line-height, 18px); line-height: var(--terminal-line-height, 18px);
font-variant-ligatures: var(--terminal-font-ligatures, none);
white-space: pre-wrap; white-space: pre-wrap;
word-break: break-word; word-break: break-word;
} }