diff --git a/docs/overview.md b/docs/overview.md index 5a95765..7290f3a 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -73,6 +73,7 @@ Use Nexus Terminal when you need to: - Shift+PageUp/PageDown/Home/End and the mouse wheel scroll the scrollback; on the alternate screen the wheel sends arrow keys, so `tmux`, `screen` and `byobu` scroll naturally. - Context menu with Copy, Paste, Select All, Find, Send Snippet, Port Forwards, Split Right, Split Down, Export Output, Clear and Reset. - Export the whole scrollback as plain text. +- Inline images in the iTerm2 format (`imgcat`, `wezterm imgcat`, `chafa -f iterm`…): PNG, JPEG and GIF, sized in cells, pixels or percent, scrolling with the text. - Shell integration (OSC 133): jump between prompts, copy the last command's output, and see each command's exit status and duration beside its prompt. - Status-bar latency for the active SSH session. diff --git a/docs/user-manual.md b/docs/user-manual.md index 04adf39..d05d7df 100644 --- a/docs/user-manual.md +++ b/docs/user-manual.md @@ -218,6 +218,10 @@ A new pane offers **Local Machine** and your terminal profiles (SSH, Telnet and ## 6. Terminal actions and search +### Inline images + +Programs that print images in the iTerm2 format (`imgcat`, `wezterm imgcat`, `chafa -f iterm`) show PNG, JPEG and GIF (first frame) images inline, sized as the program asks and never wider than the terminal; the text continues below the image. Images scroll with the text and are dropped with the scrollback. Very large images (over 8192 pixels a side) are ignored, and each terminal keeps at most 32 megapixels of images, dropping the oldest. Sixel and kitty graphics aren't supported. + ### Shell integration When the shell marks its prompts and commands (the OSC 133 sequences), Nexus Terminal knows where each command starts and ends: diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 22b7d3e..8f37cfa 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1101,6 +1101,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.8" @@ -2333,6 +2339,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gl_generator" version = "0.14.0" @@ -2847,6 +2863,8 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", + "color_quant", + "gif", "moxcms", "num-traits", "png", @@ -3696,6 +3714,7 @@ version = "0.1.0" dependencies = [ "alacritty_terminal", "arboard", + "base64 0.22.1", "chrono", "eframe", "egui", diff --git a/rust/app/Cargo.toml b/rust/app/Cargo.toml index 5f9d918..b960403 100644 --- a/rust/app/Cargo.toml +++ b/rust/app/Cargo.toml @@ -18,7 +18,8 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "std", "ansi"] } egui-phosphor = "0.14" rfd = "0.17" -image = { version = "0.25", default-features = false, features = ["png"] } +image = { version = "0.25", default-features = false, features = ["png", "gif"] } +base64 = "0.22" serde = { version = "1", features = ["derive"] } serde_json = "1" regex = "1" diff --git a/rust/app/src/osc.rs b/rust/app/src/osc.rs index e58e1ae..3987895 100644 --- a/rust/app/src/osc.rs +++ b/rust/app/src/osc.rs @@ -12,10 +12,20 @@ pub enum Osc { Finished(Option), /// 7;file://host/path Cwd { host: String, path: String }, + /// 1337;File=…;inline=1: (iTerm2 inline image): the decoded file bytes and the + /// requested width/height arguments (cells, `Npx`, `N%` or `auto`). + Image { data: Vec, width: String, height: String }, } /// Longest OSC body collected; longer ones (not ours) are dropped. const MAX_OSC: usize = 4096; +/// Inline images are bigger: up to 16 MiB of base64. +const MAX_IMAGE_OSC: usize = 16 << 20; +const IMAGE_PREFIX: &[u8] = b"1337;File="; + +fn limit(body: &[u8]) -> usize { + if body.len() >= IMAGE_PREFIX.len() && body.starts_with(IMAGE_PREFIX) { MAX_IMAGE_OSC } else { MAX_OSC } +} #[derive(Default)] enum State { @@ -59,7 +69,7 @@ impl OscScanner { State::Idle } 0x1b => State::BodyEsc(b), - c if b.len() < MAX_OSC => { + c if b.len() < limit(&b) => { b.push(c); State::Body(b) } @@ -95,6 +105,16 @@ fn parse(body: &[u8]) -> Option { _ => None, }; } + if let Some(rest) = s.strip_prefix("1337;File=") { + let (args, data) = rest.split_once(':')?; + let arg = |k: &str| args.split(';').find_map(|a| a.strip_prefix(k)?.strip_prefix('=')).map(str::to_string); + if arg("inline").as_deref() != Some("1") { + return None; + } + use base64::Engine; + let data = base64::engine::general_purpose::STANDARD.decode(data.trim()).ok()?; + return Some(Osc::Image { data, width: arg("width").unwrap_or_default(), height: arg("height").unwrap_or_default() }); + } let url = s.strip_prefix("7;")?.strip_prefix("file://")?; let (host, path) = url.split_at(url.find('/')?); Some(Osc::Cwd { host: host.into(), path: percent_decode(path) }) @@ -144,4 +164,19 @@ mod tests { long.extend(b"\x07\x1b]133;D\x07"); assert_eq!(s.scan(&long).into_iter().map(|(_, o)| o).collect::>(), vec![Osc::Finished(None)]); } + + #[test] + fn inline_images_need_inline_1_and_valid_base64() { + let img = |args: &str, data: &str| { + let seq = format!("\x1b]1337;File={args}:{data}\x07"); + OscScanner::default().scan(seq.as_bytes()).into_iter().map(|(_, o)| o).collect::>() + }; + assert_eq!(img("name=eA==;width=10;inline=1", "aGk="), vec![Osc::Image { data: b"hi".to_vec(), width: "10".into(), height: String::new() }]); + assert!(img("inline=0", "aGk=").is_empty(), "downloads (inline=0) are ignored"); + assert!(img("inline=1", "!!!").is_empty()); + // A large image body is collected, past the 4 KiB cap for other OSCs. + use base64::Engine; + let big = base64::engine::general_purpose::STANDARD.encode(vec![7u8; 100_000]); + assert!(matches!(&img("inline=1", &big)[..], [Osc::Image { data, .. }] if data.len() == 100_000)); + } } diff --git a/rust/app/src/terminal.rs b/rust/app/src/terminal.rs index 5b4236c..23ab03f 100644 --- a/rust/app/src/terminal.rs +++ b/rust/app/src/terminal.rs @@ -118,8 +118,25 @@ pub struct TermView { max_history: usize, /// The working directory the shell last reported (OSC 7): (host, path). pub reported_cwd: Option<(String, String)>, + /// Inline images (iTerm2 OSC 1337), oldest first. + images: Vec, + /// Cell size in points from the last frame, for sizing images that arrive between frames. + cell_px: (f32, f32), } +struct InlineImage { + stable: i64, + col: usize, + cols: usize, + rows: usize, + pixels: Option, + texture: Option, + size: usize, +} + +/// Pixels of inline images kept per terminal; the oldest go first. +const MAX_IMAGE_PIXELS: usize = 32 << 20; + /// One command as shell integration reported it. Lines are stable indexes (see `stable`). #[derive(Clone, Debug)] struct PromptMark { @@ -165,6 +182,8 @@ impl TermView { rotated: 0, max_history: scrollback.min(100_000), reported_cwd: None, + images: Vec::new(), + cell_px: (8.0, 16.0), } } @@ -182,7 +201,7 @@ impl TermView { /// Feeds the VT parser, counting lines a full scrollback rotates out (one per line feed at the /// bottom of the screen) so shell-integration marks stay on their lines. fn advance(&mut self, bytes: &[u8]) { - if self.prompts.is_empty() || self.term.mode().contains(TermMode::ALT_SCREEN) { + if (self.prompts.is_empty() && self.images.is_empty()) || self.term.mode().contains(TermMode::ALT_SCREEN) { self.parser.advance(&mut self.term, bytes); return; } @@ -204,6 +223,7 @@ impl TermView { if self.term.grid().history_size() < h0 { // The scrollback was cleared: earlier marks point at nothing. self.prompts.clear(); + self.images.clear(); } } @@ -249,6 +269,88 @@ impl TermView { } } Osc::Cwd { host, path } => self.reported_cwd = Some((host, path)), + Osc::Image { data, width, height } => self.place_image(&data, &width, &height), + } + } + + /// Decodes an inline image (PNG, JPEG or GIF, within size limits), anchors it at the cursor and + /// moves the cursor below it, as iTerm2 does. + fn place_image(&mut self, data: &[u8], width: &str, height: &str) { + if self.term.mode().contains(TermMode::ALT_SCREEN) { + return; + } + let decoded = (|| { + let mut r = image::ImageReader::new(std::io::Cursor::new(data)).with_guessed_format().ok()?; + let mut limits = image::Limits::default(); + limits.max_image_width = Some(8192); + limits.max_image_height = Some(8192); + limits.max_alloc = Some(256 << 20); + r.limits(limits); + Some(r.decode().ok()?.to_rgba8()) + })(); + let Some(img) = decoded else { return }; + let (cw, ch) = self.cell_px; + let (iw, ih) = (img.width() as f32, img.height() as f32); + let full_w = self.cols as f32 * cw; + let dim = |arg: &str, full: f32, cell: f32| -> Option { + let a = arg.trim(); + if a.is_empty() || a == "auto" { + None + } else if let Some(px) = a.strip_suffix("px") { + px.parse().ok() + } else if let Some(pc) = a.strip_suffix('%') { + pc.parse::().ok().map(|p| full * p / 100.0) + } else { + a.parse::().ok().map(|n| n * cell) + } + }; + let (w, h) = match (dim(width, full_w, cw), dim(height, self.rows as f32 * ch, ch)) { + (Some(w), Some(h)) => (w, h), + (Some(w), None) => (w, w * ih / iw), + (None, Some(h)) => (h * iw / ih, h), + (None, None) => (iw, ih), + }; + // Never wider than the terminal; keep the aspect ratio when shrinking. + let scale = (full_w / w).min(1.0); + let (w, h) = (w * scale, h * scale); + let cols = ((w / cw).ceil() as usize).clamp(1, self.cols as usize); + let rows = ((h / ch).ceil() as usize).clamp(1, 1000); + let size = img.width() as usize * img.height() as usize; + while !self.images.is_empty() && self.images.iter().map(|i| i.size).sum::() + size > MAX_IMAGE_PIXELS { + self.images.remove(0); + } + let pixels = egui::ColorImage::from_rgba_unmultiplied([img.width() as usize, img.height() as usize], img.as_raw()); + let (stable, col) = (self.cursor_stable(), self.term.grid().cursor.point.column.0); + self.images.push(InlineImage { stable, col, cols, rows, pixels: Some(pixels), texture: None, size }); + // Leave room: the cursor ends up on the line below the image. + self.advance(&b"\r\n".repeat(rows)); + } + + fn paint_images(&mut self, ui: &Ui, rect: Rect, origin: Pos2, cw: f32, ch: f32) { + let top = self.term.grid().topmost_line().0; + let offset = self.term.grid().display_offset() as i32; + // Line of a stable index, without borrowing self (the loop below mutates the images). + let (rotated, history) = (self.rotated, self.term.grid().history_size() as i64); + let line_of = |stable: i64| (stable - rotated - history) as i32; + // Images that scrolled out of the scrollback are gone with it. + self.images.retain(|i| line_of(i.stable) + i.rows as i32 > top); + let painter = ui.painter_at(rect); + let rows = self.rows as i32; + for img in &mut self.images { + let vr = line_of(img.stable) + offset; + if vr + img.rows as i32 <= 0 || vr >= rows { + continue; + } + let at = Rect::from_min_size(origin + vec2(img.col as f32 * cw, vr as f32 * ch), vec2(img.cols as f32 * cw, img.rows as f32 * ch)); + let tex = img.texture.get_or_insert_with(|| { + let px = img.pixels.take().unwrap_or_default(); + ui.ctx().load_texture("term-image", px, egui::TextureOptions::LINEAR) + }); + // Fit inside the reserved cells, keeping the aspect ratio. + let [tw, th] = tex.size(); + let s = (at.width() / tw as f32).min(at.height() / th as f32); + let shown = Rect::from_min_size(at.min, vec2(tw as f32 * s, th as f32 * s)); + painter.image(tex.id(), shown, Rect::from_min_max(Pos2::ZERO, pos2(1.0, 1.0)), Color32::WHITE); } } @@ -439,6 +541,8 @@ impl TermView { } self.paint(ui, rect, origin, &font, cw, ch, theme, focused); + self.cell_px = (cw, ch); + self.paint_images(ui, rect, origin, cw, ch); // Exit status of each finished command, as a bar beside its prompt (shell integration). let hover = ui.input(|i| i.pointer.hover_pos()); for (bar, exit, took) in self.status_bars(rect, origin, ch) { @@ -1177,6 +1281,26 @@ mod tests { assert_eq!(at(0), None); } + #[test] + fn inline_images_reserve_rows_and_respect_width() { + use base64::Engine; + let mut png = Vec::new(); + image::RgbaImage::from_pixel(40, 32, image::Rgba([255, 0, 0, 255])).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + let b64 = base64::engine::general_purpose::STANDARD.encode(&png); + let mut t = TermView::new(100, 14.0, CursorStyle::Block); + t.cell_px = (8.0, 16.0); + t.feed(format!("x\r\n\x1b]1337;File=inline=1;width=10:{b64}\x07after").as_bytes()); + let img = &t.images[0]; + // 10 cells wide (80 px) keeps the 40x32 aspect: 64 px = 4 rows; the text continues below. + assert_eq!((img.col, img.cols, img.rows), (0, 10, 4)); + assert_eq!(t.line_of(img.stable), Line(1)); + assert_eq!(t.term.grid().cursor.point.line, Line(5)); + assert!(t.line_text(Line(5)).0.starts_with("after")); + // Garbage is ignored rather than placed. + t.feed(b"\x1b]1337;File=inline=1:bm90IGFuIGltYWdl\x07"); + assert_eq!(t.images.len(), 1); + } + #[test] fn feeds_text_and_finds_it() { let mut t = TermView::new(1000, 14.0, CursorStyle::Block);