feat: inline images in terminals (iTerm2 OSC 1337)

Images sent with the iTerm2 inline protocol (imgcat and friends) are
decoded (PNG, JPEG, GIF first frame) and drawn at the cursor, sized from
the width/height arguments in cells, pixels or percent (aspect kept,
never wider than the terminal); the cursor moves below so text
continues after them. They are anchored to the same stable line index
as the shell-integration marks, scroll with the text and are dropped
with the scrollback or on clear.

Remote data, so decoding is bounded (8192 px a side, 256 MiB
allocation), OSC bodies are capped at 16 MiB, inline=0 downloads are
ignored, and each terminal keeps at most 32 Mpx of images, oldest
first. Sixel and kitty graphics are not implemented.

docs/overview.md

@@ -73,6 +73,7 @@ Use Nexus Terminal when you need to:
 - Shift+PageUp/PageDown/Home/End and the mouse wheel scroll the scrollback; on the alternate screen the wheel sends arrow keys, so `tmux`, `screen` and `byobu` scroll naturally.
 - Context menu with Copy, Paste, Select All, Find, Send Snippet, Port Forwards, Split Right, Split Down, Export Output, Clear and Reset.
 - Export the whole scrollback as plain text.
+- Inline images in the iTerm2 format (`imgcat`, `wezterm imgcat`, `chafa -f iterm`…): PNG, JPEG and GIF, sized in cells, pixels or percent, scrolling with the text.
 - Shell integration (OSC 133): jump between prompts, copy the last command's output, and see each command's exit status and duration beside its prompt.
 - Status-bar latency for the active SSH session.
 

docs/user-manual.md

@@ -218,6 +218,10 @@ A new pane offers **Local Machine** and your terminal profiles (SSH, Telnet and
 
 ## 6. Terminal actions and search
 
+### Inline images
+
+Programs that print images in the iTerm2 format (`imgcat`, `wezterm imgcat`, `chafa -f iterm`) show PNG, JPEG and GIF (first frame) images inline, sized as the program asks and never wider than the terminal; the text continues below the image. Images scroll with the text and are dropped with the scrollback. Very large images (over 8192 pixels a side) are ignored, and each terminal keeps at most 32 megapixels of images, dropping the oldest. Sixel and kitty graphics aren't supported.
+
 ### Shell integration
 
 When the shell marks its prompts and commands (the OSC 133 sequences), Nexus Terminal knows where each command starts and ends:

rust/Cargo.lock

@@ -1101,6 +1101,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "color_quant"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
+
 [[package]]
 name = "combine"
 version = "4.6.8"
@@ -2333,6 +2339,16 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "gif"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
+dependencies = [
+ "color_quant",
+ "weezl",
+]
+
 [[package]]
 name = "gl_generator"
 version = "0.14.0"
@@ -2847,6 +2863,8 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
 dependencies = [
  "bytemuck",
  "byteorder-lite",
+ "color_quant",
+ "gif",
  "moxcms",
  "num-traits",
  "png",
@@ -3696,6 +3714,7 @@ version = "0.1.0"
 dependencies = [
  "alacritty_terminal",
  "arboard",
+ "base64 0.22.1",
  "chrono",
  "eframe",
  "egui",

rust/app/Cargo.toml

@@ -18,7 +18,8 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "std", "ansi"] }
 egui-phosphor = "0.14"
 rfd = "0.17"
-image = { version = "0.25", default-features = false, features = ["png"] }
+image = { version = "0.25", default-features = false, features = ["png", "gif"] }
+base64 = "0.22"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 regex = "1"

rust/app/src/osc.rs

@@ -12,10 +12,20 @@ pub enum Osc {
     Finished(Option<i32>),
     /// 7;file://host/path
     Cwd { host: String, path: String },
+    /// 1337;File=…;inline=1:<base64> (iTerm2 inline image): the decoded file bytes and the
+    /// requested width/height arguments (cells, `Npx`, `N%` or `auto`).
+    Image { data: Vec<u8>, width: String, height: String },
 }
 
 /// Longest OSC body collected; longer ones (not ours) are dropped.
 const MAX_OSC: usize = 4096;
+/// Inline images are bigger: up to 16 MiB of base64.
+const MAX_IMAGE_OSC: usize = 16 << 20;
+const IMAGE_PREFIX: &[u8] = b"1337;File=";
+
+fn limit(body: &[u8]) -> usize {
+    if body.len() >= IMAGE_PREFIX.len() && body.starts_with(IMAGE_PREFIX) { MAX_IMAGE_OSC } else { MAX_OSC }
+}
 
 #[derive(Default)]
 enum State {
@@ -59,7 +69,7 @@ impl OscScanner {
                         State::Idle
                     }
                     0x1b => State::BodyEsc(b),
-                    c if b.len() < MAX_OSC => {
+                    c if b.len() < limit(&b) => {
                         b.push(c);
                         State::Body(b)
                     }
@@ -95,6 +105,16 @@ fn parse(body: &[u8]) -> Option<Osc> {
             _ => None,
         };
     }
+    if let Some(rest) = s.strip_prefix("1337;File=") {
+        let (args, data) = rest.split_once(':')?;
+        let arg = |k: &str| args.split(';').find_map(|a| a.strip_prefix(k)?.strip_prefix('=')).map(str::to_string);
+        if arg("inline").as_deref() != Some("1") {
+            return None;
+        }
+        use base64::Engine;
+        let data = base64::engine::general_purpose::STANDARD.decode(data.trim()).ok()?;
+        return Some(Osc::Image { data, width: arg("width").unwrap_or_default(), height: arg("height").unwrap_or_default() });
+    }
     let url = s.strip_prefix("7;")?.strip_prefix("file://")?;
     let (host, path) = url.split_at(url.find('/')?);
     Some(Osc::Cwd { host: host.into(), path: percent_decode(path) })
@@ -144,4 +164,19 @@ mod tests {
         long.extend(b"\x07\x1b]133;D\x07");
         assert_eq!(s.scan(&long).into_iter().map(|(_, o)| o).collect::<Vec<_>>(), vec![Osc::Finished(None)]);
     }
+
+    #[test]
+    fn inline_images_need_inline_1_and_valid_base64() {
+        let img = |args: &str, data: &str| {
+            let seq = format!("\x1b]1337;File={args}:{data}\x07");
+            OscScanner::default().scan(seq.as_bytes()).into_iter().map(|(_, o)| o).collect::<Vec<_>>()
+        };
+        assert_eq!(img("name=eA==;width=10;inline=1", "aGk="), vec![Osc::Image { data: b"hi".to_vec(), width: "10".into(), height: String::new() }]);
+        assert!(img("inline=0", "aGk=").is_empty(), "downloads (inline=0) are ignored");
+        assert!(img("inline=1", "!!!").is_empty());
+        // A large image body is collected, past the 4 KiB cap for other OSCs.
+        use base64::Engine;
+        let big = base64::engine::general_purpose::STANDARD.encode(vec![7u8; 100_000]);
+        assert!(matches!(&img("inline=1", &big)[..], [Osc::Image { data, .. }] if data.len() == 100_000));
+    }
 }

rust/app/src/terminal.rs

@@ -118,8 +118,25 @@ pub struct TermView {
     max_history: usize,
     /// The working directory the shell last reported (OSC 7): (host, path).
     pub reported_cwd: Option<(String, String)>,
+    /// Inline images (iTerm2 OSC 1337), oldest first.
+    images: Vec<InlineImage>,
+    /// Cell size in points from the last frame, for sizing images that arrive between frames.
+    cell_px: (f32, f32),
 }
 
+struct InlineImage {
+    stable: i64,
+    col: usize,
+    cols: usize,
+    rows: usize,
+    pixels: Option<egui::ColorImage>,
+    texture: Option<egui::TextureHandle>,
+    size: usize,
+}
+
+/// Pixels of inline images kept per terminal; the oldest go first.
+const MAX_IMAGE_PIXELS: usize = 32 << 20;
+
 /// One command as shell integration reported it. Lines are stable indexes (see `stable`).
 #[derive(Clone, Debug)]
 struct PromptMark {
@@ -165,6 +182,8 @@ impl TermView {
             rotated: 0,
             max_history: scrollback.min(100_000),
             reported_cwd: None,
+            images: Vec::new(),
+            cell_px: (8.0, 16.0),
         }
     }
 
@@ -182,7 +201,7 @@ impl TermView {
     /// Feeds the VT parser, counting lines a full scrollback rotates out (one per line feed at the
     /// bottom of the screen) so shell-integration marks stay on their lines.
     fn advance(&mut self, bytes: &[u8]) {
-        if self.prompts.is_empty() || self.term.mode().contains(TermMode::ALT_SCREEN) {
+        if (self.prompts.is_empty() && self.images.is_empty()) || self.term.mode().contains(TermMode::ALT_SCREEN) {
             self.parser.advance(&mut self.term, bytes);
             return;
         }
@@ -204,6 +223,7 @@ impl TermView {
         if self.term.grid().history_size() < h0 {
             // The scrollback was cleared: earlier marks point at nothing.
             self.prompts.clear();
+            self.images.clear();
         }
     }
 
@@ -249,6 +269,88 @@ impl TermView {
                 }
             }
             Osc::Cwd { host, path } => self.reported_cwd = Some((host, path)),
+            Osc::Image { data, width, height } => self.place_image(&data, &width, &height),
+        }
+    }
+
+    /// Decodes an inline image (PNG, JPEG or GIF, within size limits), anchors it at the cursor and
+    /// moves the cursor below it, as iTerm2 does.
+    fn place_image(&mut self, data: &[u8], width: &str, height: &str) {
+        if self.term.mode().contains(TermMode::ALT_SCREEN) {
+            return;
+        }
+        let decoded = (|| {
+            let mut r = image::ImageReader::new(std::io::Cursor::new(data)).with_guessed_format().ok()?;
+            let mut limits = image::Limits::default();
+            limits.max_image_width = Some(8192);
+            limits.max_image_height = Some(8192);
+            limits.max_alloc = Some(256 << 20);
+            r.limits(limits);
+            Some(r.decode().ok()?.to_rgba8())
+        })();
+        let Some(img) = decoded else { return };
+        let (cw, ch) = self.cell_px;
+        let (iw, ih) = (img.width() as f32, img.height() as f32);
+        let full_w = self.cols as f32 * cw;
+        let dim = |arg: &str, full: f32, cell: f32| -> Option<f32> {
+            let a = arg.trim();
+            if a.is_empty() || a == "auto" {
+                None
+            } else if let Some(px) = a.strip_suffix("px") {
+                px.parse().ok()
+            } else if let Some(pc) = a.strip_suffix('%') {
+                pc.parse::<f32>().ok().map(|p| full * p / 100.0)
+            } else {
+                a.parse::<f32>().ok().map(|n| n * cell)
+            }
+        };
+        let (w, h) = match (dim(width, full_w, cw), dim(height, self.rows as f32 * ch, ch)) {
+            (Some(w), Some(h)) => (w, h),
+            (Some(w), None) => (w, w * ih / iw),
+            (None, Some(h)) => (h * iw / ih, h),
+            (None, None) => (iw, ih),
+        };
+        // Never wider than the terminal; keep the aspect ratio when shrinking.
+        let scale = (full_w / w).min(1.0);
+        let (w, h) = (w * scale, h * scale);
+        let cols = ((w / cw).ceil() as usize).clamp(1, self.cols as usize);
+        let rows = ((h / ch).ceil() as usize).clamp(1, 1000);
+        let size = img.width() as usize * img.height() as usize;
+        while !self.images.is_empty() && self.images.iter().map(|i| i.size).sum::<usize>() + size > MAX_IMAGE_PIXELS {
+            self.images.remove(0);
+        }
+        let pixels = egui::ColorImage::from_rgba_unmultiplied([img.width() as usize, img.height() as usize], img.as_raw());
+        let (stable, col) = (self.cursor_stable(), self.term.grid().cursor.point.column.0);
+        self.images.push(InlineImage { stable, col, cols, rows, pixels: Some(pixels), texture: None, size });
+        // Leave room: the cursor ends up on the line below the image.
+        self.advance(&b"\r\n".repeat(rows));
+    }
+
+    fn paint_images(&mut self, ui: &Ui, rect: Rect, origin: Pos2, cw: f32, ch: f32) {
+        let top = self.term.grid().topmost_line().0;
+        let offset = self.term.grid().display_offset() as i32;
+        // Line of a stable index, without borrowing self (the loop below mutates the images).
+        let (rotated, history) = (self.rotated, self.term.grid().history_size() as i64);
+        let line_of = |stable: i64| (stable - rotated - history) as i32;
+        // Images that scrolled out of the scrollback are gone with it.
+        self.images.retain(|i| line_of(i.stable) + i.rows as i32 > top);
+        let painter = ui.painter_at(rect);
+        let rows = self.rows as i32;
+        for img in &mut self.images {
+            let vr = line_of(img.stable) + offset;
+            if vr + img.rows as i32 <= 0 || vr >= rows {
+                continue;
+            }
+            let at = Rect::from_min_size(origin + vec2(img.col as f32 * cw, vr as f32 * ch), vec2(img.cols as f32 * cw, img.rows as f32 * ch));
+            let tex = img.texture.get_or_insert_with(|| {
+                let px = img.pixels.take().unwrap_or_default();
+                ui.ctx().load_texture("term-image", px, egui::TextureOptions::LINEAR)
+            });
+            // Fit inside the reserved cells, keeping the aspect ratio.
+            let [tw, th] = tex.size();
+            let s = (at.width() / tw as f32).min(at.height() / th as f32);
+            let shown = Rect::from_min_size(at.min, vec2(tw as f32 * s, th as f32 * s));
+            painter.image(tex.id(), shown, Rect::from_min_max(Pos2::ZERO, pos2(1.0, 1.0)), Color32::WHITE);
         }
     }
 
@@ -439,6 +541,8 @@ impl TermView {
         }
 
         self.paint(ui, rect, origin, &font, cw, ch, theme, focused);
+        self.cell_px = (cw, ch);
+        self.paint_images(ui, rect, origin, cw, ch);
         // Exit status of each finished command, as a bar beside its prompt (shell integration).
         let hover = ui.input(|i| i.pointer.hover_pos());
         for (bar, exit, took) in self.status_bars(rect, origin, ch) {
@@ -1177,6 +1281,26 @@ mod tests {
         assert_eq!(at(0), None);
     }
 
+    #[test]
+    fn inline_images_reserve_rows_and_respect_width() {
+        use base64::Engine;
+        let mut png = Vec::new();
+        image::RgbaImage::from_pixel(40, 32, image::Rgba([255, 0, 0, 255])).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
+        let b64 = base64::engine::general_purpose::STANDARD.encode(&png);
+        let mut t = TermView::new(100, 14.0, CursorStyle::Block);
+        t.cell_px = (8.0, 16.0);
+        t.feed(format!("x\r\n\x1b]1337;File=inline=1;width=10:{b64}\x07after").as_bytes());
+        let img = &t.images[0];
+        // 10 cells wide (80 px) keeps the 40x32 aspect: 64 px = 4 rows; the text continues below.
+        assert_eq!((img.col, img.cols, img.rows), (0, 10, 4));
+        assert_eq!(t.line_of(img.stable), Line(1));
+        assert_eq!(t.term.grid().cursor.point.line, Line(5));
+        assert!(t.line_text(Line(5)).0.starts_with("after"));
+        // Garbage is ignored rather than placed.
+        t.feed(b"\x1b]1337;File=inline=1:bm90IGFuIGltYWdl\x07");
+        assert_eq!(t.images.len(), 1);
+    }
+
     #[test]
     fn feeds_text_and_finds_it() {
         let mut t = TermView::new(1000, 14.0, CursorStyle::Block);