feat: inline images in terminals (iTerm2 OSC 1337)

Images sent with the iTerm2 inline protocol (imgcat and friends) are
decoded (PNG, JPEG, GIF first frame) and drawn at the cursor, sized from
the width/height arguments in cells, pixels or percent (aspect kept,
never wider than the terminal); the cursor moves below so text
continues after them. They are anchored to the same stable line index
as the shell-integration marks, scroll with the text and are dropped
with the scrollback or on clear.

Remote data, so decoding is bounded (8192 px a side, 256 MiB
allocation), OSC bodies are capped at 16 MiB, inline=0 downloads are
ignored, and each terminal keeps at most 32 Mpx of images, oldest
first. Sixel and kitty graphics are not implemented.

rust/Cargo.lock

@@ -1101,6 +1101,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "color_quant"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
+
 [[package]]
 name = "combine"
 version = "4.6.8"
@@ -2333,6 +2339,16 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "gif"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
+dependencies = [
+ "color_quant",
+ "weezl",
+]
+
 [[package]]
 name = "gl_generator"
 version = "0.14.0"
@@ -2847,6 +2863,8 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
 dependencies = [
  "bytemuck",
  "byteorder-lite",
+ "color_quant",
+ "gif",
  "moxcms",
  "num-traits",
  "png",
@@ -3696,6 +3714,7 @@ version = "0.1.0"
 dependencies = [
  "alacritty_terminal",
  "arboard",
+ "base64 0.22.1",
  "chrono",
  "eframe",
  "egui",

rust/app/Cargo.toml

@@ -18,7 +18,8 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "std", "ansi"] }
 egui-phosphor = "0.14"
 rfd = "0.17"
-image = { version = "0.25", default-features = false, features = ["png"] }
+image = { version = "0.25", default-features = false, features = ["png", "gif"] }
+base64 = "0.22"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 regex = "1"

rust/app/src/osc.rs

@@ -12,10 +12,20 @@ pub enum Osc {
     Finished(Option<i32>),
     /// 7;file://host/path
     Cwd { host: String, path: String },
+    /// 1337;File=…;inline=1:<base64> (iTerm2 inline image): the decoded file bytes and the
+    /// requested width/height arguments (cells, `Npx`, `N%` or `auto`).
+    Image { data: Vec<u8>, width: String, height: String },
 }
 
 /// Longest OSC body collected; longer ones (not ours) are dropped.
 const MAX_OSC: usize = 4096;
+/// Inline images are bigger: up to 16 MiB of base64.
+const MAX_IMAGE_OSC: usize = 16 << 20;
+const IMAGE_PREFIX: &[u8] = b"1337;File=";
+
+fn limit(body: &[u8]) -> usize {
+    if body.len() >= IMAGE_PREFIX.len() && body.starts_with(IMAGE_PREFIX) { MAX_IMAGE_OSC } else { MAX_OSC }
+}
 
 #[derive(Default)]
 enum State {
@@ -59,7 +69,7 @@ impl OscScanner {
                         State::Idle
                     }
                     0x1b => State::BodyEsc(b),
-                    c if b.len() < MAX_OSC => {
+                    c if b.len() < limit(&b) => {
                         b.push(c);
                         State::Body(b)
                     }
@@ -95,6 +105,16 @@ fn parse(body: &[u8]) -> Option<Osc> {
             _ => None,
         };
     }
+    if let Some(rest) = s.strip_prefix("1337;File=") {
+        let (args, data) = rest.split_once(':')?;
+        let arg = |k: &str| args.split(';').find_map(|a| a.strip_prefix(k)?.strip_prefix('=')).map(str::to_string);
+        if arg("inline").as_deref() != Some("1") {
+            return None;
+        }
+        use base64::Engine;
+        let data = base64::engine::general_purpose::STANDARD.decode(data.trim()).ok()?;
+        return Some(Osc::Image { data, width: arg("width").unwrap_or_default(), height: arg("height").unwrap_or_default() });
+    }
     let url = s.strip_prefix("7;")?.strip_prefix("file://")?;
     let (host, path) = url.split_at(url.find('/')?);
     Some(Osc::Cwd { host: host.into(), path: percent_decode(path) })
@@ -144,4 +164,19 @@ mod tests {
         long.extend(b"\x07\x1b]133;D\x07");
         assert_eq!(s.scan(&long).into_iter().map(|(_, o)| o).collect::<Vec<_>>(), vec![Osc::Finished(None)]);
     }
+
+    #[test]
+    fn inline_images_need_inline_1_and_valid_base64() {
+        let img = |args: &str, data: &str| {
+            let seq = format!("\x1b]1337;File={args}:{data}\x07");
+            OscScanner::default().scan(seq.as_bytes()).into_iter().map(|(_, o)| o).collect::<Vec<_>>()
+        };
+        assert_eq!(img("name=eA==;width=10;inline=1", "aGk="), vec![Osc::Image { data: b"hi".to_vec(), width: "10".into(), height: String::new() }]);
+        assert!(img("inline=0", "aGk=").is_empty(), "downloads (inline=0) are ignored");
+        assert!(img("inline=1", "!!!").is_empty());
+        // A large image body is collected, past the 4 KiB cap for other OSCs.
+        use base64::Engine;
+        let big = base64::engine::general_purpose::STANDARD.encode(vec![7u8; 100_000]);
+        assert!(matches!(&img("inline=1", &big)[..], [Osc::Image { data, .. }] if data.len() == 100_000));
+    }
 }

rust/app/src/terminal.rs

@@ -118,8 +118,25 @@ pub struct TermView {
     max_history: usize,
     /// The working directory the shell last reported (OSC 7): (host, path).
     pub reported_cwd: Option<(String, String)>,
+    /// Inline images (iTerm2 OSC 1337), oldest first.
+    images: Vec<InlineImage>,
+    /// Cell size in points from the last frame, for sizing images that arrive between frames.
+    cell_px: (f32, f32),
 }
 
+struct InlineImage {
+    stable: i64,
+    col: usize,
+    cols: usize,
+    rows: usize,
+    pixels: Option<egui::ColorImage>,
+    texture: Option<egui::TextureHandle>,
+    size: usize,
+}
+
+/// Pixels of inline images kept per terminal; the oldest go first.
+const MAX_IMAGE_PIXELS: usize = 32 << 20;
+
 /// One command as shell integration reported it. Lines are stable indexes (see `stable`).
 #[derive(Clone, Debug)]
 struct PromptMark {
@@ -165,6 +182,8 @@ impl TermView {
             rotated: 0,
             max_history: scrollback.min(100_000),
             reported_cwd: None,
+            images: Vec::new(),
+            cell_px: (8.0, 16.0),
         }
     }
 
@@ -182,7 +201,7 @@ impl TermView {
     /// Feeds the VT parser, counting lines a full scrollback rotates out (one per line feed at the
     /// bottom of the screen) so shell-integration marks stay on their lines.
     fn advance(&mut self, bytes: &[u8]) {
-        if self.prompts.is_empty() || self.term.mode().contains(TermMode::ALT_SCREEN) {
+        if (self.prompts.is_empty() && self.images.is_empty()) || self.term.mode().contains(TermMode::ALT_SCREEN) {
             self.parser.advance(&mut self.term, bytes);
             return;
         }
@@ -204,6 +223,7 @@ impl TermView {
         if self.term.grid().history_size() < h0 {
             // The scrollback was cleared: earlier marks point at nothing.
             self.prompts.clear();
+            self.images.clear();
         }
     }
 
@@ -249,6 +269,88 @@ impl TermView {
                 }
             }
             Osc::Cwd { host, path } => self.reported_cwd = Some((host, path)),
+            Osc::Image { data, width, height } => self.place_image(&data, &width, &height),
+        }
+    }
+
+    /// Decodes an inline image (PNG, JPEG or GIF, within size limits), anchors it at the cursor and
+    /// moves the cursor below it, as iTerm2 does.
+    fn place_image(&mut self, data: &[u8], width: &str, height: &str) {
+        if self.term.mode().contains(TermMode::ALT_SCREEN) {
+            return;
+        }
+        let decoded = (|| {
+            let mut r = image::ImageReader::new(std::io::Cursor::new(data)).with_guessed_format().ok()?;
+            let mut limits = image::Limits::default();
+            limits.max_image_width = Some(8192);
+            limits.max_image_height = Some(8192);
+            limits.max_alloc = Some(256 << 20);
+            r.limits(limits);
+            Some(r.decode().ok()?.to_rgba8())
+        })();
+        let Some(img) = decoded else { return };
+        let (cw, ch) = self.cell_px;
+        let (iw, ih) = (img.width() as f32, img.height() as f32);
+        let full_w = self.cols as f32 * cw;
+        let dim = |arg: &str, full: f32, cell: f32| -> Option<f32> {
+            let a = arg.trim();
+            if a.is_empty() || a == "auto" {
+                None
+            } else if let Some(px) = a.strip_suffix("px") {
+                px.parse().ok()
+            } else if let Some(pc) = a.strip_suffix('%') {
+                pc.parse::<f32>().ok().map(|p| full * p / 100.0)
+            } else {
+                a.parse::<f32>().ok().map(|n| n * cell)
+            }
+        };
+        let (w, h) = match (dim(width, full_w, cw), dim(height, self.rows as f32 * ch, ch)) {
+            (Some(w), Some(h)) => (w, h),
+            (Some(w), None) => (w, w * ih / iw),
+            (None, Some(h)) => (h * iw / ih, h),
+            (None, None) => (iw, ih),
+        };
+        // Never wider than the terminal; keep the aspect ratio when shrinking.
+        let scale = (full_w / w).min(1.0);
+        let (w, h) = (w * scale, h * scale);
+        let cols = ((w / cw).ceil() as usize).clamp(1, self.cols as usize);
+        let rows = ((h / ch).ceil() as usize).clamp(1, 1000);
+        let size = img.width() as usize * img.height() as usize;
+        while !self.images.is_empty() && self.images.iter().map(|i| i.size).sum::<usize>() + size > MAX_IMAGE_PIXELS {
+            self.images.remove(0);
+        }
+        let pixels = egui::ColorImage::from_rgba_unmultiplied([img.width() as usize, img.height() as usize], img.as_raw());
+        let (stable, col) = (self.cursor_stable(), self.term.grid().cursor.point.column.0);
+        self.images.push(InlineImage { stable, col, cols, rows, pixels: Some(pixels), texture: None, size });
+        // Leave room: the cursor ends up on the line below the image.
+        self.advance(&b"\r\n".repeat(rows));
+    }
+
+    fn paint_images(&mut self, ui: &Ui, rect: Rect, origin: Pos2, cw: f32, ch: f32) {
+        let top = self.term.grid().topmost_line().0;
+        let offset = self.term.grid().display_offset() as i32;
+        // Line of a stable index, without borrowing self (the loop below mutates the images).
+        let (rotated, history) = (self.rotated, self.term.grid().history_size() as i64);
+        let line_of = |stable: i64| (stable - rotated - history) as i32;
+        // Images that scrolled out of the scrollback are gone with it.
+        self.images.retain(|i| line_of(i.stable) + i.rows as i32 > top);
+        let painter = ui.painter_at(rect);
+        let rows = self.rows as i32;
+        for img in &mut self.images {
+            let vr = line_of(img.stable) + offset;
+            if vr + img.rows as i32 <= 0 || vr >= rows {
+                continue;
+            }
+            let at = Rect::from_min_size(origin + vec2(img.col as f32 * cw, vr as f32 * ch), vec2(img.cols as f32 * cw, img.rows as f32 * ch));
+            let tex = img.texture.get_or_insert_with(|| {
+                let px = img.pixels.take().unwrap_or_default();
+                ui.ctx().load_texture("term-image", px, egui::TextureOptions::LINEAR)
+            });
+            // Fit inside the reserved cells, keeping the aspect ratio.
+            let [tw, th] = tex.size();
+            let s = (at.width() / tw as f32).min(at.height() / th as f32);
+            let shown = Rect::from_min_size(at.min, vec2(tw as f32 * s, th as f32 * s));
+            painter.image(tex.id(), shown, Rect::from_min_max(Pos2::ZERO, pos2(1.0, 1.0)), Color32::WHITE);
         }
     }
 
@@ -439,6 +541,8 @@ impl TermView {
         }
 
         self.paint(ui, rect, origin, &font, cw, ch, theme, focused);
+        self.cell_px = (cw, ch);
+        self.paint_images(ui, rect, origin, cw, ch);
         // Exit status of each finished command, as a bar beside its prompt (shell integration).
         let hover = ui.input(|i| i.pointer.hover_pos());
         for (bar, exit, took) in self.status_bars(rect, origin, ch) {
@@ -1177,6 +1281,26 @@ mod tests {
         assert_eq!(at(0), None);
     }
 
+    #[test]
+    fn inline_images_reserve_rows_and_respect_width() {
+        use base64::Engine;
+        let mut png = Vec::new();
+        image::RgbaImage::from_pixel(40, 32, image::Rgba([255, 0, 0, 255])).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
+        let b64 = base64::engine::general_purpose::STANDARD.encode(&png);
+        let mut t = TermView::new(100, 14.0, CursorStyle::Block);
+        t.cell_px = (8.0, 16.0);
+        t.feed(format!("x\r\n\x1b]1337;File=inline=1;width=10:{b64}\x07after").as_bytes());
+        let img = &t.images[0];
+        // 10 cells wide (80 px) keeps the 40x32 aspect: 64 px = 4 rows; the text continues below.
+        assert_eq!((img.col, img.cols, img.rows), (0, 10, 4));
+        assert_eq!(t.line_of(img.stable), Line(1));
+        assert_eq!(t.term.grid().cursor.point.line, Line(5));
+        assert!(t.line_text(Line(5)).0.starts_with("after"));
+        // Garbage is ignored rather than placed.
+        t.feed(b"\x1b]1337;File=inline=1:bm90IGFuIGltYWdl\x07");
+        assert_eq!(t.images.len(), 1);
+    }
+
     #[test]
     fn feeds_text_and_finds_it() {
         let mut t = TermView::new(1000, 14.0, CursorStyle::Block);