fix: image processing

apps/api/src/images/image-processing.spec.ts

@@ -2,17 +2,50 @@ import sharp from 'sharp';
 import { createHash } from 'crypto';
 import { IMAGE_SIZE, InvalidImageError, processProductImage } from './image-processing';
 
-/** A white canvas with a coloured rectangle on it, like a studio shot of a product. */
-async function shot(width: number, height: number, product: { width: number; height: number }, format: 'jpeg' | 'png' = 'png') {
-  const box = await sharp({
-    create: { width: product.width, height: product.height, channels: 3, background: { r: 200, g: 30, b: 30 } },
-  }).png().toBuffer();
+const RED = { r: 200, g: 30, b: 30 };
+
+/** A red rectangle, the "product". */
+function box(width: number, height: number) {
+  return sharp({ create: { width, height, channels: 3, background: RED } }).png().toBuffer();
+}
+
+/** A white canvas with a red product on it, like a studio shot; in the middle unless placed elsewhere. */
+async function shot(
+  width: number,
+  height: number,
+  product: { width: number; height: number; left?: number; top?: number },
+  format: 'jpeg' | 'png' = 'png',
+) {
+  const placed = product.left === undefined
+    ? { gravity: 'centre' }
+    : { left: product.left, top: product.top ?? 0 };
   return sharp({ create: { width, height, channels: 3, background: { r: 255, g: 255, b: 255 } } })
-    .composite([{ input: box, gravity: 'centre' }])
+    .composite([{ input: await box(product.width, product.height), ...placed }])
     [format]()
     .toBuffer();
 }
 
+/** Where the red product ended up in a processed picture. */
+async function redBounds(image: Buffer) {
+  const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  let left = info.width, right = -1, top = info.height, bottom = -1;
+  for (let y = 0; y < info.height; y++) {
+    for (let x = 0; x < info.width; x++) {
+      const i = (y * info.width + x) * 4;
+      if (data[i + 3] > 200 && data[i] > 150 && data[i + 1] < 90 && data[i + 2] < 90) {
+        left = Math.min(left, x); right = Math.max(right, x); top = Math.min(top, y); bottom = Math.max(bottom, y);
+      }
+    }
+  }
+  return { left, right: info.width - 1 - right, top, bottom: info.height - 1 - bottom };
+}
+
+async function pixelAt(image: Buffer, x: number, y: number) {
+  const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  const i = (y * info.width + x) * 4;
+  return { r: data[i], g: data[i + 1], b: data[i + 2], alpha: data[i + 3] };
+}
+
 /** A busy photo without a plain background: noise everywhere. */
 async function photo(width: number, height: number) {
   const pixels = Buffer.alloc(width * height * 3);
@@ -42,14 +75,60 @@ describe('processProductImage', () => {
     expect(a.id).toBe(b.id);
   });
 
-  it('fits a tall bottle whole, with transparent sides, after trimming the white background', async () => {
+  it('fits a tall bottle on white whole, centred with an even margin on a white square', async () => {
     // A bottle of 100x400 on a big white sheet: trimmed, it is 1:4, far from square
     const image = await processProductImage(await shot(1000, 1000, { width: 100, height: 400 }));
 
-    // Transparent at the sides, product in the middle from top to bottom
-    expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0);
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255);
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, IMAGE_SIZE - 5)).toBe(255);
+    const bounds = await redBounds(image.bytes);
+    // Top to bottom inside the margin, centred left to right
+    expect(bounds.top).toBeGreaterThanOrEqual(36);
+    expect(bounds.top).toBeLessThanOrEqual(44);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3);
+    // The sides are the photo's own white, not see-through
+    expect(await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).toMatchObject({ r: 255, g: 255, b: 255, alpha: 255 });
+  });
+
+  it('centres a product photographed off-centre on a plain background', async () => {
+    // The glass stands in the top-left corner of the sheet
+    const image = await processProductImage(await shot(1200, 900, { width: 300, height: 300, left: 60, top: 40 }, 'jpeg'));
+
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4);
+    expect(bounds.left).toBeLessThanOrEqual(46);
+  });
+
+  it('centres a cut-out with a transparent background, keeping it transparent', async () => {
+    const cutout = await sharp({ create: { width: 800, height: 800, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } })
+      .composite([{ input: await box(200, 200), left: 550, top: 500 }])
+      .png()
+      .toBuffer();
+
+    const image = await processProductImage(cutout);
+
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3);
+    expect(await alphaAt(image.bytes, 5, 5)).toBe(0);
+  });
+
+  it('crops a photo around its centre, even when one side is a plain wall', async () => {
+    // Left quarter a white wall, the rest busy; the drink (red) in the exact middle
+    const busy = await photo(600, 700);
+    const wall = await sharp({ create: { width: 200, height: 700, channels: 3, background: { r: 255, g: 255, b: 255 } } }).png().toBuffer();
+    const input = await sharp(busy)
+      .composite([{ input: wall, left: 0, top: 0 }, { input: await box(100, 100), gravity: 'centre' }])
+      .jpeg()
+      .toBuffer();
+
+    const image = await processProductImage(input);
+
+    // Nothing trimmed: the wall is still there, and the drink is in the middle
+    expect((await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).r).toBeGreaterThan(240);
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4);
   });
 
   it('crops a roughly square photo to a square without empty sides', async () => {
@@ -68,9 +147,9 @@ describe('processProductImage', () => {
 
     const image = await processProductImage(sideways);
 
-    // Upright the product is tall: fitted whole with transparent sides
-    expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0);
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255);
+    // Upright the product is tall (200 wide, 800 high)
+    const bounds = await redBounds(image.bytes);
+    expect(bounds.left).toBeGreaterThan(bounds.top + 100);
   });
 
   it('drops metadata such as the GPS position', async () => {

apps/api/src/images/image-processing.ts

@@ -10,15 +10,25 @@ export const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 /** Between these width/height ratios a photo is cropped to a square; outside them (bottles, cans) it is fitted whole. */
 const SQUARE_ENOUGH = { min: 0.75, max: 1.33 };
 
+/** Space kept around a product cut out of a plain background, on every side. */
+const MARGIN = 40;
+
 export type ProcessedImage = { id: string; contentType: string; width: number; height: number; bytes: Buffer };
 
 export class InvalidImageError extends Error {}
 
+type Colour = { r: number; g: number; b: number; alpha: number };
+
+const TRANSPARENT: Colour = { r: 0, g: 0, b: 0, alpha: 0 };
+
 /**
- * Turns an upload into a product picture: rotated the right way up (phone photos), a plain background trimmed
- * away, then either cropped to a square around its most interesting part (a roughly square photo of a glass) or
- * fitted whole into a transparent square (a tall bottle, which must never be cut off). Metadata such as GPS
- * position is dropped. The id is a hash of the result, so the same picture always gets the same address.
+ * Turns an upload into a product picture, rotated the right way up (phone photos):
+ * - a product on a plain background (a studio shot on white, a cut-out PNG) is trimmed and centred with an even
+ *   margin, on that same background;
+ * - any other photo is cropped around its centre to a square, or fitted whole into a transparent square when it is
+ *   far from square (a tall bottle must never be cut off).
+ * Metadata such as GPS position is dropped. The id is a hash of the result, so the same picture always gets the
+ * same address.
  */
 export async function processProductImage(input: Buffer): Promise<ProcessedImage> {
   let upright: Buffer;
@@ -28,15 +38,7 @@ export async function processProductImage(input: Buffer): Promise<ProcessedImage
     throw new InvalidImageError('Not an image this server can read (JPEG, PNG, WebP, GIF or AVIF)');
   }
 
-  const product = await trimBackground(upright);
-  const { width = 1, height = 1 } = await sharp(product).metadata();
-  const ratio = width / height;
-  const square = ratio >= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max;
-
-  const { data, info } = await sharp(product)
-    .resize(IMAGE_SIZE, IMAGE_SIZE, square
-      ? { fit: 'cover', position: sharp.strategy.attention }
-      : { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
+  const { data, info } = await ((await cutOut(upright)) ?? (await fitPhoto(upright)))
     .webp({ quality: 82 })
     .toBuffer({ resolveWithObject: true });
 
@@ -49,13 +51,81 @@ export async function processProductImage(input: Buffer): Promise<ProcessedImage
   };
 }
 
-/** Removes a plain border (a studio shot on white). Photos with a busy background come back as they were. */
-async function trimBackground(image: Buffer): Promise<Buffer> {
+/** A product on a plain background: trimmed, then centred with the same margin on every side. Null for a photo. */
+async function cutOut(image: Buffer): Promise<sharp.Sharp | null> {
+  const background = await plainBackground(image);
+  if (!background) return null;
+
+  let product: Buffer;
   try {
-    const { data, info } = await sharp(image).trim({ threshold: 12 }).toBuffer({ resolveWithObject: true });
-    // Trimming down to a sliver means the "background" was most of the picture: keep the original
-    return info.width >= 32 && info.height >= 32 ? data : image;
+    const trimmed = await sharp(image)
+      .trim({ background: background.alpha === 0 ? undefined : background, threshold: 24 })
+      .toBuffer({ resolveWithObject: true });
+    // Trimmed down to a sliver: the "background" was the picture itself
+    if (trimmed.info.width < 32 || trimmed.info.height < 32) return null;
+    product = trimmed.data;
   } catch {
-    return image;
+    return null;
   }
+
+  const inner = IMAGE_SIZE - 2 * MARGIN;
+  const fitted = await sharp(product).resize(inner, inner, { fit: 'contain', background }).toBuffer();
+  return sharp(fitted).extend({ top: MARGIN, bottom: MARGIN, left: MARGIN, right: MARGIN, background });
+}
+
+/** Any other photo: a centred square crop, or fitted whole when it is too tall or wide for that. */
+async function fitPhoto(image: Buffer): Promise<sharp.Sharp> {
+  const { width = 1, height = 1 } = await sharp(image).metadata();
+  const ratio = width / height;
+  const square = ratio >= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max;
+  return sharp(image).resize(IMAGE_SIZE, IMAGE_SIZE, square
+    ? { fit: 'cover', position: 'centre' }
+    : { fit: 'contain', background: TRANSPARENT });
+}
+
+/**
+ * The background colour when all four edges of the picture are one plain colour (or transparent), else null.
+ * A real photo has something different on at least one edge.
+ */
+async function plainBackground(image: Buffer): Promise<Colour | null> {
+  const { width = 0, height = 0 } = await sharp(image).metadata();
+  if (width < 32 || height < 32) return null;
+
+  const strip = Math.max(2, Math.round(Math.min(width, height) * 0.02));
+  const edges = [
+    { left: 0, top: 0, width, height: strip },
+    { left: 0, top: height - strip, width, height: strip },
+    { left: 0, top: 0, width: strip, height },
+    { left: width - strip, top: 0, width: strip, height },
+  ];
+  const stats = await Promise.all(
+    edges.map(async (region) => channelStats(await sharp(image).extract(region).ensureAlpha().raw().toBuffer())),
+  );
+
+  if (stats.every((channels) => channels[3].mean < 8)) return TRANSPARENT;
+
+  const plain = stats.every((channels) => channels[3].mean > 247 && channels.slice(0, 3).every((c) => c.stdev < 12));
+  if (!plain) return null;
+  const mean = (channel: number) => stats.reduce((sum, channels) => sum + channels[channel].mean, 0) / stats.length;
+  const sameColour = [0, 1, 2].every((channel) =>
+    stats.every((channels) => Math.abs(channels[channel].mean - mean(channel)) < 16),
+  );
+  if (!sameColour) return null;
+
+  return { r: Math.round(mean(0)), g: Math.round(mean(1)), b: Math.round(mean(2)), alpha: 1 };
+}
+
+/** Mean and spread of each channel of raw RGBA pixels. */
+function channelStats(rgba: Buffer): { mean: number; stdev: number }[] {
+  const pixels = rgba.length / 4;
+  return [0, 1, 2, 3].map((channel) => {
+    let sum = 0;
+    let squares = 0;
+    for (let i = channel; i < rgba.length; i += 4) {
+      sum += rgba[i];
+      squares += rgba[i] * rgba[i];
+    }
+    const mean = sum / pixels;
+    return { mean, stdev: Math.sqrt(Math.max(0, squares / pixels - mean * mean)) };
+  });
 }