fix: image processing

README.md

@@ -62,7 +62,7 @@ Swagger docs at http://localhost:3001/api/docs
 | Role   | Email              | Password   |
 |--------|--------------------|------------|
 | Admin  | admin@example.com  | Admin123!  |
-| Viewer | viewer@example.com | Viewer123! |
+| Manager | viewer@example.com | Viewer123! |
 
 ## Device Simulator
 

apps/api/src/images/image-processing.spec.ts

@@ -2,17 +2,50 @@ import sharp from 'sharp';
 import { createHash } from 'crypto';
 import { IMAGE_SIZE, InvalidImageError, processProductImage } from './image-processing';
 
-/** A white canvas with a coloured rectangle on it, like a studio shot of a product. */
+const RED = { r: 200, g: 30, b: 30 };
-async function shot(width: number, height: number, product: { width: number; height: number }, format: 'jpeg' | 'png' = 'png') {
+
-  const box = await sharp({
+/** A red rectangle, the "product". */
-    create: { width: product.width, height: product.height, channels: 3, background: { r: 200, g: 30, b: 30 } },
+function box(width: number, height: number) {
-  }).png().toBuffer();
+  return sharp({ create: { width, height, channels: 3, background: RED } }).png().toBuffer();
+}
+
+/** A white canvas with a red product on it, like a studio shot; in the middle unless placed elsewhere. */
+async function shot(
+  width: number,
+  height: number,
+  product: { width: number; height: number; left?: number; top?: number },
+  format: 'jpeg' | 'png' = 'png',
+) {
+  const placed = product.left === undefined
+    ? { gravity: 'centre' }
+    : { left: product.left, top: product.top ?? 0 };
   return sharp({ create: { width, height, channels: 3, background: { r: 255, g: 255, b: 255 } } })
-    .composite([{ input: box, gravity: 'centre' }])
+    .composite([{ input: await box(product.width, product.height), ...placed }])
     [format]()
     .toBuffer();
 }
 
+/** Where the red product ended up in a processed picture. */
+async function redBounds(image: Buffer) {
+  const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  let left = info.width, right = -1, top = info.height, bottom = -1;
+  for (let y = 0; y < info.height; y++) {
+    for (let x = 0; x < info.width; x++) {
+      const i = (y * info.width + x) * 4;
+      if (data[i + 3] > 200 && data[i] > 150 && data[i + 1] < 90 && data[i + 2] < 90) {
+        left = Math.min(left, x); right = Math.max(right, x); top = Math.min(top, y); bottom = Math.max(bottom, y);
+      }
+    }
+  }
+  return { left, right: info.width - 1 - right, top, bottom: info.height - 1 - bottom };
+}
+
+async function pixelAt(image: Buffer, x: number, y: number) {
+  const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  const i = (y * info.width + x) * 4;
+  return { r: data[i], g: data[i + 1], b: data[i + 2], alpha: data[i + 3] };
+}
+
 /** A busy photo without a plain background: noise everywhere. */
 async function photo(width: number, height: number) {
   const pixels = Buffer.alloc(width * height * 3);
@@ -42,14 +75,60 @@ describe('processProductImage', () => {
     expect(a.id).toBe(b.id);
   });
 
-  it('fits a tall bottle whole, with transparent sides, after trimming the white background', async () => {
+  it('fits a tall bottle on white whole, centred with an even margin on a white square', async () => {
     // A bottle of 100x400 on a big white sheet: trimmed, it is 1:4, far from square
     const image = await processProductImage(await shot(1000, 1000, { width: 100, height: 400 }));
 
-    // Transparent at the sides, product in the middle from top to bottom
+    const bounds = await redBounds(image.bytes);
-    expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0);
+    // Top to bottom inside the margin, centred left to right
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255);
+    expect(bounds.top).toBeGreaterThanOrEqual(36);
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, IMAGE_SIZE - 5)).toBe(255);
+    expect(bounds.top).toBeLessThanOrEqual(44);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3);
+    // The sides are the photo's own white, not see-through
+    expect(await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).toMatchObject({ r: 255, g: 255, b: 255, alpha: 255 });
+  });
+
+  it('centres a product photographed off-centre on a plain background', async () => {
+    // The glass stands in the top-left corner of the sheet
+    const image = await processProductImage(await shot(1200, 900, { width: 300, height: 300, left: 60, top: 40 }, 'jpeg'));
+
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4);
+    expect(bounds.left).toBeLessThanOrEqual(46);
+  });
+
+  it('centres a cut-out with a transparent background, keeping it transparent', async () => {
+    const cutout = await sharp({ create: { width: 800, height: 800, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } })
+      .composite([{ input: await box(200, 200), left: 550, top: 500 }])
+      .png()
+      .toBuffer();
+
+    const image = await processProductImage(cutout);
+
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3);
+    expect(await alphaAt(image.bytes, 5, 5)).toBe(0);
+  });
+
+  it('crops a photo around its centre, even when one side is a plain wall', async () => {
+    // Left quarter a white wall, the rest busy; the drink (red) in the exact middle
+    const busy = await photo(600, 700);
+    const wall = await sharp({ create: { width: 200, height: 700, channels: 3, background: { r: 255, g: 255, b: 255 } } }).png().toBuffer();
+    const input = await sharp(busy)
+      .composite([{ input: wall, left: 0, top: 0 }, { input: await box(100, 100), gravity: 'centre' }])
+      .jpeg()
+      .toBuffer();
+
+    const image = await processProductImage(input);
+
+    // Nothing trimmed: the wall is still there, and the drink is in the middle
+    expect((await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).r).toBeGreaterThan(240);
+    const bounds = await redBounds(image.bytes);
+    expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4);
+    expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4);
   });
 
   it('crops a roughly square photo to a square without empty sides', async () => {
@@ -68,9 +147,9 @@ describe('processProductImage', () => {
 
     const image = await processProductImage(sideways);
 
-    // Upright the product is tall: fitted whole with transparent sides
+    // Upright the product is tall (200 wide, 800 high)
-    expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0);
+    const bounds = await redBounds(image.bytes);
-    expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255);
+    expect(bounds.left).toBeGreaterThan(bounds.top + 100);
   });
 
   it('drops metadata such as the GPS position', async () => {

apps/api/src/images/image-processing.ts

@@ -10,15 +10,25 @@ export const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 /** Between these width/height ratios a photo is cropped to a square; outside them (bottles, cans) it is fitted whole. */
 const SQUARE_ENOUGH = { min: 0.75, max: 1.33 };
 
+/** Space kept around a product cut out of a plain background, on every side. */
+const MARGIN = 40;
+
 export type ProcessedImage = { id: string; contentType: string; width: number; height: number; bytes: Buffer };
 
 export class InvalidImageError extends Error {}
 
+type Colour = { r: number; g: number; b: number; alpha: number };
+
+const TRANSPARENT: Colour = { r: 0, g: 0, b: 0, alpha: 0 };
+
 /**
- * Turns an upload into a product picture: rotated the right way up (phone photos), a plain background trimmed
+ * Turns an upload into a product picture, rotated the right way up (phone photos):
- * away, then either cropped to a square around its most interesting part (a roughly square photo of a glass) or
+ * - a product on a plain background (a studio shot on white, a cut-out PNG) is trimmed and centred with an even
- * fitted whole into a transparent square (a tall bottle, which must never be cut off). Metadata such as GPS
+ *   margin, on that same background;
- * position is dropped. The id is a hash of the result, so the same picture always gets the same address.
+ * - any other photo is cropped around its centre to a square, or fitted whole into a transparent square when it is
+ *   far from square (a tall bottle must never be cut off).
+ * Metadata such as GPS position is dropped. The id is a hash of the result, so the same picture always gets the
+ * same address.
  */
 export async function processProductImage(input: Buffer): Promise<ProcessedImage> {
   let upright: Buffer;
@@ -28,15 +38,7 @@ export async function processProductImage(input: Buffer): Promise<ProcessedImage
     throw new InvalidImageError('Not an image this server can read (JPEG, PNG, WebP, GIF or AVIF)');
   }
 
-  const product = await trimBackground(upright);
+  const { data, info } = await ((await cutOut(upright)) ?? (await fitPhoto(upright)))
-  const { width = 1, height = 1 } = await sharp(product).metadata();
-  const ratio = width / height;
-  const square = ratio >= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max;
-
-  const { data, info } = await sharp(product)
-    .resize(IMAGE_SIZE, IMAGE_SIZE, square
-      ? { fit: 'cover', position: sharp.strategy.attention }
-      : { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
     .webp({ quality: 82 })
     .toBuffer({ resolveWithObject: true });
 
@@ -49,13 +51,81 @@ export async function processProductImage(input: Buffer): Promise<ProcessedImage
   };
 }
 
-/** Removes a plain border (a studio shot on white). Photos with a busy background come back as they were. */
+/** A product on a plain background: trimmed, then centred with the same margin on every side. Null for a photo. */
-async function trimBackground(image: Buffer): Promise<Buffer> {
+async function cutOut(image: Buffer): Promise<sharp.Sharp | null> {
+  const background = await plainBackground(image);
+  if (!background) return null;
+
+  let product: Buffer;
   try {
-    const { data, info } = await sharp(image).trim({ threshold: 12 }).toBuffer({ resolveWithObject: true });
+    const trimmed = await sharp(image)
-    // Trimming down to a sliver means the "background" was most of the picture: keep the original
+      .trim({ background: background.alpha === 0 ? undefined : background, threshold: 24 })
-    return info.width >= 32 && info.height >= 32 ? data : image;
+      .toBuffer({ resolveWithObject: true });
+    // Trimmed down to a sliver: the "background" was the picture itself
+    if (trimmed.info.width < 32 || trimmed.info.height < 32) return null;
+    product = trimmed.data;
   } catch {
-    return image;
+    return null;
   }
+
+  const inner = IMAGE_SIZE - 2 * MARGIN;
+  const fitted = await sharp(product).resize(inner, inner, { fit: 'contain', background }).toBuffer();
+  return sharp(fitted).extend({ top: MARGIN, bottom: MARGIN, left: MARGIN, right: MARGIN, background });
+}
+
+/** Any other photo: a centred square crop, or fitted whole when it is too tall or wide for that. */
+async function fitPhoto(image: Buffer): Promise<sharp.Sharp> {
+  const { width = 1, height = 1 } = await sharp(image).metadata();
+  const ratio = width / height;
+  const square = ratio >= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max;
+  return sharp(image).resize(IMAGE_SIZE, IMAGE_SIZE, square
+    ? { fit: 'cover', position: 'centre' }
+    : { fit: 'contain', background: TRANSPARENT });
+}
+
+/**
+ * The background colour when all four edges of the picture are one plain colour (or transparent), else null.
+ * A real photo has something different on at least one edge.
+ */
+async function plainBackground(image: Buffer): Promise<Colour | null> {
+  const { width = 0, height = 0 } = await sharp(image).metadata();
+  if (width < 32 || height < 32) return null;
+
+  const strip = Math.max(2, Math.round(Math.min(width, height) * 0.02));
+  const edges = [
+    { left: 0, top: 0, width, height: strip },
+    { left: 0, top: height - strip, width, height: strip },
+    { left: 0, top: 0, width: strip, height },
+    { left: width - strip, top: 0, width: strip, height },
+  ];
+  const stats = await Promise.all(
+    edges.map(async (region) => channelStats(await sharp(image).extract(region).ensureAlpha().raw().toBuffer())),
+  );
+
+  if (stats.every((channels) => channels[3].mean < 8)) return TRANSPARENT;
+
+  const plain = stats.every((channels) => channels[3].mean > 247 && channels.slice(0, 3).every((c) => c.stdev < 12));
+  if (!plain) return null;
+  const mean = (channel: number) => stats.reduce((sum, channels) => sum + channels[channel].mean, 0) / stats.length;
+  const sameColour = [0, 1, 2].every((channel) =>
+    stats.every((channels) => Math.abs(channels[channel].mean - mean(channel)) < 16),
+  );
+  if (!sameColour) return null;
+
+  return { r: Math.round(mean(0)), g: Math.round(mean(1)), b: Math.round(mean(2)), alpha: 1 };
+}
+
+/** Mean and spread of each channel of raw RGBA pixels. */
+function channelStats(rgba: Buffer): { mean: number; stdev: number }[] {
+  const pixels = rgba.length / 4;
+  return [0, 1, 2, 3].map((channel) => {
+    let sum = 0;
+    let squares = 0;
+    for (let i = channel; i < rgba.length; i += 4) {
+      sum += rgba[i];
+      squares += rgba[i] * rgba[i];
+    }
+    const mean = sum / pixels;
+    return { mean, stdev: Math.sqrt(Math.max(0, squares / pixels - mean * mean)) };
+  });
 }

docs/api.md

@@ -33,6 +33,8 @@ Swagger UI: `http://localhost:3001/api/docs`
 ### Devices
 `POST /devices/register` - Pair a terminal; every pairing adds one
 `POST /devices/:deviceId/sync` - Sync device state (device auth)
+`POST /devices/:deviceId/images` - Upload a picture taken on the terminal (device auth, multipart field `file`):
+  `{ imageId }`. Processed like a dashboard upload (see Images); the terminal links it with a `product_updated` event
 `POST /devices/:deviceId/commands/:commandId/ack` - Acknowledge command (device auth)
 `GET /devices/generation-code` - Generate registration code (admin)
 
@@ -49,6 +51,12 @@ and `docs/server-stock-source-of-truth.md`.
   - `settled`: `amountInCents`, `paymentMethod` (`cash`, `payconiq`, `payment_pending` or `other`), optional
     `commandId` and `items` (`[{ productId?, productName, quantity, unitPriceInCents }]`)
   - `stock_set`: `productId`, `quantity` (a count ≥ 0, or `null` to stop tracking); no `tabId`
+  - `product_created` / `product_updated` / `product_archived`: `productId` (the terminal makes the id of a new
+    product) and `product` with the fields that changed: any of `name`, `category`, `priceInCents`,
+    `lowStockThreshold`, `archived`, `imageId`. `product_created` needs `name` and `priceInCents`; no `tabId`.
+    Every field keeps the time it last changed, and an event only changes a field it changed later (by
+    `occurredAt`), so an offline rename and a dashboard price change both survive. An `imageId` the server doesn't
+    have is ignored; an update of a product the venue doesn't have is acknowledged and dropped
 
   Each id is applied once (tab state and stock in the same transaction). `acknowledgedEventIds` lists the applied
   ones, including repeats; `rejectedEvents` (`[{ id, reason }]`) were invalid and never applied. The terminal drops
@@ -62,10 +70,12 @@ and `docs/server-stock-source-of-truth.md`.
   first, at most 500; the response's `settlementsCursor` is sent back next time.
 - `openTabs` is only used once per venue, on the first sync after the server started holding the tabs: the terminal's
   own tabs (and catalog stock) are taken over and the events of that request are stored without applying them again.
-- The dashboard manages the products. `products` in the request (at most 2000) is only used on that same first sync,
+- Products are changed on the dashboard and, through the product events above, on terminals. `products` in the
-  to take over the terminal's catalog with its stock; after that it is ignored. The response's `catalog` lists every
+  request (at most 2000) is only used on that same first sync, to take over the terminal's catalog with its stock;
-  product of the venue: `[{ id, name, category, priceInCents, lowStockThreshold, archived }]`. The terminal adds and
+  after that it is ignored. The response's `catalog` lists every product of the venue:
-  updates its products from it and hides archived products and products the catalog doesn't list.
+  `[{ id, name, category, priceInCents, lowStockThreshold, archived, imageId }]`. The terminal adds and updates its
+  products from it (replaying its unsent product events on top), hides archived products and products the catalog
+  doesn't list, and downloads pictures it doesn't have yet from `GET /images/:imageId`.
 - Planned customers are opened as tabs by the server, once; `plannedTabNames` is always empty.
 - Dashboard stock adjustments are applied by the server right away; `stockAdjustments` is always empty.
   `stockAdjustmentResults` from older app versions are still recorded and acknowledged.
@@ -95,15 +105,27 @@ Names match case-insensitively and keep their existing spelling; `replace` also
 `GET /venues/:venueId/stock` - The venue's products (archived ones too, with `archivedAt`) with stock, low stock
   level and pending adjustments
 `GET /venues/:venueId/stock/adjustments?productId=&limit=&offset=` - Stock adjustment history, newest first
-`POST /venues/:venueId/stock/adjustments` - Request stock changes (admins and viewers): `{ adjustments: [{ productId, type: 'add' | 'set', quantity, note? }] }`.
+`POST /venues/:venueId/stock/adjustments` - Request stock changes (admins and managers): `{ adjustments: [{ productId, type: 'add' | 'set', quantity, note? }] }`.
   `add` takes a non-zero quantity (negative for breakage) and needs a tracked product; `set` takes 0 or more.
-`POST /venues/:venueId/stock/adjustments/:id/cancel` - Cancel an adjustment the terminal hasn't received yet (admins and viewers)
+`POST /venues/:venueId/stock/adjustments/:id/cancel` - Cancel an adjustment the terminal hasn't received yet (admins and managers)
-`POST /venues/:venueId/stock/products` - Add a product (admin): `{ name, category?, priceInCents, stock?, lowStockThreshold? }`.
+`POST /venues/:venueId/stock/products` - Add a product (admins and managers): `{ name, category?, priceInCents, stock?,
-  `stock` is the starting count (recorded as an applied `set` adjustment); null or left out: not tracked.
+  lowStockThreshold? }`. `stock` is the starting count (recorded as an applied `set` adjustment); null or left out:
-  `product_name_exists` when an active product has the same name (ignoring case).
+  not tracked. `product_name_exists` when an active product has the same name (ignoring case). Terminals may still
-`PATCH /venues/:venueId/stock/products/:productId` - Change a product (admin): any of `{ name, category, priceInCents,
+  create products with the same name; the products page marks those as duplicates.
+`PATCH /venues/:venueId/stock/products/:productId` - Change a product (admins and managers): any of `{ name, category, priceInCents,
   lowStockThreshold, archived }`. `archived: true` deletes it for the terminals (kept for tabs and history),
   `archived: false` brings it back. The stock itself changes through adjustments.
+`POST /venues/:venueId/stock/products/:productId/image` - Set a product's picture (admins and managers; multipart
+  field `file`, see Images): the product with its new `imageId`
+`DELETE /venues/:venueId/stock/products/:productId/image` - Remove a product's picture (admins and managers)
+
+### Images
+`GET /images/:imageId` - A product picture (public, `image/webp`, cached for a year: an id never changes content).
+  Uploads (JPEG, PNG, WebP, GIF or AVIF, at most 10 MB) are turned upright and stored as a 512×512 WebP without
+  metadata. A product on a plain background (all four edges one colour, or transparent) is trimmed and centred
+  with a 40 px margin on that background. Any other photo is cropped around its centre when roughly square, or
+  fitted whole on a transparent background when not (bottles, cans). The id is the SHA-256 of the stored file.
+  `invalid_image` when the upload can't be read.
 
 ### Commands
 `POST /venues/:venueId/commands` - Settle a tab: `{ tabId, type: "close_tab", expectedVersion, paymentMethod }`. The

docs/security.md

@@ -19,8 +19,10 @@
 ## Authorization
 
 ### Role-Based Access
-- `admin`: Full access, including users, device pairing, planned customers and sending reminders
+- `admin`: Full access, including users and pairing or unpairing terminals
-- `viewer` (shown as staff): views tabs and commands and can close tabs
+- `manager` (the people running the venue): products and their pictures, stock, tabs, planned customers and
+  reminders. Accounts and tokens with the former role `viewer` are treated as managers
+- Product pictures are public by id (`GET /images/:imageId`); the id is a SHA-256 hash, so it can't be guessed
 
 ### Venue Isolation
 - Every user and device is scoped to exactly one venue