diff --git a/README.md b/README.md index 47d80c6..8cd4659 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Swagger docs at http://localhost:3001/api/docs | Role | Email | Password | |--------|--------------------|------------| | Admin | admin@example.com | Admin123! | -| Viewer | viewer@example.com | Viewer123! | +| Manager | viewer@example.com | Viewer123! | ## Device Simulator diff --git a/apps/api/src/images/image-processing.spec.ts b/apps/api/src/images/image-processing.spec.ts index e105461..6f117a8 100644 --- a/apps/api/src/images/image-processing.spec.ts +++ b/apps/api/src/images/image-processing.spec.ts @@ -2,17 +2,50 @@ import sharp from 'sharp'; import { createHash } from 'crypto'; import { IMAGE_SIZE, InvalidImageError, processProductImage } from './image-processing'; -/** A white canvas with a coloured rectangle on it, like a studio shot of a product. */ -async function shot(width: number, height: number, product: { width: number; height: number }, format: 'jpeg' | 'png' = 'png') { - const box = await sharp({ - create: { width: product.width, height: product.height, channels: 3, background: { r: 200, g: 30, b: 30 } }, - }).png().toBuffer(); +const RED = { r: 200, g: 30, b: 30 }; + +/** A red rectangle, the "product". */ +function box(width: number, height: number) { + return sharp({ create: { width, height, channels: 3, background: RED } }).png().toBuffer(); +} + +/** A white canvas with a red product on it, like a studio shot; in the middle unless placed elsewhere. */ +async function shot( + width: number, + height: number, + product: { width: number; height: number; left?: number; top?: number }, + format: 'jpeg' | 'png' = 'png', +) { + const placed = product.left === undefined + ? { gravity: 'centre' } + : { left: product.left, top: product.top ?? 0 }; return sharp({ create: { width, height, channels: 3, background: { r: 255, g: 255, b: 255 } } }) - .composite([{ input: box, gravity: 'centre' }]) + .composite([{ input: await box(product.width, product.height), ...placed }]) [format]() .toBuffer(); } +/** Where the red product ended up in a processed picture. */ +async function redBounds(image: Buffer) { + const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true }); + let left = info.width, right = -1, top = info.height, bottom = -1; + for (let y = 0; y < info.height; y++) { + for (let x = 0; x < info.width; x++) { + const i = (y * info.width + x) * 4; + if (data[i + 3] > 200 && data[i] > 150 && data[i + 1] < 90 && data[i + 2] < 90) { + left = Math.min(left, x); right = Math.max(right, x); top = Math.min(top, y); bottom = Math.max(bottom, y); + } + } + } + return { left, right: info.width - 1 - right, top, bottom: info.height - 1 - bottom }; +} + +async function pixelAt(image: Buffer, x: number, y: number) { + const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true }); + const i = (y * info.width + x) * 4; + return { r: data[i], g: data[i + 1], b: data[i + 2], alpha: data[i + 3] }; +} + /** A busy photo without a plain background: noise everywhere. */ async function photo(width: number, height: number) { const pixels = Buffer.alloc(width * height * 3); @@ -42,14 +75,60 @@ describe('processProductImage', () => { expect(a.id).toBe(b.id); }); - it('fits a tall bottle whole, with transparent sides, after trimming the white background', async () => { + it('fits a tall bottle on white whole, centred with an even margin on a white square', async () => { // A bottle of 100x400 on a big white sheet: trimmed, it is 1:4, far from square const image = await processProductImage(await shot(1000, 1000, { width: 100, height: 400 })); - // Transparent at the sides, product in the middle from top to bottom - expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0); - expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255); - expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, IMAGE_SIZE - 5)).toBe(255); + const bounds = await redBounds(image.bytes); + // Top to bottom inside the margin, centred left to right + expect(bounds.top).toBeGreaterThanOrEqual(36); + expect(bounds.top).toBeLessThanOrEqual(44); + expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3); + expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3); + // The sides are the photo's own white, not see-through + expect(await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).toMatchObject({ r: 255, g: 255, b: 255, alpha: 255 }); + }); + + it('centres a product photographed off-centre on a plain background', async () => { + // The glass stands in the top-left corner of the sheet + const image = await processProductImage(await shot(1200, 900, { width: 300, height: 300, left: 60, top: 40 }, 'jpeg')); + + const bounds = await redBounds(image.bytes); + expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4); + expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4); + expect(bounds.left).toBeLessThanOrEqual(46); + }); + + it('centres a cut-out with a transparent background, keeping it transparent', async () => { + const cutout = await sharp({ create: { width: 800, height: 800, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } }) + .composite([{ input: await box(200, 200), left: 550, top: 500 }]) + .png() + .toBuffer(); + + const image = await processProductImage(cutout); + + const bounds = await redBounds(image.bytes); + expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(3); + expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(3); + expect(await alphaAt(image.bytes, 5, 5)).toBe(0); + }); + + it('crops a photo around its centre, even when one side is a plain wall', async () => { + // Left quarter a white wall, the rest busy; the drink (red) in the exact middle + const busy = await photo(600, 700); + const wall = await sharp({ create: { width: 200, height: 700, channels: 3, background: { r: 255, g: 255, b: 255 } } }).png().toBuffer(); + const input = await sharp(busy) + .composite([{ input: wall, left: 0, top: 0 }, { input: await box(100, 100), gravity: 'centre' }]) + .jpeg() + .toBuffer(); + + const image = await processProductImage(input); + + // Nothing trimmed: the wall is still there, and the drink is in the middle + expect((await pixelAt(image.bytes, 5, IMAGE_SIZE / 2)).r).toBeGreaterThan(240); + const bounds = await redBounds(image.bytes); + expect(Math.abs(bounds.left - bounds.right)).toBeLessThanOrEqual(4); + expect(Math.abs(bounds.top - bounds.bottom)).toBeLessThanOrEqual(4); }); it('crops a roughly square photo to a square without empty sides', async () => { @@ -68,9 +147,9 @@ describe('processProductImage', () => { const image = await processProductImage(sideways); - // Upright the product is tall: fitted whole with transparent sides - expect(await alphaAt(image.bytes, 5, IMAGE_SIZE / 2)).toBe(0); - expect(await alphaAt(image.bytes, IMAGE_SIZE / 2, 5)).toBe(255); + // Upright the product is tall (200 wide, 800 high) + const bounds = await redBounds(image.bytes); + expect(bounds.left).toBeGreaterThan(bounds.top + 100); }); it('drops metadata such as the GPS position', async () => { diff --git a/apps/api/src/images/image-processing.ts b/apps/api/src/images/image-processing.ts index 14e1768..3f54870 100644 --- a/apps/api/src/images/image-processing.ts +++ b/apps/api/src/images/image-processing.ts @@ -10,15 +10,25 @@ export const MAX_UPLOAD_BYTES = 10 * 1024 * 1024; /** Between these width/height ratios a photo is cropped to a square; outside them (bottles, cans) it is fitted whole. */ const SQUARE_ENOUGH = { min: 0.75, max: 1.33 }; +/** Space kept around a product cut out of a plain background, on every side. */ +const MARGIN = 40; + export type ProcessedImage = { id: string; contentType: string; width: number; height: number; bytes: Buffer }; export class InvalidImageError extends Error {} +type Colour = { r: number; g: number; b: number; alpha: number }; + +const TRANSPARENT: Colour = { r: 0, g: 0, b: 0, alpha: 0 }; + /** - * Turns an upload into a product picture: rotated the right way up (phone photos), a plain background trimmed - * away, then either cropped to a square around its most interesting part (a roughly square photo of a glass) or - * fitted whole into a transparent square (a tall bottle, which must never be cut off). Metadata such as GPS - * position is dropped. The id is a hash of the result, so the same picture always gets the same address. + * Turns an upload into a product picture, rotated the right way up (phone photos): + * - a product on a plain background (a studio shot on white, a cut-out PNG) is trimmed and centred with an even + * margin, on that same background; + * - any other photo is cropped around its centre to a square, or fitted whole into a transparent square when it is + * far from square (a tall bottle must never be cut off). + * Metadata such as GPS position is dropped. The id is a hash of the result, so the same picture always gets the + * same address. */ export async function processProductImage(input: Buffer): Promise { let upright: Buffer; @@ -28,15 +38,7 @@ export async function processProductImage(input: Buffer): Promise= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max; - - const { data, info } = await sharp(product) - .resize(IMAGE_SIZE, IMAGE_SIZE, square - ? { fit: 'cover', position: sharp.strategy.attention } - : { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } }) + const { data, info } = await ((await cutOut(upright)) ?? (await fitPhoto(upright))) .webp({ quality: 82 }) .toBuffer({ resolveWithObject: true }); @@ -49,13 +51,81 @@ export async function processProductImage(input: Buffer): Promise { +/** A product on a plain background: trimmed, then centred with the same margin on every side. Null for a photo. */ +async function cutOut(image: Buffer): Promise { + const background = await plainBackground(image); + if (!background) return null; + + let product: Buffer; try { - const { data, info } = await sharp(image).trim({ threshold: 12 }).toBuffer({ resolveWithObject: true }); - // Trimming down to a sliver means the "background" was most of the picture: keep the original - return info.width >= 32 && info.height >= 32 ? data : image; + const trimmed = await sharp(image) + .trim({ background: background.alpha === 0 ? undefined : background, threshold: 24 }) + .toBuffer({ resolveWithObject: true }); + // Trimmed down to a sliver: the "background" was the picture itself + if (trimmed.info.width < 32 || trimmed.info.height < 32) return null; + product = trimmed.data; } catch { - return image; + return null; } + + const inner = IMAGE_SIZE - 2 * MARGIN; + const fitted = await sharp(product).resize(inner, inner, { fit: 'contain', background }).toBuffer(); + return sharp(fitted).extend({ top: MARGIN, bottom: MARGIN, left: MARGIN, right: MARGIN, background }); +} + +/** Any other photo: a centred square crop, or fitted whole when it is too tall or wide for that. */ +async function fitPhoto(image: Buffer): Promise { + const { width = 1, height = 1 } = await sharp(image).metadata(); + const ratio = width / height; + const square = ratio >= SQUARE_ENOUGH.min && ratio <= SQUARE_ENOUGH.max; + return sharp(image).resize(IMAGE_SIZE, IMAGE_SIZE, square + ? { fit: 'cover', position: 'centre' } + : { fit: 'contain', background: TRANSPARENT }); +} + +/** + * The background colour when all four edges of the picture are one plain colour (or transparent), else null. + * A real photo has something different on at least one edge. + */ +async function plainBackground(image: Buffer): Promise { + const { width = 0, height = 0 } = await sharp(image).metadata(); + if (width < 32 || height < 32) return null; + + const strip = Math.max(2, Math.round(Math.min(width, height) * 0.02)); + const edges = [ + { left: 0, top: 0, width, height: strip }, + { left: 0, top: height - strip, width, height: strip }, + { left: 0, top: 0, width: strip, height }, + { left: width - strip, top: 0, width: strip, height }, + ]; + const stats = await Promise.all( + edges.map(async (region) => channelStats(await sharp(image).extract(region).ensureAlpha().raw().toBuffer())), + ); + + if (stats.every((channels) => channels[3].mean < 8)) return TRANSPARENT; + + const plain = stats.every((channels) => channels[3].mean > 247 && channels.slice(0, 3).every((c) => c.stdev < 12)); + if (!plain) return null; + const mean = (channel: number) => stats.reduce((sum, channels) => sum + channels[channel].mean, 0) / stats.length; + const sameColour = [0, 1, 2].every((channel) => + stats.every((channels) => Math.abs(channels[channel].mean - mean(channel)) < 16), + ); + if (!sameColour) return null; + + return { r: Math.round(mean(0)), g: Math.round(mean(1)), b: Math.round(mean(2)), alpha: 1 }; +} + +/** Mean and spread of each channel of raw RGBA pixels. */ +function channelStats(rgba: Buffer): { mean: number; stdev: number }[] { + const pixels = rgba.length / 4; + return [0, 1, 2, 3].map((channel) => { + let sum = 0; + let squares = 0; + for (let i = channel; i < rgba.length; i += 4) { + sum += rgba[i]; + squares += rgba[i] * rgba[i]; + } + const mean = sum / pixels; + return { mean, stdev: Math.sqrt(Math.max(0, squares / pixels - mean * mean)) }; + }); } diff --git a/docs/api.md b/docs/api.md index 98bb0e8..e5acc3c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -33,6 +33,8 @@ Swagger UI: `http://localhost:3001/api/docs` ### Devices `POST /devices/register` - Pair a terminal; every pairing adds one `POST /devices/:deviceId/sync` - Sync device state (device auth) +`POST /devices/:deviceId/images` - Upload a picture taken on the terminal (device auth, multipart field `file`): + `{ imageId }`. Processed like a dashboard upload (see Images); the terminal links it with a `product_updated` event `POST /devices/:deviceId/commands/:commandId/ack` - Acknowledge command (device auth) `GET /devices/generation-code` - Generate registration code (admin) @@ -49,6 +51,12 @@ and `docs/server-stock-source-of-truth.md`. - `settled`: `amountInCents`, `paymentMethod` (`cash`, `payconiq`, `payment_pending` or `other`), optional `commandId` and `items` (`[{ productId?, productName, quantity, unitPriceInCents }]`) - `stock_set`: `productId`, `quantity` (a count ≥ 0, or `null` to stop tracking); no `tabId` + - `product_created` / `product_updated` / `product_archived`: `productId` (the terminal makes the id of a new + product) and `product` with the fields that changed: any of `name`, `category`, `priceInCents`, + `lowStockThreshold`, `archived`, `imageId`. `product_created` needs `name` and `priceInCents`; no `tabId`. + Every field keeps the time it last changed, and an event only changes a field it changed later (by + `occurredAt`), so an offline rename and a dashboard price change both survive. An `imageId` the server doesn't + have is ignored; an update of a product the venue doesn't have is acknowledged and dropped Each id is applied once (tab state and stock in the same transaction). `acknowledgedEventIds` lists the applied ones, including repeats; `rejectedEvents` (`[{ id, reason }]`) were invalid and never applied. The terminal drops @@ -62,10 +70,12 @@ and `docs/server-stock-source-of-truth.md`. first, at most 500; the response's `settlementsCursor` is sent back next time. - `openTabs` is only used once per venue, on the first sync after the server started holding the tabs: the terminal's own tabs (and catalog stock) are taken over and the events of that request are stored without applying them again. -- The dashboard manages the products. `products` in the request (at most 2000) is only used on that same first sync, - to take over the terminal's catalog with its stock; after that it is ignored. The response's `catalog` lists every - product of the venue: `[{ id, name, category, priceInCents, lowStockThreshold, archived }]`. The terminal adds and - updates its products from it and hides archived products and products the catalog doesn't list. +- Products are changed on the dashboard and, through the product events above, on terminals. `products` in the + request (at most 2000) is only used on that same first sync, to take over the terminal's catalog with its stock; + after that it is ignored. The response's `catalog` lists every product of the venue: + `[{ id, name, category, priceInCents, lowStockThreshold, archived, imageId }]`. The terminal adds and updates its + products from it (replaying its unsent product events on top), hides archived products and products the catalog + doesn't list, and downloads pictures it doesn't have yet from `GET /images/:imageId`. - Planned customers are opened as tabs by the server, once; `plannedTabNames` is always empty. - Dashboard stock adjustments are applied by the server right away; `stockAdjustments` is always empty. `stockAdjustmentResults` from older app versions are still recorded and acknowledged. @@ -95,15 +105,27 @@ Names match case-insensitively and keep their existing spelling; `replace` also `GET /venues/:venueId/stock` - The venue's products (archived ones too, with `archivedAt`) with stock, low stock level and pending adjustments `GET /venues/:venueId/stock/adjustments?productId=&limit=&offset=` - Stock adjustment history, newest first -`POST /venues/:venueId/stock/adjustments` - Request stock changes (admins and viewers): `{ adjustments: [{ productId, type: 'add' | 'set', quantity, note? }] }`. +`POST /venues/:venueId/stock/adjustments` - Request stock changes (admins and managers): `{ adjustments: [{ productId, type: 'add' | 'set', quantity, note? }] }`. `add` takes a non-zero quantity (negative for breakage) and needs a tracked product; `set` takes 0 or more. -`POST /venues/:venueId/stock/adjustments/:id/cancel` - Cancel an adjustment the terminal hasn't received yet (admins and viewers) -`POST /venues/:venueId/stock/products` - Add a product (admin): `{ name, category?, priceInCents, stock?, lowStockThreshold? }`. - `stock` is the starting count (recorded as an applied `set` adjustment); null or left out: not tracked. - `product_name_exists` when an active product has the same name (ignoring case). -`PATCH /venues/:venueId/stock/products/:productId` - Change a product (admin): any of `{ name, category, priceInCents, +`POST /venues/:venueId/stock/adjustments/:id/cancel` - Cancel an adjustment the terminal hasn't received yet (admins and managers) +`POST /venues/:venueId/stock/products` - Add a product (admins and managers): `{ name, category?, priceInCents, stock?, + lowStockThreshold? }`. `stock` is the starting count (recorded as an applied `set` adjustment); null or left out: + not tracked. `product_name_exists` when an active product has the same name (ignoring case). Terminals may still + create products with the same name; the products page marks those as duplicates. +`PATCH /venues/:venueId/stock/products/:productId` - Change a product (admins and managers): any of `{ name, category, priceInCents, lowStockThreshold, archived }`. `archived: true` deletes it for the terminals (kept for tabs and history), `archived: false` brings it back. The stock itself changes through adjustments. +`POST /venues/:venueId/stock/products/:productId/image` - Set a product's picture (admins and managers; multipart + field `file`, see Images): the product with its new `imageId` +`DELETE /venues/:venueId/stock/products/:productId/image` - Remove a product's picture (admins and managers) + +### Images +`GET /images/:imageId` - A product picture (public, `image/webp`, cached for a year: an id never changes content). + Uploads (JPEG, PNG, WebP, GIF or AVIF, at most 10 MB) are turned upright and stored as a 512×512 WebP without + metadata. A product on a plain background (all four edges one colour, or transparent) is trimmed and centred + with a 40 px margin on that background. Any other photo is cropped around its centre when roughly square, or + fitted whole on a transparent background when not (bottles, cans). The id is the SHA-256 of the stored file. + `invalid_image` when the upload can't be read. ### Commands `POST /venues/:venueId/commands` - Settle a tab: `{ tabId, type: "close_tab", expectedVersion, paymentMethod }`. The diff --git a/docs/security.md b/docs/security.md index fc97865..88c90b4 100644 --- a/docs/security.md +++ b/docs/security.md @@ -19,8 +19,10 @@ ## Authorization ### Role-Based Access -- `admin`: Full access, including users, device pairing, planned customers and sending reminders -- `viewer` (shown as staff): views tabs and commands and can close tabs +- `admin`: Full access, including users and pairing or unpairing terminals +- `manager` (the people running the venue): products and their pictures, stock, tabs, planned customers and + reminders. Accounts and tokens with the former role `viewer` are treated as managers +- Product pictures are public by id (`GET /images/:imageId`); the id is a SHA-256 hash, so it can't be guessed ### Venue Isolation - Every user and device is scoped to exactly one venue