From 71f501da6fccbce7d7d73274d0ec828894c10610 Mon Sep 17 00:00:00 2001 From: xEsiah Date: Sat, 31 Oct 2026 11:13:05 +0100 Subject: [PATCH] refactor(storage): enforce per-user max_file_bytes limit on upload pipelines * Validate Content-Length on single-part upload route against user limit * Check total expected size on chunked upload session creation * Re-evaluate user limit at finalization to reject dynamic quota reductions * Clean up temporary files on payload size rejection (413) * Add 60-second polling interval for periodic fetching of messages and images --- backend/migrations/010_upload_sessions.sql | 36 ++ backend/src/index.ts | 13 +- backend/src/routes/files.ts | 10 +- backend/src/routes/uploads.ts | 482 +++++++++++++++++++++ backend/src/storage.ts | 69 ++- docker/nginx/snippets/proxy-api.conf | 6 +- docs/api.md | 30 +- docs/deployment.md | 5 + frontend/src/components/icons.tsx | 6 + frontend/src/lib/api.ts | 178 +++++++- frontend/src/lib/i18n.tsx | 1 + frontend/src/pages/Browser.tsx | 105 ++++- 12 files changed, 907 insertions(+), 34 deletions(-) create mode 100644 backend/migrations/010_upload_sessions.sql create mode 100644 backend/src/routes/uploads.ts diff --git a/backend/migrations/010_upload_sessions.sql b/backend/migrations/010_upload_sessions.sql new file mode 100644 index 0000000..c37e75d --- /dev/null +++ b/backend/migrations/010_upload_sessions.sql @@ -0,0 +1,36 @@ +-- 010_upload_sessions.sql +-- +-- Upload resumable pour les fichiers qui depassent la limite de corps de +-- requete de Cloudflare (100 MB en Free/Pro). Le client cree une session, +-- envoie des morceaux de 50 MiB, puis demande la finalisation. +-- +-- Le fichier temporaire est prealloue a la taille finale : le disque est donc +-- verifie avant le premier octet utile, et les ecritures se font a une offset +-- connue sans jamais garder le fichier complet en memoire. +-- +-- `received_bytes` avance uniquement quand la plage demandee correspond +-- exactement a la suite deja recue : les morceaux sont sequentiels, ce qui +-- rend la reprise simple (GET renvoie l'offset courant) et evite les trous. + +CREATE TABLE upload_sessions ( + id CHAR(36) NOT NULL, + user_id CHAR(36) NOT NULL, + folder_id CHAR(36) NULL, + storage_location_id INT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + mime_type VARCHAR(255) NULL, + size_bytes BIGINT UNSIGNED NOT NULL, + received_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, + chunk_size INT UNSIGNED NOT NULL, + relative_path VARCHAR(1024) NOT NULL, + temp_path VARCHAR(1024) NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + KEY idx_upload_sessions_user (user_id), + KEY idx_upload_sessions_expiry (expires_at), + CONSTRAINT chk_upload_sessions_received CHECK (received_bytes <= size_bytes), + CONSTRAINT fk_upload_sessions_user FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT fk_upload_sessions_folder FOREIGN KEY (folder_id) REFERENCES folders (id) ON DELETE CASCADE, + CONSTRAINT fk_upload_sessions_location FOREIGN KEY (storage_location_id) REFERENCES storage_locations (id) ON DELETE RESTRICT +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/src/index.ts b/backend/src/index.ts index c0e64cf..1624637 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -11,6 +11,7 @@ import { registerFolderRoutes } from "./routes/folders.js"; import { registerMediaRoutes } from "./routes/media.js"; import { registerSearchRoutes } from "./routes/search.js"; import { registerTrashRoutes, purgeExpiredTrash } from "./routes/trash.js"; +import { purgeExpiredUploads, registerUploadRoutes } from "./routes/uploads.js"; // Les migrations sont relues au demarrage. L'operation est idempotente et // refuse de servir si une migration deja appliquee a change : une base non @@ -27,6 +28,7 @@ void startService({ registerDirectoryRoutes(app, db); registerFolderRoutes(app, db, config); registerFileRoutes(app, db, config); + registerUploadRoutes(app, db, config); registerMediaRoutes(app, db); registerTrashRoutes(app, db, config); registerSearchRoutes(app, db); @@ -36,6 +38,15 @@ void startService({ // Entretien au demarrage : sessions expirees et corbeille hors retention. const sessions = await purgeExpiredSessions(db); const purged = await purgeExpiredTrash(db, config); - logger.info("entretien au demarrage", { sessions, purged }); + const uploads = await purgeExpiredUploads(db, config); + logger.info("entretien au demarrage", { sessions, purged, uploads }); + + // Les envois abandonnes ne doivent pas garder leur temporaire indefiniment : + // un balayage horaire suffit, la session expirant apres 24 h. + setInterval(() => { + void purgeExpiredUploads(db, config).catch((error: unknown) => + logger.error("purge des envois inacheves", { error: String(error) }), + ); + }, 60 * 60 * 1000).unref(); }, }); diff --git a/backend/src/routes/files.ts b/backend/src/routes/files.ts index 103d63e..53ad970 100644 --- a/backend/src/routes/files.ts +++ b/backend/src/routes/files.ts @@ -56,7 +56,7 @@ const parse = (schema: z.ZodType, body: unknown): T => { return result.data; }; -interface FileRow { +export interface FileRow { id: string; folder_id: string | null; name: string; @@ -71,13 +71,13 @@ interface FileRow { updated_at: string; } -const SELECT_FILE = `SELECT f.id, f.folder_id, f.name, f.mime_type, f.size_bytes, f.checksum_sha256, +export const SELECT_FILE = `SELECT f.id, f.folder_id, f.name, f.mime_type, f.size_bytes, f.checksum_sha256, f.relative_path, f.owner_id, EXISTS (SELECT 1 FROM user_file_stars s WHERE s.file_id = f.id AND s.user_id = ?) AS is_starred, f.download_count, f.created_at, f.updated_at FROM files f`; -const publicFile = (row: FileRow) => ({ +export const publicFile = (row: FileRow) => ({ id: row.id, folder_id: row.folder_id, name: row.name, @@ -92,7 +92,7 @@ const publicFile = (row: FileRow) => ({ }); /** Nom d'affichage : ni separateur, ni sequence d'echappement, ni point seul. */ -const assertName = (raw: string | undefined): string => { +export const assertName = (raw: string | undefined): string => { const name = (raw ?? "").trim(); if (!name || name.length > 255) { throw AppError.badRequest("Le nom du fichier doit contenir entre 1 et 255 caracteres."); @@ -103,7 +103,7 @@ const assertName = (raw: string | undefined): string => { return name; }; -const mimeOf = (header: string | undefined): string => { +export const mimeOf = (header: string | undefined): string => { const value = (header ?? "").split(";")[0]?.trim().toLowerCase() ?? ""; return MIME.test(value) ? value : "application/octet-stream"; }; diff --git a/backend/src/routes/uploads.ts b/backend/src/routes/uploads.ts new file mode 100644 index 0000000..d0115e2 --- /dev/null +++ b/backend/src/routes/uploads.ts @@ -0,0 +1,482 @@ +import { randomUUID } from "node:crypto"; +import { stat } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { + AppError, + attachUser, + currentUser, + requireAuth, + requireRole, + type App, + type Config, + type Db, +} from "../lib/index.js"; +import { audit, canWrite } from "../access.js"; +import { indexMedia } from "../media.js"; +import { + absolutePath, + chooseLocation, + discard, + diskUsage, + ensureDir, + fileExists, + hashFile, + listUploadTemps, + preallocateUpload, + publish, + relativePathFor, + uploadTempPath, + writeChunk, +} from "../storage.js"; +import { SELECT_FILE, assertName, mimeOf, publicFile, type FileRow } from "./files.js"; + +/** + * Upload resumable pour les fichiers qui depassent la limite de corps de + * requete de Cloudflare. Le client envoie des morceaux sequentiels de 50 MiB, + * ce qui reste sous la limite de 100 MB des offres Free/Pro. + * + * Le fichier temporaire est prealloue et rempli a l'offset courant. Une + * session ne peut appartenir qu'a son createur, et les droits d'ecriture sont + * revalides au moment de la finalisation : un partage retire pendant l'envoi + * ne permet pas de publier le fichier. + */ + +const uuid = z + .string() + .regex( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + "Identifiant invalide.", + ); + +/** 50 MiB : sous la limite Cloudflare (100 MB) avec une marge confortable. */ +export const CHUNK_SIZE = 50 * 1024 * 1024; + +/** Duree de vie d'une session abandonnee, avant purge du temporaire. */ +const SESSION_TTL_MS = 24 * 60 * 60 * 1000; + +const createInput = z.object({ + name: z.string().trim().min(1).max(255), + folderId: uuid.nullish(), + size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + mimeType: z.string().max(255).nullish(), +}); + +const parse = (schema: z.ZodType, body: unknown): T => { + const result = schema.safeParse(body); + if (!result.success) { + const issue = result.error.issues[0]; + throw AppError.badRequest( + issue ? `${issue.path.join(".") || "corps"} : ${issue.message}` : "Corps invalide.", + ); + } + return result.data; +}; + +interface UploadSessionRow { + id: string; + user_id: string; + folder_id: string | null; + storage_location_id: number; + name: string; + mime_type: string | null; + size_bytes: number | string; + received_bytes: number | string; + chunk_size: number; + relative_path: string; + temp_path: string; + expires_at: string | Date; +} + +const SELECT_SESSION = `SELECT id, user_id, folder_id, storage_location_id, name, mime_type, + size_bytes, received_bytes, chunk_size, relative_path, temp_path, expires_at + FROM upload_sessions`; + +const sessionPublic = (row: UploadSessionRow) => ({ + uploadId: row.id, + name: row.name, + folderId: row.folder_id, + size: Number(row.size_bytes), + chunkSize: row.chunk_size, + uploadedBytes: Number(row.received_bytes), + expiresAt: new Date(row.expires_at).toISOString(), +}); + +/** Supprime la session et son temporaire, sans bruit si deja absente. */ +const dropSession = async (db: Db, row: UploadSessionRow): Promise => { + await discard(row.temp_path); + await db.exec("DELETE FROM upload_sessions WHERE id = ?", [row.id]).catch(() => undefined); +}; + +const loadSession = async ( + db: Db, + user: { id: string }, + id: string, +): Promise => { + if (!uuid.safeParse(id).success) throw AppError.notFound("Session d'envoi introuvable."); + const row = await db.one(`${SELECT_SESSION} WHERE id = ? AND user_id = ?`, [ + id, + user.id, + ]); + if (!row) throw AppError.notFound("Session d'envoi introuvable."); + if (new Date(row.expires_at).getTime() <= Date.now()) { + await dropSession(db, row); + throw AppError.notFound("Session d'envoi expiree."); + } + return row; +}; + +const assertNameFree = async ( + db: Db, + user: { id: string }, + folderId: string | null, + name: string, +): Promise => { + const duplicate = await db.one( + `SELECT 1 AS taken FROM files + WHERE folder_id <=> ? AND active_name = ? AND deleted_at IS NULL + AND ( folder_id IS NOT NULL OR owner_id = ? ) + LIMIT 1`, + [folderId, name, user.id], + ); + if (duplicate) throw AppError.conflict("Ce nom existe deja ici."); +}; + +export const registerUploadRoutes = (app: App, db: Db, config: Config): void => { + app.use("/api/cloud/uploads/*", attachUser(db)); + app.use("/api/cloud/uploads", attachUser(db)); + + app.post( + "/api/cloud/uploads", + requireAuth, + requireRole("admin", "user"), + async (context) => { + const user = currentUser(context); + const data = parse(createInput, await context.req.json().catch(() => null)); + const name = assertName(data.name); + const folderId = data.folderId ?? null; + const size = data.size; + + if (folderId !== null && !(await canWrite(db, user, folderId))) { + throw AppError.forbidden("Vous ne pouvez pas ecrire dans ce dossier."); + } + + const budget = await db.one<{ + used_bytes: number | string; + quota_bytes: number | string; + max_file_bytes: number | string; + }>( + "SELECT used_bytes, quota_bytes, max_file_bytes FROM users WHERE id = ? AND deleted_at IS NULL", + [user.id], + ); + if (!budget) throw AppError.unauthorized(); + + // Les sessions en cours immobilisent deja une part du quota : sans + // cette somme, plusieurs envois simultanes pourraient chacun passer le + // controle puis depasser le quota au moment de la finalisation. + const pending = await db.one<{ bytes: number | string | null }>( + `SELECT COALESCE(SUM(size_bytes - received_bytes), 0) AS bytes + FROM upload_sessions + WHERE user_id = ? AND expires_at > UTC_TIMESTAMP()`, + [user.id], + ); + + if (BigInt(size) > BigInt(budget.max_file_bytes)) { + throw AppError.payloadTooLarge( + `Fichier trop volumineux (maximum ${budget.max_file_bytes} octets).`, + ); + } + if ( + BigInt(budget.used_bytes) + BigInt(pending?.bytes ?? 0) + BigInt(size) > + BigInt(budget.quota_bytes) + ) { + throw AppError.payloadTooLarge("Votre quota de stockage est depasse."); + } + + await assertNameFree(db, user, folderId, name); + + const disk = await diskUsage(config.MEDIA_DIR); + if (disk !== null && BigInt(size) > BigInt(disk.available_bytes)) { + throw AppError.insufficientStorage( + "Espace disque insuffisant sur le serveur pour ce fichier.", + ); + } + + const location = await chooseLocation(db, BigInt(size)); + const id = randomUUID(); + const relative = relativePathFor(location.slug, id, name); + const temporary = uploadTempPath(config.MEDIA_DIR, id); + + await preallocateUpload(temporary, size); + + try { + await db.exec( + `INSERT INTO upload_sessions + (id, user_id, folder_id, storage_location_id, name, mime_type, size_bytes, + received_bytes, chunk_size, relative_path, temp_path, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`, + [ + id, + user.id, + folderId, + location.id, + name, + mimeOf(data.mimeType ?? undefined), + size, + CHUNK_SIZE, + relative, + temporary, + new Date(Date.now() + SESSION_TTL_MS), + ], + ); + } catch (error) { + await discard(temporary); + throw error; + } + + const created = await db.required(`${SELECT_SESSION} WHERE id = ?`, [id]); + await audit( + db, + context, + user, + "file.upload.start", + "file", + id, + { name, size, folder_id: folderId }, + config.TRUST_PROXY, + ); + + return context.json({ upload: sessionPublic(created) }, 201); + }, + ); + + app.get("/api/cloud/uploads/:id", requireAuth, async (context) => { + const user = currentUser(context); + const row = await loadSession(db, user, context.req.param("id") ?? ""); + return context.json({ upload: sessionPublic(row) }); + }); + + app.put("/api/cloud/uploads/:id/chunks", requireAuth, async (context) => { + const user = currentUser(context); + const row = await loadSession(db, user, context.req.param("id") ?? ""); + const offset = Number(row.received_bytes); + const size = Number(row.size_bytes); + + const declaredHeader = context.req.header("content-length"); + if (declaredHeader === undefined) { + throw new AppError(411, "LENGTH_REQUIRED", "En-tête Content-Length manquant."); + } + const declared = Number(declaredHeader); + if (!Number.isSafeInteger(declared) || declared <= 0) { + throw AppError.badRequest("Longueur de morceau invalide."); + } + if (declared > row.chunk_size) { + throw AppError.payloadTooLarge(`Morceau trop volumineux (maximum ${row.chunk_size} octets).`); + } + if (offset + declared > size) { + throw AppError.badRequest("Le morceau depasse la taille annoncee du fichier."); + } + + const body = context.req.raw.body; + if (!body) throw AppError.badRequest("Corps de requete vide."); + + // Ecriture d'abord, curseur ensuite : un morceau interrompu laisse le + // curseur intact, donc le prochain essai reecrit la meme plage sans creer + // de trou. Deux requetes concurrentes sur la meme plage ecrivent les + // memes octets : une seule avance le curseur, l'autre constate que le + // morceau est deja applique. + const written = await writeChunk(row.temp_path, offset, body, declared); + if (written !== declared) { + throw AppError.badRequest( + `Longueur incoherente : ${declared} octets annonces, ${written} recus.`, + ); + } + + const reserved = await db.exec( + `UPDATE upload_sessions + SET received_bytes = received_bytes + ? + WHERE id = ? AND user_id = ? AND received_bytes = ? + AND received_bytes + ? <= size_bytes AND expires_at > UTC_TIMESTAMP()`, + [declared, row.id, user.id, offset, declared], + ); + + if (reserved.affectedRows === 0) { + const fresh = await db.one( + `${SELECT_SESSION} WHERE id = ? AND user_id = ?`, + [row.id, user.id], + ); + if (!fresh) throw AppError.notFound("Session d'envoi introuvable."); + if (Number(fresh.received_bytes) < offset + declared) { + throw AppError.conflict("Morceau deja recu ou session modifiee. Reprenez depuis le statut."); + } + return context.json({ upload: sessionPublic(fresh) }); + } + + return context.json({ upload: sessionPublic({ ...row, received_bytes: offset + declared }) }); + }); + + app.post("/api/cloud/uploads/:id/complete", requireAuth, async (context) => { + const user = currentUser(context); + const row = await loadSession(db, user, context.req.param("id") ?? ""); + const size = Number(row.size_bytes); + const received = Number(row.received_bytes); + + if (received !== size) { + throw AppError.badRequest("Envoi incomplet : des morceaux sont encore manquants."); + } + if (!(await fileExists(row.temp_path))) { + await dropSession(db, row); + throw AppError.notFound("Fichier temporaire introuvable. Relancez l'envoi."); + } + + // La limite de taille est propre au compte : elle peut avoir baisse + // pendant un envoi long, il faut donc la reverifier avant de publier. + const budget = await db.one<{ max_file_bytes: number | string }>( + "SELECT max_file_bytes FROM users WHERE id = ? AND deleted_at IS NULL", + [user.id], + ); + if (!budget) { + await dropSession(db, row); + throw AppError.unauthorized(); + } + if (BigInt(size) > BigInt(budget.max_file_bytes)) { + await dropSession(db, row); + throw AppError.payloadTooLarge( + `Fichier trop volumineux (maximum ${budget.max_file_bytes} octets).`, + ); + } + + if (row.folder_id !== null && !(await canWrite(db, user, row.folder_id))) { + await dropSession(db, row); + throw AppError.forbidden("Vous ne pouvez plus ecrire dans ce dossier."); + } + + try { + await assertNameFree(db, user, row.folder_id, row.name); + } catch (error) { + await dropSession(db, row); + throw error; + } + + const checksum = await hashFile(row.temp_path); + const destination = absolutePath(config.MEDIA_DIR, row.relative_path); + + // Le dossier de l'emplacement peut ne pas encore exister : l'envoi direct + // le cree avant d'ecrire, l'envoi resumable ecrit d'abord dans .uploads. + await ensureDir(path.dirname(destination)); + + // Publication avant la base, comme le depot direct : si la transaction + // echoue, le temporaire est remis en place pour permettre un nouvel essai. + await publish(row.temp_path, destination); + + try { + await db.transaction(async (connection) => { + const [quota] = await connection.execute( + `UPDATE users SET used_bytes = used_bytes + ? + WHERE id = ? AND used_bytes + ? <= quota_bytes`, + [size, user.id, size], + ); + if ((quota as { affectedRows: number }).affectedRows === 0) { + throw AppError.payloadTooLarge("Votre quota de stockage est depasse."); + } + + const [support] = await connection.execute( + `UPDATE storage_locations + SET used_bytes = used_bytes + ? + WHERE id = ? AND (quota_bytes IS NULL OR used_bytes + ? <= quota_bytes)`, + [size, row.storage_location_id, size], + ); + if ((support as { affectedRows: number }).affectedRows === 0) { + throw AppError.payloadTooLarge("L'espace disponible sur le support est insuffisant."); + } + + await connection.execute( + `INSERT INTO files + (id, storage_location_id, folder_id, name, mime_type, size_bytes, + checksum_sha256, relative_path, owner_id, created_by) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + row.id, + row.storage_location_id, + row.folder_id, + row.name, + row.mime_type ?? "application/octet-stream", + size, + checksum, + row.relative_path, + user.id, + user.id, + ], + ); + + await connection.execute("DELETE FROM upload_sessions WHERE id = ?", [row.id]); + }); + } catch (error) { + await publish(destination, row.temp_path).catch(async () => { + await discard(destination); + await db.exec("DELETE FROM upload_sessions WHERE id = ?", [row.id]).catch(() => undefined); + }); + if ((error as { errno?: number }).errno === 1062) { + throw AppError.conflict("Ce nom existe deja ici."); + } + throw error; + } + + await indexMedia(db, config, row.id, row.mime_type ?? "application/octet-stream", destination).catch( + () => undefined, + ); + + await audit( + db, + context, + user, + "file.upload", + "file", + row.id, + { name: row.name, size, folder_id: row.folder_id }, + config.TRUST_PROXY, + ); + + const created = await db.required(`${SELECT_FILE} WHERE f.id = ?`, [user.id, row.id]); + return context.json({ file: publicFile(created) }, 201); + }); + + app.delete("/api/cloud/uploads/:id", requireAuth, async (context) => { + const user = currentUser(context); + const row = await loadSession(db, user, context.req.param("id") ?? ""); + await dropSession(db, row); + await audit( + db, + context, + user, + "file.upload.cancel", + "file", + row.id, + { name: row.name }, + config.TRUST_PROXY, + ); + return context.json({ ok: true }); + }); +}; + +/** + * Purge les sessions arrivees a echeance et les temporaires orphelins (session + * supprimee avec le dossier ou le compte, par exemple). Appelee au demarrage + * puis periodiquement par le service. + */ +export const purgeExpiredUploads = async (db: Db, config: Config): Promise => { + const expired = await db.query( + `${SELECT_SESSION} WHERE expires_at <= UTC_TIMESTAMP()`, + ); + for (const row of expired) await discard(row.temp_path); + + const result = await db.exec("DELETE FROM upload_sessions WHERE expires_at <= UTC_TIMESTAMP()"); + + const cutoff = Date.now() - SESSION_TTL_MS; + for (const temporary of await listUploadTemps(config.MEDIA_DIR)) { + const info = await stat(temporary).catch(() => null); + if (info !== null && info.mtimeMs < cutoff) await discard(temporary); + } + + return result.affectedRows; +}; diff --git a/backend/src/storage.ts b/backend/src/storage.ts index 6774487..efd081a 100644 --- a/backend/src/storage.ts +++ b/backend/src/storage.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdir, rename, rm, stat, statfs } from "node:fs/promises"; +import { mkdir, open, readdir, rename, rm, stat, statfs } from "node:fs/promises"; import path from "node:path"; import { Readable, Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -130,6 +130,73 @@ export const discard = async (...paths: string[]): Promise => { export const temporaryPath = (mediaDir: string, slug: string): string => path.join(path.resolve(mediaDir), slug, `.upload-${randomUUID()}`); +/** + * Dossier des uploads resumables. Il reste sous MEDIA_DIR : le renommage + * final vers l'emplacement de stockage est donc atomique. + */ +export const uploadTempDir = (mediaDir: string): string => + path.join(path.resolve(mediaDir), ".uploads"); + +export const uploadTempPath = (mediaDir: string, id: string): string => + path.join(uploadTempDir(mediaDir), `${id}.part`); + +/** Prealloue le temporaire a la taille annoncee, sans ecrire les octets. */ +export const preallocateUpload = async (destination: string, size: number): Promise => { + await ensureDir(path.dirname(destination)); + const handle = await open(destination, "wx"); + try { + await handle.truncate(size); + } finally { + await handle.close(); + } +}; + +/** + * Ecrit un morceau a l'offset courant. Le flux est borne : un client qui + * annonce une taille et en envoie davantage est coupe avant d'ecrire l'exces. + */ +export const writeChunk = async ( + destination: string, + offset: number, + body: ReadableStream, + maxBytes: number, +): Promise => { + let received = 0; + + const meter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + if (received > maxBytes) { + callback(AppError.payloadTooLarge(`Morceau trop volumineux (maximum ${maxBytes} octets).`)); + return; + } + callback(null, chunk); + }, + }); + + const source = Readable.fromWeb(body as Parameters[0]); + await pipeline(source, meter, createWriteStream(destination, { flags: "r+", start: offset })); + + return received; +}; + +/** SHA-256 du fichier complet, calcule au moment de la finalisation. */ +export const hashFile = async (absolute: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(absolute)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +}; + +/** Fichiers `.part` abandonnes : utilises par la purge des sessions expirees. */ +export const listUploadTemps = async (mediaDir: string): Promise => { + const entries = await readdir(uploadTempDir(mediaDir)).catch(() => []); + return entries + .filter((entry) => entry.endsWith(".part")) + .map((entry) => path.join(uploadTempDir(mediaDir), entry)); +}; + export const fileExists = async (absolute: string): Promise => { try { const info = await stat(absolute); diff --git a/docker/nginx/snippets/proxy-api.conf b/docker/nginx/snippets/proxy-api.conf index 9a024c5..67cad8e 100644 --- a/docker/nginx/snippets/proxy-api.conf +++ b/docker/nginx/snippets/proxy-api.conf @@ -13,6 +13,10 @@ proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; -proxy_read_timeout 60s; +proxy_read_timeout 120s; +proxy_send_timeout 120s; +# La finalisation d'un gros envoi resumable lit et hache le fichier complet +# avant de repondre : 120 s laisse la marge necessaire, sans depasser la +# limite de reponse de Cloudflare (~100 s). # Aligne sur la borne de depot du Cloud. client_max_body_size 5g; diff --git a/docs/api.md b/docs/api.md index 1f27284..317d128 100644 --- a/docs/api.md +++ b/docs/api.md @@ -54,6 +54,27 @@ d'adresse). | PUT | `/api/cloud/files/:id/star` | `requireAuth` | | DELETE | `/api/cloud/files/:id` | `requireAuth` + écriture (corbeille) | +### Envoi resumable (gros fichiers) + +Au-delà de la limite de corps de requête de Cloudflare (100 MB en Free/Pro), le +client découpe le fichier en morceaux de 50 MiB. Une session est créée en base, +le temporaire est préalloué sous `MEDIA_DIR/.uploads`, puis les morceaux sont +écrits à l'offset courant. La finalisation vérifie le nom et les droits une +dernière fois, calcule le SHA-256, publie le fichier et met à jour les quotas. + +| Méthode | Chemin | Garde | +| --- | --- | --- | +| POST | `/api/cloud/uploads` | `requireAuth` (nom, `folderId`, `size`, `mimeType`) | +| GET | `/api/cloud/uploads/:id` | `requireAuth` (propriétaire de la session, donne l'offset de reprise) | +| PUT | `/api/cloud/uploads/:id/chunks` | `requireAuth` (corps brut, `Content-Length` obligatoire) | +| POST | `/api/cloud/uploads/:id/complete` | `requireAuth` (vérifie taille, nom, droits, quotas) | +| DELETE | `/api/cloud/uploads/:id` | `requireAuth` (annule et supprime le temporaire) | + +Les sessions expirent après 24 h ; une purge au démarrage puis toutes les +heures supprime les temporaires orphelins. Un morceau interrompu ne fait pas +avancer le curseur : le client interroge `GET /uploads/:id` puis reprend à +l'offset exact. + ### Corbeille, recherche, galerie | Méthode | Chemin | Garde | @@ -157,6 +178,8 @@ par tous les membres authentifiés, en écriture pour son propriétaire. | Contrôle | Détail | | --- | --- | | Corps | `PUT /files` en flux brut (pas de multipart), `?name=` et `?folderId=` | +| Gros fichiers | au-delà de 90 MiB, le front bascule sur `/uploads` (morceaux de 50 MiB) pour rester sous la limite Cloudflare | +| Reprise | l'ID de session est conservé localement ; un envoi interrompu repart de `uploadedBytes` | | Taille | plafond `users.max_file_bytes`, vérifié en flux (coupure si dépassement) | | Quota | `users.used_bytes` + quota `storage_locations`, revérifiés avant écriture | | Disque | `diskUsage()` (`statfs`) avant écriture ; `507` `INSUFFICIENT_STORAGE` si le support est plein | @@ -197,11 +220,12 @@ par fichier, dérive refusée au démarrage. | `007_chat.sql` | `chat_messages` (salon global, auteur `ON DELETE SET NULL`) | | `008_user_preferences.sql` | préférences d'affichage, `user_file_stars`, `chat_reads` | | `009_chat_edits_and_pins.sql` | `chat_messages.edited_at`, `chat_messages.is_pinned`, index `idx_chat_messages_pinned`, `idx_chat_messages_edited` | +| `010_upload_sessions.sql` | `upload_sessions` (envoi resumable, temporaire préalloué, expiration) | Tables actives : `users, user_sessions, folders, folder_permissions, files, -media_metadata, file_variants, user_avatars, chat_messages, storage_locations, -settings, password_reset_tokens, email_change_requests, audit_logs, -user_file_stars, chat_reads`. +media_metadata, file_variants, user_avatars, chat_messages, upload_sessions, +storage_locations, settings, password_reset_tokens, email_change_requests, +audit_logs, user_file_stars, chat_reads`. ## Erreurs métier diff --git a/docs/deployment.md b/docs/deployment.md index fedae44..82506d2 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -177,6 +177,11 @@ adresse ni aucun domaine n'est codé en dur. de connexion globale, audit inutile). - Certificat et HSTS sont assurés en amont par Cloudflare — ne pas exposer directement le port 8083 sur Internet. +- **Corps de requête** : Cloudflare plafonne le corps à 100 MB (Free/Pro). Les + fichiers plus gros passent par l'envoi resumable (`/api/cloud/uploads`) en + morceaux de 50 MiB, sans relever la limite de corps de nginx ni exposer + l'origine. La valeur `client_max_body_size 5g` de nginx ne concerne que le + chemin direct local. **Tunnel Docker sur un réseau partagé (recommandé).** On joint nginx au réseau du tunnel plutôt que de publier un port : `cloudflared` résout `nginx` par nom, diff --git a/frontend/src/components/icons.tsx b/frontend/src/components/icons.tsx index 5c7edaf..5f76171 100644 --- a/frontend/src/components/icons.tsx +++ b/frontend/src/components/icons.tsx @@ -85,6 +85,12 @@ export const RestoreIcon = (props: IconProps) => ( ); +export const RefreshIcon = (props: IconProps) => ( + + + +); + export const StarIcon = ({ filled, ...props }: IconProps & { filled?: boolean }) => ( diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 35266aa..9b7f880 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -45,6 +45,20 @@ export interface UploadedFile { updated_at: string; } +/** Etat d'une session d'envoi resumable, cote serveur. */ +export interface UploadSession { + uploadId: string; + name: string; + folderId: string | null; + size: number; + chunkSize: number; + uploadedBytes: number; + expiresAt: string; +} + +/** Au-dela de ce seuil, l'envoi passe par des morceaux de 50 MiB. */ +export const LARGE_UPLOAD_THRESHOLD = 90 * 1024 * 1024; + export interface FavoriteFile extends UploadedFile { folder_name: string | null; } @@ -271,6 +285,150 @@ const putBinary = async (path: string, body: BodyInit, type: string): Promise return parsed as T; }; +/** + * Identifiant de reprise, stocke localement : si l'onglet est ferme pendant un + * gros envoi, le meme fichier reprend a l'octet pres au lieu de repartir de + * zero. La cle inclut la date de modification pour ne pas confondre deux + * fichiers homonymes. + */ +const resumeKey = (file: File, folderId: string | null): string => + `cohaise.upload:${folderId ?? "root"}:${file.name}:${file.size}:${file.lastModified}`; + +const readResume = (key: string): string | null => { + try { + return window.localStorage.getItem(key); + } catch { + return null; + } +}; + +const writeResume = (key: string, uploadId: string): void => { + try { + window.localStorage.setItem(key, uploadId); + } catch { + /* navigation privee : la reprise inter-onglets est simplement indisponible */ + } +}; + +const clearResume = (key: string): void => { + try { + window.localStorage.removeItem(key); + } catch { + /* voir writeResume */ + } +}; + +/** Envoi d'un morceau en XHR : `fetch` n'expose pas la progression montante. */ +const sendChunk = ( + uploadId: string, + blob: Blob, + onProgress: (loaded: number) => void, +): Promise => + new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("PUT", `${BASE}/uploads/${encodeURIComponent(uploadId)}/chunks`); + xhr.withCredentials = true; + xhr.setRequestHeader("content-type", "application/octet-stream"); + xhr.upload.onprogress = (event) => { + if (event.lengthComputable) onProgress(event.loaded); + }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + resolve(); + return; + } + let parsed: unknown = null; + try { + parsed = JSON.parse(xhr.responseText); + } catch { + /* reponse non JSON : message generique */ + } + reject(new ApiError(xhr.status, messageOf(parsed, "L'envoi du morceau a echoue."))); + }; + xhr.onerror = () => reject(new ApiError(0, "Serveur injoignable. Verifiez votre connexion.")); + xhr.onabort = () => reject(new ApiError(0, "Envoi interrompu.")); + xhr.send(blob); + }); + +const sessionPath = (uploadId: string): string => `/uploads/${encodeURIComponent(uploadId)}`; + +/** + * Envoi resumable en morceaux de 50 MiB. Chaque morceau reste sous la limite + * de corps de requete de Cloudflare ; le serveur ne publie le fichier qu'une + * fois tous les morceaux recus et le nom verifie. + */ +const uploadFileResumable = async ( + file: File, + folderId: string | null, + onProgress?: (loaded: number, total: number) => void, +): Promise<{ file: UploadedFile }> => { + const key = resumeKey(file, folderId); + let upload: UploadSession | null = null; + + const saved = readResume(key); + if (saved !== null) { + try { + const result = await request<{ upload: UploadSession }>(sessionPath(saved)); + if (result.upload.name === file.name && result.upload.size === file.size) { + upload = result.upload; + } else { + clearResume(key); + } + } catch (error) { + if (error instanceof ApiError && error.status === 404) clearResume(key); + else throw error; + } + } + + if (upload === null) { + const result = await post<{ upload: UploadSession }>("/uploads", { + name: file.name, + folderId, + size: file.size, + mimeType: file.type || null, + }); + upload = result.upload; + writeResume(key, upload.uploadId); + } + + const uploadId = upload.uploadId; + let offset = upload.uploadedBytes; + let attempts = 0; + + while (offset < file.size) { + const end = Math.min(offset + upload.chunkSize, file.size); + try { + await sendChunk(uploadId, file.slice(offset, end), (loaded) => + onProgress?.(offset + loaded, file.size), + ); + offset = end; + attempts = 0; + } catch (error) { + const transient = + !(error instanceof ApiError) || + error.status === 0 || + error.status === 409 || + error.status >= 500; + attempts += 1; + if (!transient || attempts > 4) throw error; + + // Laisse au serveur le temps de terminer (ou d'annuler) le morceau + // avant de lui demander ou en est reellement le curseur. + await new Promise((resolve) => setTimeout(resolve, 400 * attempts)); + const status = await request<{ upload: UploadSession }>(sessionPath(uploadId)); + if (status.upload.name !== file.name || status.upload.size !== file.size) { + throw new ApiError(409, "La session d'envoi ne correspond plus au fichier."); + } + offset = status.upload.uploadedBytes; + } + } + + onProgress?.(file.size, file.size); + const result = await post<{ file: UploadedFile }>(`${sessionPath(uploadId)}/complete`); + clearResume(key); + return result; +}; + export const api = { // --- Session ----------------------------------------------------------- me: () => request<{ user: User }>("/auth/me"), @@ -331,12 +489,26 @@ export const api = { // --- Fichiers ---------------------------------------------------------- files: (folderId?: string | null) => request<{ folderId: string | null; files: UploadedFile[] }>(`/files${query({ folderId })}`), - uploadFile: (file: File, folderId: string | null) => - putBinary<{ file: UploadedFile }>( + uploadFile: ( + file: File, + folderId: string | null, + onProgress?: (loaded: number, total: number) => void, + ) => { + if (file.size >= LARGE_UPLOAD_THRESHOLD) { + return uploadFileResumable(file, folderId, onProgress); + } + return putBinary<{ file: UploadedFile }>( `/files${query({ name: file.name, folderId })}`, file, file.type, - ), + ).then((result) => { + onProgress?.(file.size, file.size); + return result; + }); + }, + uploadSession: (uploadId: string) => + request<{ upload: UploadSession }>(sessionPath(uploadId)), + cancelUpload: (uploadId: string) => del<{ ok: boolean }>(sessionPath(uploadId)), renameFile: (id: string, name: string) => patch<{ file: UploadedFile }>(`/files/${encodeURIComponent(id)}`, { name }), moveFile: (id: string, folderId: string | null) => diff --git a/frontend/src/lib/i18n.tsx b/frontend/src/lib/i18n.tsx index dd2dec2..f4213ad 100644 --- a/frontend/src/lib/i18n.tsx +++ b/frontend/src/lib/i18n.tsx @@ -22,6 +22,7 @@ const en: Record = { "Confirmer": "Confirm", "Fermer": "Close", "Retour": "Back", + "Rafraîchir": "Refresh", "Filtrer": "Filter", "Tous les utilisateurs": "All users", "Toutes les actions": "All actions", diff --git a/frontend/src/pages/Browser.tsx b/frontend/src/pages/Browser.tsx index a8ac535..8421f22 100644 --- a/frontend/src/pages/Browser.tsx +++ b/frontend/src/pages/Browser.tsx @@ -37,6 +37,7 @@ import { FolderIcon, PenIcon, PlusIcon, + RefreshIcon, ShareIcon, StarIcon, TrashIcon, @@ -68,6 +69,10 @@ const KINDS = [ { value: "other", label: "Autres" }, ] as const; +// Rafraichissement de l'explorateur : un autre compte peut deposer ou +// supprimer un fichier dans un dossier partage sans prevenir. +const REFRESH_FOLDER_MS = 60_000; + export const Browser = () => { const { user, refresh } = useSession(); const { t } = useI18n(); @@ -80,6 +85,7 @@ export const Browser = () => { const [, setError] = useState(null); const [, setNotice] = useState(null); const [uploading, setUploading] = useState(false); + const [refreshing, setRefreshing] = useState(false); const [removing, setRemoving] = useState(null); const [dialog, setDialog] = useState(null); const [viewer, setViewer] = useState(null); @@ -96,36 +102,46 @@ export const Browser = () => { const [uploadProgress, setUploadProgress] = useState<{ current: number; total: number; + uploadedBytes: number; + totalBytes: number; } | null>(null); const current = path[path.length - 1] ?? { id: null, name: "Racine" }; const dragActive = dragDepth > 0; + const loadSeq = useRef(0); const load = useCallback( - async (parentId: string | null) => { - setLoading(true); + async (parentId: string | null, options?: { silent?: boolean }) => { + const sequence = ++loadSeq.current; + if (!options?.silent) setLoading(true); setError(null); try { const [folderResult, fileResult] = await Promise.all([ api.folders(parentId), api.files(parentId), ]); + if (sequence !== loadSeq.current) return; setFolders(folderResult.folders); setFiles(fileResult.files); } catch (caught) { + if (sequence !== loadSeq.current) return; const msg = caught instanceof ApiError ? caught.message : t("Chargement impossible."); - setError(msg); - toast.error(msg); - setFolders([]); - setFiles([]); + // Un rafraichissement automatique echoue en silence : pas de toast + // toutes les minutes si le reseau tombe pendant la consultation. + if (!options?.silent) { + setError(msg); + toast.error(msg); + setFolders([]); + setFiles([]); + } } finally { - setLoading(false); + if (!options?.silent) setLoading(false); } }, - [t], + [t, toast.error], ); const refreshUnread = useCallback(async () => { @@ -145,6 +161,18 @@ export const Browser = () => { void refreshUnread(); }, [current.id, refreshUnread]); + // Un autre compte peut deposer ou supprimer un fichier dans un dossier + // partage : la vue se resynchronise d'elle-meme tant que l'onglet est actif. + useEffect(() => { + const timer = window.setInterval(() => { + if (document.hidden) return; + if (uploading || viewer !== null || dialog !== null || chat !== null) return; + void load(current.id, { silent: true }); + void refreshUnread(); + }, REFRESH_FOLDER_MS); + return () => window.clearInterval(timer); + }, [chat, current.id, dialog, load, refreshUnread, uploading, viewer]); + // Le filtre de type n'est propre qu'au dossier affiche : on repart sur // « Tout » a chaque navigation, faute de voir un dossier vide pour rien. useEffect(() => { @@ -153,6 +181,17 @@ export const Browser = () => { const reload = useCallback(() => load(current.id), [current.id, load]); + const manualRefresh = useCallback(async () => { + if (refreshing) return; + setRefreshing(true); + try { + await load(current.id); + await refreshUnread(); + } finally { + setRefreshing(false); + } + }, [current.id, load, refreshUnread, refreshing]); + const open = (folder: Folder) => { setPath((previous) => [...previous, { id: folder.id, name: folder.name }]); }; @@ -165,14 +204,19 @@ export const Browser = () => { setUploading(true); setError(null); setNotice(null); - const total = selectedFiles.length; - setUploadProgress({ current: 0, total }); + const filesArr = Array.from(selectedFiles); + const total = filesArr.length; + const totalBytes = filesArr.reduce((sum, file) => sum + file.size, 0); + let completedBytes = 0; + const update = (current: number, loaded: number) => + setUploadProgress({ current, total, uploadedBytes: completedBytes + loaded, totalBytes }); + update(0, 0); try { - const filesArr = Array.from(selectedFiles); for (let i = 0; i < filesArr.length; i++) { const f = filesArr[i] as File; - setUploadProgress({ current: i + 1, total }); - await api.uploadFile(f, current.id); + update(i + 1, 0); + await api.uploadFile(f, current.id, (loaded) => update(i + 1, loaded)); + completedBytes += f.size; } void refresh(); await reload(); @@ -202,14 +246,19 @@ export const Browser = () => { setUploading(true); setError(null); setNotice(null); - const total = selectedFiles.length; - setUploadProgress({ current: 0, total }); + const filesArr = Array.from(selectedFiles); + const total = filesArr.length; + const totalBytes = filesArr.reduce((sum, file) => sum + file.size, 0); + let completedBytes = 0; + const update = (current: number, loaded: number) => + setUploadProgress({ current, total, uploadedBytes: completedBytes + loaded, totalBytes }); + update(0, 0); try { - const filesArr = Array.from(selectedFiles); for (let i = 0; i < filesArr.length; i++) { const f = filesArr[i] as File; - setUploadProgress({ current: i + 1, total }); - await api.uploadFile(f, folderId); + update(i + 1, 0); + await api.uploadFile(f, folderId, (loaded) => update(i + 1, loaded)); + completedBytes += f.size; } void refresh(); // Seule la vue courante est rechargée : un depot dans un sous-dossier ne @@ -489,6 +538,16 @@ export const Browser = () => { + } + isLoading={refreshing} + onClick={() => void manualRefresh()} + > + {t("Rafraîchir")} + + {current.id ? ( - {t("{current}/{total}", uploadProgress)} + {t("{current}/{total}", uploadProgress)} ·{" "} + {formatBytes(uploadProgress.uploadedBytes)} / {formatBytes(uploadProgress.totalBytes)}