refactor(storage): enforce per-user max_file_bytes limit on upload pipelines (#20)
CD — Deployment / deploy (push) Successful in 22s

* Validate Content-Length on single-part upload route against user limit
* Check total expected size on chunked upload session creation
* Re-evaluate user limit at finalization to reject dynamic quota reductions
* Clean up temporary files on payload size rejection (413)
* Add 60-second polling interval for periodic fetching of messages and images

---------

Co-authored-by: xEsiah <[email protected]>
Reviewed-on: #20
Reviewed-by: Esiah <[email protected]>
Co-authored-by: Gitea Actions <-[email protected]>
This commit was merged in pull request #20.
This commit is contained in:
Gitea Actions
2026-10-31 10:16:28 +00:00
committed by Esiah
co-authored by Esiah
parent b52417053f
commit 3542b33c89
12 changed files with 907 additions and 34 deletions
@@ -0,0 +1,36 @@
-- 010_upload_sessions.sql
--
-- Upload resumable pour les fichiers qui depassent la limite de corps de
-- requete de Cloudflare (100 MB en Free/Pro). Le client cree une session,
-- envoie des morceaux de 50 MiB, puis demande la finalisation.
--
-- Le fichier temporaire est prealloue a la taille finale : le disque est donc
-- verifie avant le premier octet utile, et les ecritures se font a une offset
-- connue sans jamais garder le fichier complet en memoire.
--
-- `received_bytes` avance uniquement quand la plage demandee correspond
-- exactement a la suite deja recue : les morceaux sont sequentiels, ce qui
-- rend la reprise simple (GET renvoie l'offset courant) et evite les trous.
CREATE TABLE upload_sessions (
id CHAR(36) NOT NULL,
user_id CHAR(36) NOT NULL,
folder_id CHAR(36) NULL,
storage_location_id INT UNSIGNED NOT NULL,
name VARCHAR(255) NOT NULL,
mime_type VARCHAR(255) NULL,
size_bytes BIGINT UNSIGNED NOT NULL,
received_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
chunk_size INT UNSIGNED NOT NULL,
relative_path VARCHAR(1024) NOT NULL,
temp_path VARCHAR(1024) NOT NULL,
expires_at DATETIME NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
KEY idx_upload_sessions_user (user_id),
KEY idx_upload_sessions_expiry (expires_at),
CONSTRAINT chk_upload_sessions_received CHECK (received_bytes <= size_bytes),
CONSTRAINT fk_upload_sessions_user FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
CONSTRAINT fk_upload_sessions_folder FOREIGN KEY (folder_id) REFERENCES folders (id) ON DELETE CASCADE,
CONSTRAINT fk_upload_sessions_location FOREIGN KEY (storage_location_id) REFERENCES storage_locations (id) ON DELETE RESTRICT
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+12 -1
View File
@@ -11,6 +11,7 @@ import { registerFolderRoutes } from "./routes/folders.js";
import { registerMediaRoutes } from "./routes/media.js";
import { registerSearchRoutes } from "./routes/search.js";
import { registerTrashRoutes, purgeExpiredTrash } from "./routes/trash.js";
import { purgeExpiredUploads, registerUploadRoutes } from "./routes/uploads.js";
// Les migrations sont relues au demarrage. L'operation est idempotente et
// refuse de servir si une migration deja appliquee a change : une base non
@@ -27,6 +28,7 @@ void startService({
registerDirectoryRoutes(app, db);
registerFolderRoutes(app, db, config);
registerFileRoutes(app, db, config);
registerUploadRoutes(app, db, config);
registerMediaRoutes(app, db);
registerTrashRoutes(app, db, config);
registerSearchRoutes(app, db);
@@ -36,6 +38,15 @@ void startService({
// Entretien au demarrage : sessions expirees et corbeille hors retention.
const sessions = await purgeExpiredSessions(db);
const purged = await purgeExpiredTrash(db, config);
logger.info("entretien au demarrage", { sessions, purged });
const uploads = await purgeExpiredUploads(db, config);
logger.info("entretien au demarrage", { sessions, purged, uploads });
// Les envois abandonnes ne doivent pas garder leur temporaire indefiniment :
// un balayage horaire suffit, la session expirant apres 24 h.
setInterval(() => {
void purgeExpiredUploads(db, config).catch((error: unknown) =>
logger.error("purge des envois inacheves", { error: String(error) }),
);
}, 60 * 60 * 1000).unref();
},
});
+5 -5
View File
@@ -56,7 +56,7 @@ const parse = <T>(schema: z.ZodType<T>, body: unknown): T => {
return result.data;
};
interface FileRow {
export interface FileRow {
id: string;
folder_id: string | null;
name: string;
@@ -71,13 +71,13 @@ interface FileRow {
updated_at: string;
}
const SELECT_FILE = `SELECT f.id, f.folder_id, f.name, f.mime_type, f.size_bytes, f.checksum_sha256,
export const SELECT_FILE = `SELECT f.id, f.folder_id, f.name, f.mime_type, f.size_bytes, f.checksum_sha256,
f.relative_path, f.owner_id,
EXISTS (SELECT 1 FROM user_file_stars s WHERE s.file_id = f.id AND s.user_id = ?) AS is_starred,
f.download_count, f.created_at, f.updated_at
FROM files f`;
const publicFile = (row: FileRow) => ({
export const publicFile = (row: FileRow) => ({
id: row.id,
folder_id: row.folder_id,
name: row.name,
@@ -92,7 +92,7 @@ const publicFile = (row: FileRow) => ({
});
/** Nom d'affichage : ni separateur, ni sequence d'echappement, ni point seul. */
const assertName = (raw: string | undefined): string => {
export const assertName = (raw: string | undefined): string => {
const name = (raw ?? "").trim();
if (!name || name.length > 255) {
throw AppError.badRequest("Le nom du fichier doit contenir entre 1 et 255 caracteres.");
@@ -103,7 +103,7 @@ const assertName = (raw: string | undefined): string => {
return name;
};
const mimeOf = (header: string | undefined): string => {
export const mimeOf = (header: string | undefined): string => {
const value = (header ?? "").split(";")[0]?.trim().toLowerCase() ?? "";
return MIME.test(value) ? value : "application/octet-stream";
};
+482
View File
@@ -0,0 +1,482 @@
import { randomUUID } from "node:crypto";
import { stat } from "node:fs/promises";
import path from "node:path";
import { z } from "zod";
import {
AppError,
attachUser,
currentUser,
requireAuth,
requireRole,
type App,
type Config,
type Db,
} from "../lib/index.js";
import { audit, canWrite } from "../access.js";
import { indexMedia } from "../media.js";
import {
absolutePath,
chooseLocation,
discard,
diskUsage,
ensureDir,
fileExists,
hashFile,
listUploadTemps,
preallocateUpload,
publish,
relativePathFor,
uploadTempPath,
writeChunk,
} from "../storage.js";
import { SELECT_FILE, assertName, mimeOf, publicFile, type FileRow } from "./files.js";
/**
* Upload resumable pour les fichiers qui depassent la limite de corps de
* requete de Cloudflare. Le client envoie des morceaux sequentiels de 50 MiB,
* ce qui reste sous la limite de 100 MB des offres Free/Pro.
*
* Le fichier temporaire est prealloue et rempli a l'offset courant. Une
* session ne peut appartenir qu'a son createur, et les droits d'ecriture sont
* revalides au moment de la finalisation : un partage retire pendant l'envoi
* ne permet pas de publier le fichier.
*/
const uuid = z
.string()
.regex(
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i,
"Identifiant invalide.",
);
/** 50 MiB : sous la limite Cloudflare (100 MB) avec une marge confortable. */
export const CHUNK_SIZE = 50 * 1024 * 1024;
/** Duree de vie d'une session abandonnee, avant purge du temporaire. */
const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
const createInput = z.object({
name: z.string().trim().min(1).max(255),
folderId: uuid.nullish(),
size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
mimeType: z.string().max(255).nullish(),
});
const parse = <T>(schema: z.ZodType<T>, body: unknown): T => {
const result = schema.safeParse(body);
if (!result.success) {
const issue = result.error.issues[0];
throw AppError.badRequest(
issue ? `${issue.path.join(".") || "corps"} : ${issue.message}` : "Corps invalide.",
);
}
return result.data;
};
interface UploadSessionRow {
id: string;
user_id: string;
folder_id: string | null;
storage_location_id: number;
name: string;
mime_type: string | null;
size_bytes: number | string;
received_bytes: number | string;
chunk_size: number;
relative_path: string;
temp_path: string;
expires_at: string | Date;
}
const SELECT_SESSION = `SELECT id, user_id, folder_id, storage_location_id, name, mime_type,
size_bytes, received_bytes, chunk_size, relative_path, temp_path, expires_at
FROM upload_sessions`;
const sessionPublic = (row: UploadSessionRow) => ({
uploadId: row.id,
name: row.name,
folderId: row.folder_id,
size: Number(row.size_bytes),
chunkSize: row.chunk_size,
uploadedBytes: Number(row.received_bytes),
expiresAt: new Date(row.expires_at).toISOString(),
});
/** Supprime la session et son temporaire, sans bruit si deja absente. */
const dropSession = async (db: Db, row: UploadSessionRow): Promise<void> => {
await discard(row.temp_path);
await db.exec("DELETE FROM upload_sessions WHERE id = ?", [row.id]).catch(() => undefined);
};
const loadSession = async (
db: Db,
user: { id: string },
id: string,
): Promise<UploadSessionRow> => {
if (!uuid.safeParse(id).success) throw AppError.notFound("Session d'envoi introuvable.");
const row = await db.one<UploadSessionRow>(`${SELECT_SESSION} WHERE id = ? AND user_id = ?`, [
id,
user.id,
]);
if (!row) throw AppError.notFound("Session d'envoi introuvable.");
if (new Date(row.expires_at).getTime() <= Date.now()) {
await dropSession(db, row);
throw AppError.notFound("Session d'envoi expiree.");
}
return row;
};
const assertNameFree = async (
db: Db,
user: { id: string },
folderId: string | null,
name: string,
): Promise<void> => {
const duplicate = await db.one(
`SELECT 1 AS taken FROM files
WHERE folder_id <=> ? AND active_name = ? AND deleted_at IS NULL
AND ( folder_id IS NOT NULL OR owner_id = ? )
LIMIT 1`,
[folderId, name, user.id],
);
if (duplicate) throw AppError.conflict("Ce nom existe deja ici.");
};
export const registerUploadRoutes = (app: App, db: Db, config: Config): void => {
app.use("/api/cloud/uploads/*", attachUser(db));
app.use("/api/cloud/uploads", attachUser(db));
app.post(
"/api/cloud/uploads",
requireAuth,
requireRole("admin", "user"),
async (context) => {
const user = currentUser(context);
const data = parse(createInput, await context.req.json().catch(() => null));
const name = assertName(data.name);
const folderId = data.folderId ?? null;
const size = data.size;
if (folderId !== null && !(await canWrite(db, user, folderId))) {
throw AppError.forbidden("Vous ne pouvez pas ecrire dans ce dossier.");
}
const budget = await db.one<{
used_bytes: number | string;
quota_bytes: number | string;
max_file_bytes: number | string;
}>(
"SELECT used_bytes, quota_bytes, max_file_bytes FROM users WHERE id = ? AND deleted_at IS NULL",
[user.id],
);
if (!budget) throw AppError.unauthorized();
// Les sessions en cours immobilisent deja une part du quota : sans
// cette somme, plusieurs envois simultanes pourraient chacun passer le
// controle puis depasser le quota au moment de la finalisation.
const pending = await db.one<{ bytes: number | string | null }>(
`SELECT COALESCE(SUM(size_bytes - received_bytes), 0) AS bytes
FROM upload_sessions
WHERE user_id = ? AND expires_at > UTC_TIMESTAMP()`,
[user.id],
);
if (BigInt(size) > BigInt(budget.max_file_bytes)) {
throw AppError.payloadTooLarge(
`Fichier trop volumineux (maximum ${budget.max_file_bytes} octets).`,
);
}
if (
BigInt(budget.used_bytes) + BigInt(pending?.bytes ?? 0) + BigInt(size) >
BigInt(budget.quota_bytes)
) {
throw AppError.payloadTooLarge("Votre quota de stockage est depasse.");
}
await assertNameFree(db, user, folderId, name);
const disk = await diskUsage(config.MEDIA_DIR);
if (disk !== null && BigInt(size) > BigInt(disk.available_bytes)) {
throw AppError.insufficientStorage(
"Espace disque insuffisant sur le serveur pour ce fichier.",
);
}
const location = await chooseLocation(db, BigInt(size));
const id = randomUUID();
const relative = relativePathFor(location.slug, id, name);
const temporary = uploadTempPath(config.MEDIA_DIR, id);
await preallocateUpload(temporary, size);
try {
await db.exec(
`INSERT INTO upload_sessions
(id, user_id, folder_id, storage_location_id, name, mime_type, size_bytes,
received_bytes, chunk_size, relative_path, temp_path, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`,
[
id,
user.id,
folderId,
location.id,
name,
mimeOf(data.mimeType ?? undefined),
size,
CHUNK_SIZE,
relative,
temporary,
new Date(Date.now() + SESSION_TTL_MS),
],
);
} catch (error) {
await discard(temporary);
throw error;
}
const created = await db.required<UploadSessionRow>(`${SELECT_SESSION} WHERE id = ?`, [id]);
await audit(
db,
context,
user,
"file.upload.start",
"file",
id,
{ name, size, folder_id: folderId },
config.TRUST_PROXY,
);
return context.json({ upload: sessionPublic(created) }, 201);
},
);
app.get("/api/cloud/uploads/:id", requireAuth, async (context) => {
const user = currentUser(context);
const row = await loadSession(db, user, context.req.param("id") ?? "");
return context.json({ upload: sessionPublic(row) });
});
app.put("/api/cloud/uploads/:id/chunks", requireAuth, async (context) => {
const user = currentUser(context);
const row = await loadSession(db, user, context.req.param("id") ?? "");
const offset = Number(row.received_bytes);
const size = Number(row.size_bytes);
const declaredHeader = context.req.header("content-length");
if (declaredHeader === undefined) {
throw new AppError(411, "LENGTH_REQUIRED", "En-tête Content-Length manquant.");
}
const declared = Number(declaredHeader);
if (!Number.isSafeInteger(declared) || declared <= 0) {
throw AppError.badRequest("Longueur de morceau invalide.");
}
if (declared > row.chunk_size) {
throw AppError.payloadTooLarge(`Morceau trop volumineux (maximum ${row.chunk_size} octets).`);
}
if (offset + declared > size) {
throw AppError.badRequest("Le morceau depasse la taille annoncee du fichier.");
}
const body = context.req.raw.body;
if (!body) throw AppError.badRequest("Corps de requete vide.");
// Ecriture d'abord, curseur ensuite : un morceau interrompu laisse le
// curseur intact, donc le prochain essai reecrit la meme plage sans creer
// de trou. Deux requetes concurrentes sur la meme plage ecrivent les
// memes octets : une seule avance le curseur, l'autre constate que le
// morceau est deja applique.
const written = await writeChunk(row.temp_path, offset, body, declared);
if (written !== declared) {
throw AppError.badRequest(
`Longueur incoherente : ${declared} octets annonces, ${written} recus.`,
);
}
const reserved = await db.exec(
`UPDATE upload_sessions
SET received_bytes = received_bytes + ?
WHERE id = ? AND user_id = ? AND received_bytes = ?
AND received_bytes + ? <= size_bytes AND expires_at > UTC_TIMESTAMP()`,
[declared, row.id, user.id, offset, declared],
);
if (reserved.affectedRows === 0) {
const fresh = await db.one<UploadSessionRow>(
`${SELECT_SESSION} WHERE id = ? AND user_id = ?`,
[row.id, user.id],
);
if (!fresh) throw AppError.notFound("Session d'envoi introuvable.");
if (Number(fresh.received_bytes) < offset + declared) {
throw AppError.conflict("Morceau deja recu ou session modifiee. Reprenez depuis le statut.");
}
return context.json({ upload: sessionPublic(fresh) });
}
return context.json({ upload: sessionPublic({ ...row, received_bytes: offset + declared }) });
});
app.post("/api/cloud/uploads/:id/complete", requireAuth, async (context) => {
const user = currentUser(context);
const row = await loadSession(db, user, context.req.param("id") ?? "");
const size = Number(row.size_bytes);
const received = Number(row.received_bytes);
if (received !== size) {
throw AppError.badRequest("Envoi incomplet : des morceaux sont encore manquants.");
}
if (!(await fileExists(row.temp_path))) {
await dropSession(db, row);
throw AppError.notFound("Fichier temporaire introuvable. Relancez l'envoi.");
}
// La limite de taille est propre au compte : elle peut avoir baisse
// pendant un envoi long, il faut donc la reverifier avant de publier.
const budget = await db.one<{ max_file_bytes: number | string }>(
"SELECT max_file_bytes FROM users WHERE id = ? AND deleted_at IS NULL",
[user.id],
);
if (!budget) {
await dropSession(db, row);
throw AppError.unauthorized();
}
if (BigInt(size) > BigInt(budget.max_file_bytes)) {
await dropSession(db, row);
throw AppError.payloadTooLarge(
`Fichier trop volumineux (maximum ${budget.max_file_bytes} octets).`,
);
}
if (row.folder_id !== null && !(await canWrite(db, user, row.folder_id))) {
await dropSession(db, row);
throw AppError.forbidden("Vous ne pouvez plus ecrire dans ce dossier.");
}
try {
await assertNameFree(db, user, row.folder_id, row.name);
} catch (error) {
await dropSession(db, row);
throw error;
}
const checksum = await hashFile(row.temp_path);
const destination = absolutePath(config.MEDIA_DIR, row.relative_path);
// Le dossier de l'emplacement peut ne pas encore exister : l'envoi direct
// le cree avant d'ecrire, l'envoi resumable ecrit d'abord dans .uploads.
await ensureDir(path.dirname(destination));
// Publication avant la base, comme le depot direct : si la transaction
// echoue, le temporaire est remis en place pour permettre un nouvel essai.
await publish(row.temp_path, destination);
try {
await db.transaction(async (connection) => {
const [quota] = await connection.execute(
`UPDATE users SET used_bytes = used_bytes + ?
WHERE id = ? AND used_bytes + ? <= quota_bytes`,
[size, user.id, size],
);
if ((quota as { affectedRows: number }).affectedRows === 0) {
throw AppError.payloadTooLarge("Votre quota de stockage est depasse.");
}
const [support] = await connection.execute(
`UPDATE storage_locations
SET used_bytes = used_bytes + ?
WHERE id = ? AND (quota_bytes IS NULL OR used_bytes + ? <= quota_bytes)`,
[size, row.storage_location_id, size],
);
if ((support as { affectedRows: number }).affectedRows === 0) {
throw AppError.payloadTooLarge("L'espace disponible sur le support est insuffisant.");
}
await connection.execute(
`INSERT INTO files
(id, storage_location_id, folder_id, name, mime_type, size_bytes,
checksum_sha256, relative_path, owner_id, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
row.id,
row.storage_location_id,
row.folder_id,
row.name,
row.mime_type ?? "application/octet-stream",
size,
checksum,
row.relative_path,
user.id,
user.id,
],
);
await connection.execute("DELETE FROM upload_sessions WHERE id = ?", [row.id]);
});
} catch (error) {
await publish(destination, row.temp_path).catch(async () => {
await discard(destination);
await db.exec("DELETE FROM upload_sessions WHERE id = ?", [row.id]).catch(() => undefined);
});
if ((error as { errno?: number }).errno === 1062) {
throw AppError.conflict("Ce nom existe deja ici.");
}
throw error;
}
await indexMedia(db, config, row.id, row.mime_type ?? "application/octet-stream", destination).catch(
() => undefined,
);
await audit(
db,
context,
user,
"file.upload",
"file",
row.id,
{ name: row.name, size, folder_id: row.folder_id },
config.TRUST_PROXY,
);
const created = await db.required<FileRow>(`${SELECT_FILE} WHERE f.id = ?`, [user.id, row.id]);
return context.json({ file: publicFile(created) }, 201);
});
app.delete("/api/cloud/uploads/:id", requireAuth, async (context) => {
const user = currentUser(context);
const row = await loadSession(db, user, context.req.param("id") ?? "");
await dropSession(db, row);
await audit(
db,
context,
user,
"file.upload.cancel",
"file",
row.id,
{ name: row.name },
config.TRUST_PROXY,
);
return context.json({ ok: true });
});
};
/**
* Purge les sessions arrivees a echeance et les temporaires orphelins (session
* supprimee avec le dossier ou le compte, par exemple). Appelee au demarrage
* puis periodiquement par le service.
*/
export const purgeExpiredUploads = async (db: Db, config: Config): Promise<number> => {
const expired = await db.query<UploadSessionRow>(
`${SELECT_SESSION} WHERE expires_at <= UTC_TIMESTAMP()`,
);
for (const row of expired) await discard(row.temp_path);
const result = await db.exec("DELETE FROM upload_sessions WHERE expires_at <= UTC_TIMESTAMP()");
const cutoff = Date.now() - SESSION_TTL_MS;
for (const temporary of await listUploadTemps(config.MEDIA_DIR)) {
const info = await stat(temporary).catch(() => null);
if (info !== null && info.mtimeMs < cutoff) await discard(temporary);
}
return result.affectedRows;
};
+68 -1
View File
@@ -1,6 +1,6 @@
import { createHash, randomUUID } from "node:crypto";
import { createReadStream, createWriteStream } from "node:fs";
import { mkdir, rename, rm, stat, statfs } from "node:fs/promises";
import { mkdir, open, readdir, rename, rm, stat, statfs } from "node:fs/promises";
import path from "node:path";
import { Readable, Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
@@ -130,6 +130,73 @@ export const discard = async (...paths: string[]): Promise<void> => {
export const temporaryPath = (mediaDir: string, slug: string): string =>
path.join(path.resolve(mediaDir), slug, `.upload-${randomUUID()}`);
/**
* Dossier des uploads resumables. Il reste sous MEDIA_DIR : le renommage
* final vers l'emplacement de stockage est donc atomique.
*/
export const uploadTempDir = (mediaDir: string): string =>
path.join(path.resolve(mediaDir), ".uploads");
export const uploadTempPath = (mediaDir: string, id: string): string =>
path.join(uploadTempDir(mediaDir), `${id}.part`);
/** Prealloue le temporaire a la taille annoncee, sans ecrire les octets. */
export const preallocateUpload = async (destination: string, size: number): Promise<void> => {
await ensureDir(path.dirname(destination));
const handle = await open(destination, "wx");
try {
await handle.truncate(size);
} finally {
await handle.close();
}
};
/**
* Ecrit un morceau a l'offset courant. Le flux est borne : un client qui
* annonce une taille et en envoie davantage est coupe avant d'ecrire l'exces.
*/
export const writeChunk = async (
destination: string,
offset: number,
body: ReadableStream<Uint8Array>,
maxBytes: number,
): Promise<number> => {
let received = 0;
const meter = new Transform({
transform(chunk: Buffer, _encoding, callback) {
received += chunk.length;
if (received > maxBytes) {
callback(AppError.payloadTooLarge(`Morceau trop volumineux (maximum ${maxBytes} octets).`));
return;
}
callback(null, chunk);
},
});
const source = Readable.fromWeb(body as Parameters<typeof Readable.fromWeb>[0]);
await pipeline(source, meter, createWriteStream(destination, { flags: "r+", start: offset }));
return received;
};
/** SHA-256 du fichier complet, calcule au moment de la finalisation. */
export const hashFile = async (absolute: string): Promise<string> => {
const hash = createHash("sha256");
for await (const chunk of createReadStream(absolute)) {
hash.update(chunk as Buffer);
}
return hash.digest("hex");
};
/** Fichiers `.part` abandonnes : utilises par la purge des sessions expirees. */
export const listUploadTemps = async (mediaDir: string): Promise<string[]> => {
const entries = await readdir(uploadTempDir(mediaDir)).catch(() => []);
return entries
.filter((entry) => entry.endsWith(".part"))
.map((entry) => path.join(uploadTempDir(mediaDir), entry));
};
export const fileExists = async (absolute: string): Promise<boolean> => {
try {
const info = await stat(absolute);
+5 -1
View File
@@ -13,6 +13,10 @@ proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
proxy_read_timeout 60s;
proxy_read_timeout 120s;
proxy_send_timeout 120s;
# La finalisation d'un gros envoi resumable lit et hache le fichier complet
# avant de repondre : 120 s laisse la marge necessaire, sans depasser la
# limite de reponse de Cloudflare (~100 s).
# Aligne sur la borne de depot du Cloud.
client_max_body_size 5g;
+27 -3
View File
@@ -54,6 +54,27 @@ d'adresse).
| PUT | `/api/cloud/files/:id/star` | `requireAuth` |
| DELETE | `/api/cloud/files/:id` | `requireAuth` + écriture (corbeille) |
### Envoi resumable (gros fichiers)
Au-delà de la limite de corps de requête de Cloudflare (100 MB en Free/Pro), le
client découpe le fichier en morceaux de 50 MiB. Une session est créée en base,
le temporaire est préalloué sous `MEDIA_DIR/.uploads`, puis les morceaux sont
écrits à l'offset courant. La finalisation vérifie le nom et les droits une
dernière fois, calcule le SHA-256, publie le fichier et met à jour les quotas.
| Méthode | Chemin | Garde |
| --- | --- | --- |
| POST | `/api/cloud/uploads` | `requireAuth` (nom, `folderId`, `size`, `mimeType`) |
| GET | `/api/cloud/uploads/:id` | `requireAuth` (propriétaire de la session, donne l'offset de reprise) |
| PUT | `/api/cloud/uploads/:id/chunks` | `requireAuth` (corps brut, `Content-Length` obligatoire) |
| POST | `/api/cloud/uploads/:id/complete` | `requireAuth` (vérifie taille, nom, droits, quotas) |
| DELETE | `/api/cloud/uploads/:id` | `requireAuth` (annule et supprime le temporaire) |
Les sessions expirent après 24 h ; une purge au démarrage puis toutes les
heures supprime les temporaires orphelins. Un morceau interrompu ne fait pas
avancer le curseur : le client interroge `GET /uploads/:id` puis reprend à
l'offset exact.
### Corbeille, recherche, galerie
| Méthode | Chemin | Garde |
@@ -157,6 +178,8 @@ par tous les membres authentifiés, en écriture pour son propriétaire.
| Contrôle | Détail |
| --- | --- |
| Corps | `PUT /files` en flux brut (pas de multipart), `?name=` et `?folderId=` |
| Gros fichiers | au-delà de 90 MiB, le front bascule sur `/uploads` (morceaux de 50 MiB) pour rester sous la limite Cloudflare |
| Reprise | l'ID de session est conservé localement ; un envoi interrompu repart de `uploadedBytes` |
| Taille | plafond `users.max_file_bytes`, vérifié en flux (coupure si dépassement) |
| Quota | `users.used_bytes` + quota `storage_locations`, revérifiés avant écriture |
| Disque | `diskUsage()` (`statfs`) avant écriture ; `507` `INSUFFICIENT_STORAGE` si le support est plein |
@@ -197,11 +220,12 @@ par fichier, dérive refusée au démarrage.
| `007_chat.sql` | `chat_messages` (salon global, auteur `ON DELETE SET NULL`) |
| `008_user_preferences.sql` | préférences d'affichage, `user_file_stars`, `chat_reads` |
| `009_chat_edits_and_pins.sql` | `chat_messages.edited_at`, `chat_messages.is_pinned`, index `idx_chat_messages_pinned`, `idx_chat_messages_edited` |
| `010_upload_sessions.sql` | `upload_sessions` (envoi resumable, temporaire préalloué, expiration) |
Tables actives : `users, user_sessions, folders, folder_permissions, files,
media_metadata, file_variants, user_avatars, chat_messages, storage_locations,
settings, password_reset_tokens, email_change_requests, audit_logs,
user_file_stars, chat_reads`.
media_metadata, file_variants, user_avatars, chat_messages, upload_sessions,
storage_locations, settings, password_reset_tokens, email_change_requests,
audit_logs, user_file_stars, chat_reads`.
## Erreurs métier
+5
View File
@@ -177,6 +177,11 @@ adresse ni aucun domaine n'est codé en dur.
de connexion globale, audit inutile).
- Certificat et HSTS sont assurés en amont par Cloudflare — ne pas exposer
directement le port 8083 sur Internet.
- **Corps de requête** : Cloudflare plafonne le corps à 100 MB (Free/Pro). Les
fichiers plus gros passent par l'envoi resumable (`/api/cloud/uploads`) en
morceaux de 50 MiB, sans relever la limite de corps de nginx ni exposer
l'origine. La valeur `client_max_body_size 5g` de nginx ne concerne que le
chemin direct local.
**Tunnel Docker sur un réseau partagé (recommandé).** On joint nginx au réseau
du tunnel plutôt que de publier un port : `cloudflared` résout `nginx` par nom,
+6
View File
@@ -85,6 +85,12 @@ export const RestoreIcon = (props: IconProps) => (
</svg>
);
export const RefreshIcon = (props: IconProps) => (
<svg {...base(props)}>
<path d="M20 12a8 8 0 1 1-2.3-5.6M20 4v4h-4" />
</svg>
);
export const StarIcon = ({ filled, ...props }: IconProps & { filled?: boolean }) => (
<svg {...base(props)} fill={filled ? "currentColor" : "none"}>
<path d="m12 3.5 2.6 5.3 5.9.9-4.2 4.1 1 5.8L12 20.8l-5.3 2.8 1-5.8L3.5 14l5.9-.9Z" />
+175 -3
View File
@@ -45,6 +45,20 @@ export interface UploadedFile {
updated_at: string;
}
/** Etat d'une session d'envoi resumable, cote serveur. */
export interface UploadSession {
uploadId: string;
name: string;
folderId: string | null;
size: number;
chunkSize: number;
uploadedBytes: number;
expiresAt: string;
}
/** Au-dela de ce seuil, l'envoi passe par des morceaux de 50 MiB. */
export const LARGE_UPLOAD_THRESHOLD = 90 * 1024 * 1024;
export interface FavoriteFile extends UploadedFile {
folder_name: string | null;
}
@@ -271,6 +285,150 @@ const putBinary = async <T>(path: string, body: BodyInit, type: string): Promise
return parsed as T;
};
/**
* Identifiant de reprise, stocke localement : si l'onglet est ferme pendant un
* gros envoi, le meme fichier reprend a l'octet pres au lieu de repartir de
* zero. La cle inclut la date de modification pour ne pas confondre deux
* fichiers homonymes.
*/
const resumeKey = (file: File, folderId: string | null): string =>
`cohaise.upload:${folderId ?? "root"}:${file.name}:${file.size}:${file.lastModified}`;
const readResume = (key: string): string | null => {
try {
return window.localStorage.getItem(key);
} catch {
return null;
}
};
const writeResume = (key: string, uploadId: string): void => {
try {
window.localStorage.setItem(key, uploadId);
} catch {
/* navigation privee : la reprise inter-onglets est simplement indisponible */
}
};
const clearResume = (key: string): void => {
try {
window.localStorage.removeItem(key);
} catch {
/* voir writeResume */
}
};
/** Envoi d'un morceau en XHR : `fetch` n'expose pas la progression montante. */
const sendChunk = (
uploadId: string,
blob: Blob,
onProgress: (loaded: number) => void,
): Promise<void> =>
new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("PUT", `${BASE}/uploads/${encodeURIComponent(uploadId)}/chunks`);
xhr.withCredentials = true;
xhr.setRequestHeader("content-type", "application/octet-stream");
xhr.upload.onprogress = (event) => {
if (event.lengthComputable) onProgress(event.loaded);
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
resolve();
return;
}
let parsed: unknown = null;
try {
parsed = JSON.parse(xhr.responseText);
} catch {
/* reponse non JSON : message generique */
}
reject(new ApiError(xhr.status, messageOf(parsed, "L'envoi du morceau a echoue.")));
};
xhr.onerror = () => reject(new ApiError(0, "Serveur injoignable. Verifiez votre connexion."));
xhr.onabort = () => reject(new ApiError(0, "Envoi interrompu."));
xhr.send(blob);
});
const sessionPath = (uploadId: string): string => `/uploads/${encodeURIComponent(uploadId)}`;
/**
* Envoi resumable en morceaux de 50 MiB. Chaque morceau reste sous la limite
* de corps de requete de Cloudflare ; le serveur ne publie le fichier qu'une
* fois tous les morceaux recus et le nom verifie.
*/
const uploadFileResumable = async (
file: File,
folderId: string | null,
onProgress?: (loaded: number, total: number) => void,
): Promise<{ file: UploadedFile }> => {
const key = resumeKey(file, folderId);
let upload: UploadSession | null = null;
const saved = readResume(key);
if (saved !== null) {
try {
const result = await request<{ upload: UploadSession }>(sessionPath(saved));
if (result.upload.name === file.name && result.upload.size === file.size) {
upload = result.upload;
} else {
clearResume(key);
}
} catch (error) {
if (error instanceof ApiError && error.status === 404) clearResume(key);
else throw error;
}
}
if (upload === null) {
const result = await post<{ upload: UploadSession }>("/uploads", {
name: file.name,
folderId,
size: file.size,
mimeType: file.type || null,
});
upload = result.upload;
writeResume(key, upload.uploadId);
}
const uploadId = upload.uploadId;
let offset = upload.uploadedBytes;
let attempts = 0;
while (offset < file.size) {
const end = Math.min(offset + upload.chunkSize, file.size);
try {
await sendChunk(uploadId, file.slice(offset, end), (loaded) =>
onProgress?.(offset + loaded, file.size),
);
offset = end;
attempts = 0;
} catch (error) {
const transient =
!(error instanceof ApiError) ||
error.status === 0 ||
error.status === 409 ||
error.status >= 500;
attempts += 1;
if (!transient || attempts > 4) throw error;
// Laisse au serveur le temps de terminer (ou d'annuler) le morceau
// avant de lui demander ou en est reellement le curseur.
await new Promise((resolve) => setTimeout(resolve, 400 * attempts));
const status = await request<{ upload: UploadSession }>(sessionPath(uploadId));
if (status.upload.name !== file.name || status.upload.size !== file.size) {
throw new ApiError(409, "La session d'envoi ne correspond plus au fichier.");
}
offset = status.upload.uploadedBytes;
}
}
onProgress?.(file.size, file.size);
const result = await post<{ file: UploadedFile }>(`${sessionPath(uploadId)}/complete`);
clearResume(key);
return result;
};
export const api = {
// --- Session -----------------------------------------------------------
me: () => request<{ user: User }>("/auth/me"),
@@ -331,12 +489,26 @@ export const api = {
// --- Fichiers ----------------------------------------------------------
files: (folderId?: string | null) =>
request<{ folderId: string | null; files: UploadedFile[] }>(`/files${query({ folderId })}`),
uploadFile: (file: File, folderId: string | null) =>
putBinary<{ file: UploadedFile }>(
uploadFile: (
file: File,
folderId: string | null,
onProgress?: (loaded: number, total: number) => void,
) => {
if (file.size >= LARGE_UPLOAD_THRESHOLD) {
return uploadFileResumable(file, folderId, onProgress);
}
return putBinary<{ file: UploadedFile }>(
`/files${query({ name: file.name, folderId })}`,
file,
file.type,
),
).then((result) => {
onProgress?.(file.size, file.size);
return result;
});
},
uploadSession: (uploadId: string) =>
request<{ upload: UploadSession }>(sessionPath(uploadId)),
cancelUpload: (uploadId: string) => del<{ ok: boolean }>(sessionPath(uploadId)),
renameFile: (id: string, name: string) =>
patch<{ file: UploadedFile }>(`/files/${encodeURIComponent(id)}`, { name }),
moveFile: (id: string, folderId: string | null) =>
+1
View File
@@ -22,6 +22,7 @@ const en: Record<string, string> = {
"Confirmer": "Confirm",
"Fermer": "Close",
"Retour": "Back",
"Rafraîchir": "Refresh",
"Filtrer": "Filter",
"Tous les utilisateurs": "All users",
"Toutes les actions": "All actions",
+85 -20
View File
@@ -37,6 +37,7 @@ import {
FolderIcon,
PenIcon,
PlusIcon,
RefreshIcon,
ShareIcon,
StarIcon,
TrashIcon,
@@ -68,6 +69,10 @@ const KINDS = [
{ value: "other", label: "Autres" },
] as const;
// Rafraichissement de l'explorateur : un autre compte peut deposer ou
// supprimer un fichier dans un dossier partage sans prevenir.
const REFRESH_FOLDER_MS = 60_000;
export const Browser = () => {
const { user, refresh } = useSession();
const { t } = useI18n();
@@ -80,6 +85,7 @@ export const Browser = () => {
const [, setError] = useState<string | null>(null);
const [, setNotice] = useState<string | null>(null);
const [uploading, setUploading] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const [removing, setRemoving] = useState<string | null>(null);
const [dialog, setDialog] = useState<Dialog>(null);
const [viewer, setViewer] = useState<number | null>(null);
@@ -96,36 +102,46 @@ export const Browser = () => {
const [uploadProgress, setUploadProgress] = useState<{
current: number;
total: number;
uploadedBytes: number;
totalBytes: number;
} | null>(null);
const current = path[path.length - 1] ?? { id: null, name: "Racine" };
const dragActive = dragDepth > 0;
const loadSeq = useRef(0);
const load = useCallback(
async (parentId: string | null) => {
setLoading(true);
async (parentId: string | null, options?: { silent?: boolean }) => {
const sequence = ++loadSeq.current;
if (!options?.silent) setLoading(true);
setError(null);
try {
const [folderResult, fileResult] = await Promise.all([
api.folders(parentId),
api.files(parentId),
]);
if (sequence !== loadSeq.current) return;
setFolders(folderResult.folders);
setFiles(fileResult.files);
} catch (caught) {
if (sequence !== loadSeq.current) return;
const msg =
caught instanceof ApiError
? caught.message
: t("Chargement impossible.");
setError(msg);
toast.error(msg);
setFolders([]);
setFiles([]);
// Un rafraichissement automatique echoue en silence : pas de toast
// toutes les minutes si le reseau tombe pendant la consultation.
if (!options?.silent) {
setError(msg);
toast.error(msg);
setFolders([]);
setFiles([]);
}
} finally {
setLoading(false);
if (!options?.silent) setLoading(false);
}
},
[t],
[t, toast.error],
);
const refreshUnread = useCallback(async () => {
@@ -145,6 +161,18 @@ export const Browser = () => {
void refreshUnread();
}, [current.id, refreshUnread]);
// Un autre compte peut deposer ou supprimer un fichier dans un dossier
// partage : la vue se resynchronise d'elle-meme tant que l'onglet est actif.
useEffect(() => {
const timer = window.setInterval(() => {
if (document.hidden) return;
if (uploading || viewer !== null || dialog !== null || chat !== null) return;
void load(current.id, { silent: true });
void refreshUnread();
}, REFRESH_FOLDER_MS);
return () => window.clearInterval(timer);
}, [chat, current.id, dialog, load, refreshUnread, uploading, viewer]);
// Le filtre de type n'est propre qu'au dossier affiche : on repart sur
// « Tout » a chaque navigation, faute de voir un dossier vide pour rien.
useEffect(() => {
@@ -153,6 +181,17 @@ export const Browser = () => {
const reload = useCallback(() => load(current.id), [current.id, load]);
const manualRefresh = useCallback(async () => {
if (refreshing) return;
setRefreshing(true);
try {
await load(current.id);
await refreshUnread();
} finally {
setRefreshing(false);
}
}, [current.id, load, refreshUnread, refreshing]);
const open = (folder: Folder) => {
setPath((previous) => [...previous, { id: folder.id, name: folder.name }]);
};
@@ -165,14 +204,19 @@ export const Browser = () => {
setUploading(true);
setError(null);
setNotice(null);
const total = selectedFiles.length;
setUploadProgress({ current: 0, total });
const filesArr = Array.from(selectedFiles);
const total = filesArr.length;
const totalBytes = filesArr.reduce((sum, file) => sum + file.size, 0);
let completedBytes = 0;
const update = (current: number, loaded: number) =>
setUploadProgress({ current, total, uploadedBytes: completedBytes + loaded, totalBytes });
update(0, 0);
try {
const filesArr = Array.from(selectedFiles);
for (let i = 0; i < filesArr.length; i++) {
const f = filesArr[i] as File;
setUploadProgress({ current: i + 1, total });
await api.uploadFile(f, current.id);
update(i + 1, 0);
await api.uploadFile(f, current.id, (loaded) => update(i + 1, loaded));
completedBytes += f.size;
}
void refresh();
await reload();
@@ -202,14 +246,19 @@ export const Browser = () => {
setUploading(true);
setError(null);
setNotice(null);
const total = selectedFiles.length;
setUploadProgress({ current: 0, total });
const filesArr = Array.from(selectedFiles);
const total = filesArr.length;
const totalBytes = filesArr.reduce((sum, file) => sum + file.size, 0);
let completedBytes = 0;
const update = (current: number, loaded: number) =>
setUploadProgress({ current, total, uploadedBytes: completedBytes + loaded, totalBytes });
update(0, 0);
try {
const filesArr = Array.from(selectedFiles);
for (let i = 0; i < filesArr.length; i++) {
const f = filesArr[i] as File;
setUploadProgress({ current: i + 1, total });
await api.uploadFile(f, folderId);
update(i + 1, 0);
await api.uploadFile(f, folderId, (loaded) => update(i + 1, loaded));
completedBytes += f.size;
}
void refresh();
// Seule la vue courante est rechargée : un depot dans un sous-dossier ne
@@ -489,6 +538,16 @@ export const Browser = () => {
</ol>
</nav>
<Button
variant="secondary"
size="sm"
iconLeading={<RefreshIcon className="h-3.5 w-3.5" />}
isLoading={refreshing}
onClick={() => void manualRefresh()}
>
{t("Rafraîchir")}
</Button>
{current.id ? (
<Button
variant="secondary"
@@ -539,14 +598,20 @@ export const Browser = () => {
{t("Envoi en cours")}
</span>
<span className="text-xs text-muted-foreground">
{t("{current}/{total}", uploadProgress)}
{t("{current}/{total}", uploadProgress)} ·{" "}
{formatBytes(uploadProgress.uploadedBytes)} / {formatBytes(uploadProgress.totalBytes)}
</span>
</div>
<div className="h-1.5 w-full overflow-hidden rounded-full bg-surface-muted">
<div
className="h-full rounded-full bg-accent transition-[width] duration-200"
style={{
width: `${Math.round((uploadProgress.current / uploadProgress.total) * 100)}%`,
width: `${Math.min(
100,
Math.round(
(uploadProgress.uploadedBytes / Math.max(1, uploadProgress.totalBytes)) * 100,
),
)}%`,
}}
/>
</div>