feature: file uploading

.gitignore

@@ -1,2 +1,3 @@
 data/
 .fsdb
+public/files/*

README.md

@@ -50,6 +50,7 @@ feature discussions.
 		- [X] punycode urls before url extraction? (see: https://stackoverflow.com/a/26618995)
 		- [ ] gif support
 			- [X] gif embeds
+			- [X] mp4 embeds
 			- [ ] start/stop gif control
 			- [ ] hide control
 		- [ ] inline image support

deno.json

@@ -7,7 +7,7 @@
 	"tasks": {
 		"lint": "deno lint",
 		"fmt": "deno fmt",
-		"serve": "FSDB_ROOT=$PWD/.fsdb TRACE_ERROR_RESPONSES=true SERVERUS_TYPESCRIPT_IMPORT_LOGGING=true deno --allow-env --allow-read --allow-write --allow-net @andyburke/serverus --root ./public --hostname 0.0.0.0",
+		"serve": "FSDB_ROOT=$PWD/.fsdb TRACE_ERROR_RESPONSES=true SERVERUS_TYPESCRIPT_IMPORT_LOGGING=true SERVERUS_PUT_PATHS_ALLOWED=./public/files SERVERUS_DELETE_PATHS_ALLOWED=./public/files deno --allow-env --allow-read --allow-write --allow-net @andyburke/serverus --root ./public --hostname 0.0.0.0",
 		"test": "DENO_ENV=test FSDB_ROOT=$PWD/tests/data/$(date --iso-8601=seconds) SERVERUS_ROOT=$PWD/public deno test --allow-env --allow-read --allow-write --allow-net --allow-import --trace-leaks --fail-fast tests/"
 	},
 	"test": {
@@ -34,10 +34,11 @@
 	"imports": {
 		"@andyburke/fsdb": "jsr:@andyburke/fsdb@^1.0.2",
 		"@andyburke/lurid": "jsr:@andyburke/lurid@^0.2.0",
-		"@andyburke/serverus": "jsr:@andyburke/serverus@^0.9.8",
+		"@andyburke/serverus": "jsr:@andyburke/serverus@^0.12.2",
 		"@da/bcrypt": "jsr:@da/bcrypt@^1.0.1",
 		"@std/assert": "jsr:@std/assert@^1.0.13",
 		"@std/encoding": "jsr:@std/encoding@^1.0.10",
+		"@std/fs": "jsr:@std/fs@^1.0.19",
 		"@std/http": "jsr:@std/http@^1.0.20",
 		"@std/path": "jsr:@std/path@^1.1.1"
 	}

deno.lock

@@ -3,11 +3,10 @@
   "specifiers": {
     "jsr:@andyburke/fsdb@^1.0.2": "1.0.2",
     "jsr:@andyburke/lurid@0.2": "0.2.0",
-    "jsr:@andyburke/serverus@~0.9.8": "0.9.8",
+    "jsr:@andyburke/serverus@~0.12.2": "0.12.2",
     "jsr:@da/bcrypt@*": "1.0.1",
     "jsr:@da/bcrypt@^1.0.1": "1.0.1",
     "jsr:@std/assert@^1.0.13": "1.0.13",
-    "jsr:@std/async@^1.0.13": "1.0.13",
     "jsr:@std/cli@^1.0.19": "1.0.21",
     "jsr:@std/cli@^1.0.20": "1.0.21",
     "jsr:@std/cli@^1.0.21": "1.0.21",
@@ -24,7 +23,8 @@
     "jsr:@std/net@^1.0.4": "1.0.4",
     "jsr:@std/path@^1.1.0": "1.1.1",
     "jsr:@std/path@^1.1.1": "1.1.1",
-    "jsr:@std/streams@^1.0.10": "1.0.10"
+    "jsr:@std/streams@^1.0.10": "1.0.10",
+    "npm:@types/node@*": "22.15.15"
   },
   "jsr": {
     "@andyburke/fsdb@1.0.2": {
@@ -41,11 +41,9 @@
         "jsr:@std/cli@^1.0.19"
       ]
     },
-    "@andyburke/serverus@0.9.8": {
+    "@andyburke/serverus@0.12.2": {
-      "integrity": "6d806a5fd50b67edfaeee12f1a59bfc3a0cb22d9bcffba0910bf5b56d0473059",
+      "integrity": "17cf6d7cb58857c4bc34ee96aa718c05edf0fd4fe159afc5890253e50bd99c3a",
       "dependencies": [
-        "jsr:@andyburke/serverus",
-        "jsr:@std/async",
         "jsr:@std/cli@^1.0.21",
         "jsr:@std/fmt@^1.0.6",
         "jsr:@std/fs@^1.0.19",
@@ -63,9 +61,6 @@
         "jsr:@std/internal@^1.0.6"
       ]
     },
-    "@std/async@1.0.13": {
-      "integrity": "1d76ca5d324aef249908f7f7fe0d39aaf53198e5420604a59ab5c035adc97c96"
-    },
     "@std/cli@1.0.21": {
       "integrity": "cd25b050bdf6282e321854e3822bee624f07aca7636a3a76d95f77a3a919ca2a"
     },
@@ -118,6 +113,17 @@
       "integrity": "75c0b1431873cd0d8b3d679015220204d36d3c7420d93b60acfc379eb0dc30af"
     }
   },
+  "npm": {
+    "@types/node@22.15.15": {
+      "integrity": "sha512-R5muMcZob3/Jjchn5LcO8jdKwSCbzqmPB6ruBxMcf9kbxtniZHP327s6C37iOfuw8mbKK3cAQa7sEl7afLrQ8A==",
+      "dependencies": [
+        "undici-types"
+      ]
+    },
+    "undici-types@6.21.0": {
+      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="
+    }
+  },
   "redirects": {
     "https://jsr.io/@da/bcrypt/1.0.1/src/worker.ts": "https://jsr.io/@da/bcrypt/1.0.1/"
   },
@@ -125,10 +131,11 @@
     "dependencies": [
       "jsr:@andyburke/fsdb@^1.0.2",
       "jsr:@andyburke/lurid@0.2",
-      "jsr:@andyburke/serverus@~0.9.8",
+      "jsr:@andyburke/serverus@~0.12.2",
       "jsr:@da/bcrypt@^1.0.1",
       "jsr:@std/assert@^1.0.13",
       "jsr:@std/encoding@^1.0.10",
+      "jsr:@std/fs@^1.0.19",
       "jsr:@std/http@^1.0.20",
       "jsr:@std/path@^1.1.1"
     ]

public/_pre.ts

@@ -0,0 +1,26 @@
+import { PRECHECKS } from '@andyburke/serverus/handlers/static';
+import { get_session, get_user, require_user } from '../utils/prechecks.ts';
+import * as CANNED_RESPONSES from '../utils/canned_responses.ts';
+
+export function load() {
+	PRECHECKS.PUT = [
+		get_session,
+		get_user,
+		require_user,
+
+		(request: Request, meta: Record<string, any>): Response | undefined => {
+			const can_write_own_files = meta.user?.permissions.includes('files.write.own');
+			const can_write_all_files = meta.user?.permissions.includes('files.write.all');
+
+			const path = new URL(request.url).pathname;
+
+			const is_to_files = path.toLowerCase().startsWith('/files/');
+			const is_to_home_dir = meta.user?.id && path.toLowerCase().startsWith(`/files/users/${meta.user.id}/`);
+
+			const has_permission = is_to_files && (can_write_all_files || (can_write_own_files && is_to_home_dir));
+			if (!has_permission) {
+				return CANNED_RESPONSES.permission_denied();
+			}
+		}
+	];
+}

public/api/users/index.ts

@@ -10,6 +10,7 @@ import * as bcrypt from '@da/bcrypt';
 
 // TODO: figure out a better solution for doling out permissions
 const DEFAULT_USER_PERMISSIONS: string[] = [
+	'files.write.own',
 	'self.read',
 	'self.write',
 	'rooms.read',

tests/01_file_uploads.test.ts

@@ -0,0 +1,309 @@
+import { api, API_CLIENT } from '../utils/api.ts';
+import * as asserts from '@std/assert';
+import { USER } from '../models/user.ts';
+import { EPHEMERAL_SERVER, get_ephemeral_listen_server, random_username, set_user_permissions } from './helpers.ts';
+import { Cookie, getSetCookies } from '@std/http/cookie';
+import { generateTotp } from '../utils/totp.ts';
+import * as fs from '@std/fs';
+import * as path from '@std/path';
+import { ensureFile } from '@std/fs';
+
+Deno.test({
+	name: 'file uploading (home directory)',
+	permissions: {
+		env: true,
+		read: true,
+		write: true,
+		net: true
+	},
+
+	sanitizeResources: false,
+	sanitizeOps: false,
+
+	fn: async () => {
+		let test_server_info: EPHEMERAL_SERVER | null = null;
+		try {
+			test_server_info = await get_ephemeral_listen_server();
+			const client: API_CLIENT = api({
+				prefix: '/api',
+				hostname: test_server_info.hostname,
+				port: test_server_info.port
+			});
+
+			const username = random_username();
+			const password = 'password';
+
+			const user_creation_response: Record<string, any> = await client.fetch('/users', {
+				method: 'POST',
+				json: {
+					username,
+					password
+				}
+			});
+
+			asserts.assert(user_creation_response?.user);
+			asserts.assert(user_creation_response?.session);
+
+			let cookies: Cookie[] = [];
+			const auth_response: any = await client.fetch('/auth', {
+				method: 'POST',
+				json: {
+					username,
+					password: 'password'
+				},
+				done: (response) => {
+					cookies = getSetCookies(response.headers);
+				}
+			});
+
+			const user: USER | undefined = auth_response.user;
+			asserts.assert(user);
+			asserts.assert(user.id);
+
+			const session: Record<string, any> | undefined = auth_response.session;
+			asserts.assert(session);
+
+			cookies.push({
+				name: 'totp',
+				value: await generateTotp(session?.secret ?? ''),
+				maxAge: 30,
+				expires: Date.now() + 30_000,
+				path: '/'
+			});
+
+			const headers_for_upload_request = new Headers();
+			for (const cookie of cookies) {
+				headers_for_upload_request.append(`x-${cookie.name}`, cookie.value);
+			}
+			headers_for_upload_request.append(
+				'cookie',
+				cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ')
+			);
+
+			const upload_body = new FormData();
+			upload_body.append(
+				'file',
+				new File(['this is the test content'], 'test_uploading_to_home_dir.txt')
+			);
+			const upload_response = await fetch(
+				`http://${test_server_info.hostname}:${test_server_info.port}/files/users/${user.id}/test_uploading_to_home_dir.txt`,
+				{
+					method: 'PUT',
+					headers: headers_for_upload_request,
+					body: upload_body
+				}
+			);
+
+			asserts.assert(upload_response.ok);
+			const upload_response_body = await upload_response.text();
+			asserts.assert(upload_response_body);
+
+			const get_response = await fetch(
+				`http://${test_server_info.hostname}:${test_server_info.port}/files/users/${user.id}/test_uploading_to_home_dir.txt`
+			);
+
+			asserts.assert(get_response.ok);
+			asserts.assert(get_response.body);
+
+			const local_download_path = path.join(Deno.cwd(), 'files', 'users', user.id, 'test_uploading_to_home_dir.txt-downloaded');
+			await ensureFile(local_download_path);
+
+			const file = await Deno.open(local_download_path, { truncate: true, write: true });
+			await get_response.body.pipeTo(file.writable);
+
+			const download_content = await Deno.readTextFile(local_download_path);
+			asserts.assert(download_content);
+			asserts.assertEquals(download_content, 'this is the test content');
+
+			await Deno.remove(local_download_path);
+			asserts.assert(!fs.existsSync(local_download_path));
+
+			const uploaded_to_homedir_path = path.join(Deno.cwd(), 'files', 'users', user.id, 'test_uploading_to_home_dir.txt');
+			await Deno.remove(uploaded_to_homedir_path);
+			asserts.assert(!fs.existsSync(uploaded_to_homedir_path));
+
+			let dir = path.dirname(uploaded_to_homedir_path);
+			do {
+				if (dir.endsWith('files')) {
+					break;
+				}
+
+				const files = Deno.readDirSync(dir);
+				let has_files = false;
+				for (const _ of files) {
+					has_files = true;
+					break;
+				}
+
+				if (has_files) {
+					dir = '';
+					break;
+				}
+
+				await Deno.remove(dir);
+				dir = path.dirname(dir);
+			} while (dir.length);
+		} finally {
+			if (test_server_info) {
+				await test_server_info?.server?.stop();
+			}
+		}
+	}
+});
+
+Deno.test({
+	name: 'file uploading (outside home directory)',
+	permissions: {
+		env: true,
+		read: true,
+		write: true,
+		net: true
+	},
+
+	sanitizeResources: false,
+	sanitizeOps: false,
+
+	fn: async () => {
+		let test_server_info: EPHEMERAL_SERVER | null = null;
+		try {
+			test_server_info = await get_ephemeral_listen_server();
+			const client: API_CLIENT = api({
+				prefix: '/api',
+				hostname: test_server_info.hostname,
+				port: test_server_info.port
+			});
+
+			const username = random_username();
+			const password = 'password';
+
+			const user_creation_response: Record<string, any> = await client.fetch('/users', {
+				method: 'POST',
+				json: {
+					username,
+					password
+				}
+			});
+
+			asserts.assert(user_creation_response?.user);
+			asserts.assert(user_creation_response?.session);
+
+			let cookies: Cookie[] = [];
+			const auth_response: any = await client.fetch('/auth', {
+				method: 'POST',
+				json: {
+					username,
+					password: 'password'
+				},
+				done: (response) => {
+					cookies = getSetCookies(response.headers);
+				}
+			});
+
+			const user: USER | undefined = auth_response.user;
+			asserts.assert(user);
+			asserts.assert(user.id);
+
+			const session: Record<string, any> | undefined = auth_response.session;
+			asserts.assert(session);
+
+			cookies.push({
+				name: 'totp',
+				value: await generateTotp(session?.secret ?? ''),
+				maxAge: 30,
+				expires: Date.now() + 30_000,
+				path: '/'
+			});
+
+			const headers_for_upload_request = new Headers();
+			for (const cookie of cookies) {
+				headers_for_upload_request.append(`x-${cookie.name}`, cookie.value);
+			}
+			headers_for_upload_request.append(
+				'cookie',
+				cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ')
+			);
+
+			const upload_body = new FormData();
+			upload_body.append(
+				'file',
+				new File(['this is the root dir test content'], 'test_uploading_to_root_dir.txt')
+			);
+
+			const disallowed_upload_response = await fetch(
+				`http://${test_server_info.hostname}:${test_server_info.port}/files/test_uploading_to_root_dir.txt`,
+				{
+					method: 'PUT',
+					headers: headers_for_upload_request,
+					body: upload_body
+				}
+			);
+
+			asserts.assert(!disallowed_upload_response.ok);
+			await disallowed_upload_response.text();
+
+			await set_user_permissions(client, user, session, [...user.permissions, 'files.write.all']);
+
+			const allowed_upload_response = await fetch(
+				`http://${test_server_info.hostname}:${test_server_info.port}/files/test_uploading_to_root_dir.txt`,
+				{
+					method: 'PUT',
+					headers: headers_for_upload_request,
+					body: upload_body
+				}
+			);
+
+			asserts.assert(allowed_upload_response.ok);
+			await allowed_upload_response.text();
+
+			const get_response = await fetch(
+				`http://${test_server_info.hostname}:${test_server_info.port}/files/test_uploading_to_root_dir.txt`
+			);
+
+			asserts.assert(get_response.ok);
+			asserts.assert(get_response.body);
+
+			const local_download_path = path.join(Deno.cwd(), 'files', 'test_uploading_to_root_dir.txt-downloaded');
+			await ensureFile(local_download_path);
+
+			const file = await Deno.open(local_download_path, { truncate: true, write: true });
+			await get_response.body.pipeTo(file.writable);
+
+			const download_content = await Deno.readTextFile(local_download_path);
+			asserts.assert(download_content);
+			asserts.assertEquals(download_content, 'this is the root dir test content');
+
+			await Deno.remove(local_download_path);
+			asserts.assert(!fs.existsSync(local_download_path));
+
+			const uploaded_to_root_dir_path = path.join(Deno.cwd(), 'files', 'test_uploading_to_root_dir.txt');
+			await Deno.remove(uploaded_to_root_dir_path);
+			asserts.assert(!fs.existsSync(uploaded_to_root_dir_path));
+
+			let dir = path.dirname(uploaded_to_root_dir_path);
+			do {
+				if (dir.endsWith('files')) {
+					break;
+				}
+
+				const files = Deno.readDirSync(dir);
+				let has_files = false;
+				for (const _ of files) {
+					has_files = true;
+					break;
+				}
+
+				if (has_files) {
+					dir = '';
+					break;
+				}
+
+				await Deno.remove(dir);
+				dir = path.dirname(dir);
+			} while (dir.length);
+		} finally {
+			if (test_server_info) {
+				await test_server_info?.server?.stop();
+			}
+		}
+	}
+});

utils/canned_responses.ts

@@ -1,3 +1,18 @@
+export function unknown(error?: Error): Response {
+	if (Deno.env.get('TRACE_ERROR_RESPONSES')) {
+		console.trace(error ?? 'unknown error');
+	}
+	return Response.json({
+		error: {
+			message: error?.message ?? 'Unknown Error',
+			cause: error?.cause ?? 'unknown_error',
+			stack: Deno.env.get('TRACE_ERROR_RESPONSES') ? error?.stack ?? '' : null
+		}
+	}, {
+		status: 500
+	});
+}
+
 export function not_found(): Response {
 	if (Deno.env.get('TRACE_ERROR_RESPONSES')) {
 		console.trace('not_found');
@@ -12,6 +27,20 @@ export function not_found(): Response {
 	});
 }
 
+export function conflict(): Response {
+	if (Deno.env.get('TRACE_ERROR_RESPONSES')) {
+		console.trace('conflict');
+	}
+	return Response.json({
+		error: {
+			message: 'Conflict - this resource already exists.',
+			cause: 'conflict'
+		}
+	}, {
+		status: 409
+	});
+}
+
 export function permission_denied(): Response {
 	if (Deno.env.get('TRACE_ERROR_RESPONSES')) {
 		console.trace('permission_denied');