fsdb/fsdb.ts

import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';

export type FSDB_COLLECTION_CONFIG = {
	name: string;
	id_field: string;
	indexers?: Record<string, FSDB_INDEXER<any>>;
	organize: (id: string) => string[];
	root: string;
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<FSDB_COLLECTION_CONFIG, 'id_field' | 'organize' | 'root'>;

export type FSDB_SEARCH_OPTIONS = {
	limit: number;
	offset?: number;
	before?: string;
	after?: string;
	modified_before?: string;
	modified_after?: string;
	id_before?: string;
	id_after?: string;
};

export interface FSDB_INDEXER<T> {
	set_fsdb_root(root: string): void;
	index(item: T, authoritative_path: string): Promise<string[]>;
	remove(item: T, authoritative_path: string): Promise<string[]>;
	lookup(value: string, options?: FSDB_SEARCH_OPTIONS): Promise<string[]>;
}

/** Represents a collection of like items within the database on disk. */
export class FSDB_COLLECTION<T extends Record<string, any>> {
	private config: FSDB_COLLECTION_CONFIG;
	public INDEX: Record<string, FSDB_INDEXER<any>>;
	private event_listeners: Record<string, []>;

	constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
		this.config = {
			...{
				id_field: 'id',
				organize: by_lurid,
				root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
			},
			...(input_config ?? {})
		};

		this.event_listeners = {};

		this.INDEX = this.config.indexers ?? {};
		for (const indexer of Object.values(this.INDEX)) {
			indexer.set_fsdb_root(this.config.root);
		}

		let existing_collection_info: any = undefined;
		try {
			const existing_collection_info_content: string = Deno.readTextFileSync(
				path.resolve(path.join(this.config.root), '.fsdb.collection.json')
			);
			existing_collection_info = JSON.parse(existing_collection_info_content);
		} catch (error) {
			if (!(error instanceof Deno.errors.NotFound)) {
				throw error;
			}
		}

		if (existing_collection_info) {
			if (this.config.name !== existing_collection_info.name) {
				console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.');
			}

			if (this.config.root !== existing_collection_info.root) {
				console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.');
			}

			if (this.config.id_field !== existing_collection_info.id_field) {
				console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.');
			}

			if (
				Object.keys(this.config.indexers ?? {}).sort().join('|') !==
					Object.keys(existing_collection_info.indexers ?? {}).sort().join('|')
			) {
				console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.');
			}
		}

		const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json'));
		const collection_info_json: string = JSON.stringify(this.config, null, '\t');
		Deno.mkdirSync(path.dirname(collection_info_file_path), {
			recursive: true
		});
		Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
	}

	/** Get the "organized" path for the given item within the database. */
	public get_organized_item_path(item: any, id_field?: string): string {
		const id: string = item[id_field ?? this.config.id_field];
		const path_elements: string[] = this.config.organize(id);
		const resolved_item_path = path.resolve(path.join(this.config.root, ...path_elements));
		return resolved_item_path;
	}

	/** Get the "organized" path for the given id within the database. */
	public get_organized_id_path(id: string): string {
		return this.get_organized_item_path({ id }, 'id');
	}

	private async ensure_item_path(item: any, id_field?: string): Promise<string> {
		const organized_item_path: string = this.get_organized_item_path(item, id_field);
		const organized_item_dir: string = path.dirname(organized_item_path);
		await fs.ensureDir(organized_item_dir);

		return organized_item_path;
	}

	private async write_item(item: T, override_path?: string): Promise<void> {
		const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field);
		Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));

		this.emit('write', {
			item,
			item_path
		});

		if (this.config.indexers) {
			for (const indexer of Object.values(this.config.indexers)) {
				await (indexer as FSDB_INDEXER<T>).index(item, item_path);
				this.emit('index', {
					item,
					item_path,
					indexer
				});
			}
		}
	}

	/** Get an item from the collection given its id. */
	async get(id: string): Promise<T | null> {
		const item_path: string = this.get_organized_id_path(id);
		const item_exists: boolean = await fs.exists(item_path);

		if (!item_exists) {
			return null;
		}

		const content: string = await Deno.readTextFile(item_path);
		const item: T = JSON.parse(content);

		this.emit('get', {
			item,
			item_path
		});

		return item;
	}

	/** Create an item in the collection. */
	async create(item: T): Promise<T> {
		const item_path: string = this.get_organized_item_path(item);
		const item_exists: boolean = await fs.exists(item_path);

		if (item_exists) {
			throw new Error('item already exists', {
				cause: 'item_exists'
			});
		}

		await this.write_item(item);

		this.emit('create', {
			item,
			item_path
		});

		return item;
	}

	/** Update the given item in the collection, requiring the id to be stable. */
	async update(item: T): Promise<T> {
		const item_path: string = this.get_organized_item_path(item);
		const id: string = item[this.config.id_field];
		const previous: T | null = await this.get(id);

		if (!previous) {
			throw new Error('item does not exist', {
				cause: 'item_does_not_exist'
			});
		}

		await this.write_item(item, item_path);

		this.emit('update', {
			item,
			previous,
			item_path
		});

		return item;
	}

	/** Delete the given item from the collection. */
	async delete(item: T): Promise<T | null> {
		const item_path = this.get_organized_item_path(item);
		const item_exists = await fs.exists(item_path);

		if (!item_exists) {
			return null;
		}

		if (this.config.indexers) {
			for (const indexer of Object.values(this.config.indexers)) {
				await (indexer as FSDB_INDEXER<any>).remove(item, item_path);
			}
		}

		await Deno.remove(item_path);

		let dir = path.dirname(item_path);
		do {
			const files = Deno.readDirSync(dir);
			let has_files = false;
			for (const _ of files) {
				has_files = true;
				break;
			}

			if (has_files) {
				dir = '';
				break;
			}

			await Deno.remove(dir);
			dir = path.dirname(dir);
		} while (dir.length);

		this.emit('delete', {
			item
		});

		return item;
	}

	/** Iterate through the items. */
	async all(input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');

		const options: FSDB_SEARCH_OPTIONS = {
			...{
				limit: 100,
				offset: 0
			},
			...(input_options ?? {})
		};

		const results: T[] = [];

		const limit = options?.limit ?? 100;
		const offset = options?.offset ?? 0;
		let counter = 0;

		// TODO: better way to get a pattern to match files in this collection?
		const root_stat = await Deno.lstat(this.config.root);

		if (!root_stat.isDirectory) {
			console.warn(`missing root directory for fsdb collection:`);
			console.dir(this.config);
			return results;
		}

		for await (
			const entry of fs.walk(this.config.root, {
				includeDirs: false,
				includeSymlinks: false,
				skip: [/\.fsdb\.collection\.json$/],
				exts: ['json']
			})
		) {
			let item_stat = null;
			if (options.before) {
				item_stat = item_stat ?? await Deno.lstat(entry.path);
				const birthtime = (item_stat.birthtime ?? new Date(0)).toISOString();
				if (birthtime > options.before) {
					continue;
				}
			}

			if (options.after) {
				item_stat = item_stat ?? await Deno.lstat(entry.path);
				if ((item_stat.birthtime ?? new Date(0)).toISOString() < options.after) {
					continue;
				}
			}

			if (options.modified_before) {
				item_stat = item_stat ?? await Deno.lstat(entry.path);
				if ((item_stat.mtime ?? new Date(0)).toISOString() > options.modified_before) {
					continue;
				}
			}

			if (options.modified_after) {
				item_stat = item_stat ?? await Deno.lstat(entry.path);
				if ((item_stat.mtime ?? new Date(0)).toISOString() < options.modified_after) {
					continue;
				}
			}

			let item_id = null;
			if (options.id_before) {
				item_id = item_id ?? entry.name.replace(/\.json$/, '');
				if (item_id >= options.id_before) {
					continue;
				}
			}

			if (options.id_after) {
				item_id = item_id ?? entry.name.replace(/\.json$/, '');
				if (item_id <= options.id_after) {
					continue;
				}
			}

			if (counter < offset) {
				++counter;
				continue;
			}

			const content = await Deno.readTextFile(entry.path);
			results.push(JSON.parse(content));
			++counter;

			if (counter >= (offset + limit)) {
				break;
			}
		}

		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
		if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));

		this.emit('all', {
			options,
			results
		});

		return results;
	}

	/** Use indexes to search for matching items. */
	async find(criteria: Record<string, any>, input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');

		const options: FSDB_SEARCH_OPTIONS = {
			...{
				limit: 100,
				offset: 0
			},
			...(input_options ?? {})
		};

		const results: T[] = [];
		const item_paths: string[] = [];

		for (const search_key of Object.keys(criteria)) {
			const indexer_for_search_key: FSDB_INDEXER<T> | undefined = this.INDEX[search_key];
			const value: string = criteria[search_key];
			if (indexer_for_search_key) {
				item_paths.push(...await indexer_for_search_key.lookup(value, input_options));
			}
		}

		const limit = options?.limit ?? 100;
		const offset = options?.offset ?? 0;
		let counter = 0;

		for await (const item_path of item_paths) {
			if (counter < offset) {
				++counter;
				continue;
			}

			const content = await Deno.readTextFile(item_path);
			results.push(JSON.parse(content));
			++counter;

			if (counter >= (offset + limit)) {
				break;
			}
		}

		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
		if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));

		this.emit('find', {
			criteria,
			options,
			results
		});

		return results;
	}

	public on(event: string, handler: (event_data: any) => void) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
		if (!listeners.includes(handler)) {
			listeners.push(handler);
		}

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				on: {
					event,
					handler
				},
				listeners
			});
		}
	}

	public off(event: string, handler: (event_data: any) => void) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
		if (listeners.includes(handler)) {
			listeners.splice(listeners.indexOf(handler), 1);
		}

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				off: {
					event: event,
					handler
				},
				listeners
			});
		}
	}

	private emit(event_name: string, event_data: any) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? [];

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				emitting: {
					event_name,
					event_data,
					listeners
				}
			});
		}

		for (const listener of listeners) {
			listener(event_data);
		}
	}
}