fsdb/fsdb.ts

/**
 * We just write to the disk to reduce complexity.
 * @example
 *
 * ```ts
 * import * as fsdb from '@andyburke/fsdb';
 * import { FSDB_INDEXER_SYMLINKS } from '@andyburke/fsdb/indexers';
 * import { by_character, by_email, by_phone } from '@andyburke/fsdb/organizers';
 *
 *	type ITEM = {
 *		id: string;
 *		email: string;
 *		phone: string;
 *		value: string;
 *	};
 *
 *	const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
 *		name: 'test-03-items',
 *		root: get_data_dir() + '/test-03-items',
 *		indexers: {
 *			email: new FSDB_INDEXER_SYMLINKS<ITEM>({
 *				name: 'email',
 *				field: 'email',
 *				organize: by_email
 *			}),
 *			phone: new FSDB_INDEXER_SYMLINKS<ITEM>({
 *				name: 'phone',
 *				field: 'phone',
 *				organize: by_phone
 *			}),
 *			value: new FSDB_INDEXER_SYMLINKS<ITEM>({
 *				name: 'value',
 *				organize: by_character,
 *				get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3),
 *				to_many: true
 *			})
 *		}
 *	});
 *
 *	const item = {
 *		id: lurid(),
 *		email: random_email_address(),
 *		phone: random_phone_number(),
 *		value: random_sentence()
 *	};
 *
 *	const stored_item: ITEM = await item_collection.create(item);
 *	const fetched_by_email: ITEM = (await item_collection.find({ email: item.email })).map((entry) => entry.load()).shift();
 *	const fetched_by_phone: ITEM = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load()).shift();
 *	const fetched_by_word_in_value: ITEM = (await item_collection.find({ value: word })).map((entry) => entry.load()).shift();
 * ```
 *
 * @module
*/

import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';
import { walk, WALK_ENTRY } from './utils/walk.ts';

export type { WALK_ENTRY };

export type FSDB_COLLECTION_CONFIG = {
	name: string;
	id_field: string;
	indexers?: Record<string, FSDB_INDEXER<any>>;
	organize: (id: string) => string[];
	root: string;
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<FSDB_COLLECTION_CONFIG, 'id_field' | 'organize' | 'root'>;

export type FSDB_SEARCH_OPTIONS<T> = {
	limit?: number;
	offset?: number;
	filter?: (entry: WALK_ENTRY<T>) => boolean;
	sort?: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>) => number;
};

export interface FSDB_INDEXER<T> {
	set_fsdb_root(root: string): void;
	index(item: T, authoritative_path: string): Promise<string[]>;
	remove(item: T, authoritative_path: string): Promise<string[]>;
	lookup(value: string, options?: FSDB_SEARCH_OPTIONS<T>): Promise<string[]>;
}

/**
 * Represents a collection of like items within the database on disk.
 */
export class FSDB_COLLECTION<T extends Record<string, any>> {
	private config: FSDB_COLLECTION_CONFIG;
	public INDEX: Record<string, FSDB_INDEXER<any>>;
	private event_listeners: Record<string, []>;

	constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
		this.config = {
			...{
				id_field: 'id',
				organize: by_lurid,
				root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
			},
			...(input_config ?? {})
		};

		this.event_listeners = {};

		this.INDEX = this.config.indexers ?? {};
		for (const indexer of Object.values(this.INDEX)) {
			indexer.set_fsdb_root(this.config.root);
		}

		let existing_collection_info: any = undefined;
		try {
			const existing_collection_info_content: string = Deno.readTextFileSync(
				path.resolve(path.join(this.config.root), '.fsdb.collection.json')
			);
			existing_collection_info = JSON.parse(existing_collection_info_content);
		} catch (error) {
			if (!(error instanceof Deno.errors.NotFound)) {
				throw error;
			}
		}

		if (existing_collection_info) {
			if (this.config.name !== existing_collection_info.name) {
				console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.');
			}

			if (this.config.root !== existing_collection_info.root) {
				console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.');
			}

			if (this.config.id_field !== existing_collection_info.id_field) {
				console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.');
			}

			if (
				Object.keys(this.config.indexers ?? {}).sort().join('|') !==
					Object.keys(existing_collection_info.indexers ?? {}).sort().join('|')
			) {
				console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.');
			}
		}

		const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json'));
		const collection_info_json: string = JSON.stringify(this.config, null, '\t');
		Deno.mkdirSync(path.dirname(collection_info_file_path), {
			recursive: true
		});
		Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
	}

	/**
	 * Get the "organized" path for the given item within the database.
	 *
	 * @param	{any}		item		The item to organize.
	 * @param	{string}	[id_field] 	Optional override for id_field.
	 *
	 * @returns	{string}				An organized, resolved item path.
	 */
	public get_organized_item_path(item: any, id_field?: string): string {
		const id: string = item[id_field ?? this.config.id_field];
		const path_elements: string[] = this.config.organize(id);
		const resolved_item_path = path.resolve(path.join(this.config.root, ...path_elements));
		return resolved_item_path;
	}

	/**
	 * Get the "organized" path for the given id within the database.
	 *
	 * @param	{string}		id		The id to organize.
	 *
	 * @returns	{string}				An organized, resolved path for the id.
	 */
	public get_organized_id_path(id: string): string {
		return this.get_organized_item_path({ id }, 'id');
	}

	/**
	 * Ensure the item's directory exists.
	 *
	 * @param	{any}		item		The item to be stored.
	 * @param	{string}	[id_field]	Optional override of the id_field.
	 * @returns	{string}				An organized, resolved path for the item, with the containing directory created.
	 */
	private async ensure_item_path(item: any, id_field?: string): Promise<string> {
		const organized_item_path: string = this.get_organized_item_path(item, id_field);
		const organized_item_dir: string = path.dirname(organized_item_path);
		await fs.ensureDir(organized_item_dir);

		return organized_item_path;
	}

	private async write_item(item: T, override_path?: string): Promise<void> {
		const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field);
		Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));

		this.emit('write', {
			item,
			item_path
		});

		if (this.config.indexers) {
			for (const indexer of Object.values(this.config.indexers)) {
				await (indexer as FSDB_INDEXER<T>).index(item, item_path);
				this.emit('index', {
					item,
					item_path,
					indexer
				});
			}
		}
	}

	/**
	 * Get an item from the collection.
	 *
	 * @param	{string}	id			The item id to look up.
	 *
	 * @returns	{Promise<T | null>}		Returns a promise for either the item or `null` if it cannot be found.
	 */
	async get(id: string): Promise<T | null> {
		const item_path: string = this.get_organized_id_path(id);
		const item_exists: boolean = await fs.exists(item_path);

		if (!item_exists) {
			return null;
		}

		const content: string = await Deno.readTextFile(item_path);
		const item: T = JSON.parse(content);

		this.emit('get', {
			item,
			item_path
		});

		return item;
	}

	/**
	 * Create an item in the collection.
	 *
	 * @param	{T}		item		The item to be stored in the collection.
	 *
	 * @returns	{Promise<T>}		Returns a promise for the item after it has been stored.
	 */
	async create(item: T): Promise<T> {
		const item_path: string = this.get_organized_item_path(item);
		const item_exists: boolean = await fs.exists(item_path);

		if (item_exists) {
			throw new Error('item already exists', {
				cause: 'item_exists'
			});
		}

		await this.write_item(item);

		this.emit('create', {
			item,
			item_path
		});

		return item;
	}

	/**
	 * Update the given item in the collection, requiring the id to be stable.
	 *
	 * @param	{T}				item		The item to be updated in the collection.
	 *
	 * @returns	{Promise<T>}				Returns a promise for the item after it has been updated.
	 */
	async update(item: T): Promise<T> {
		const item_path: string = this.get_organized_item_path(item);
		const id: string = item[this.config.id_field];
		const previous: T | null = await this.get(id);

		if (!previous) {
			throw new Error('item does not exist', {
				cause: 'item_does_not_exist'
			});
		}

		await this.write_item(item, item_path);

		this.emit('update', {
			item,
			previous,
			item_path
		});

		return item;
	}

	/**
	 * Delete the given item from the collection.
	 *
	 * @param	{T}				item		The item to be updated in the collection.
	 *
	 * @returns	{Promise<T>}				Returns a promise for the item after it has been deleted or `null` if the item wasn't in the collection.
	 */
	async delete(item: T): Promise<T | null> {
		const item_path = this.get_organized_item_path(item);
		const item_exists = await fs.exists(item_path);

		if (!item_exists) {
			return null;
		}

		if (this.config.indexers) {
			for (const indexer of Object.values(this.config.indexers)) {
				await (indexer as FSDB_INDEXER<any>).remove(item, item_path);
			}
		}

		await Deno.remove(item_path);

		let dir = path.dirname(item_path);
		do {
			const files = Deno.readDirSync(dir);
			let has_files = false;
			for (const _ of files) {
				has_files = true;
				break;
			}

			if (has_files) {
				dir = '';
				break;
			}

			await Deno.remove(dir);
			dir = path.dirname(dir);
		} while (dir.length);

		this.emit('delete', {
			item
		});

		return item;
	}

	/**
	 * Iterate through all items in the collection.
	 *
	 * @param	{FSDB_SEARCH_OPTIONS}	options		The item to be updated in the collection.
	 *
	 * @returns	{Promise<WALK_ENTRY<T>[]>}			Returns an array of `WALK_ENTRY` items for items given the input options.
	 */
	async all({
		limit = 100,
		offset = 0,
		filter = undefined,
		sort = undefined
	}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');

		const results: WALK_ENTRY<T>[] = [];

		let counter = 0;

		// TODO: better way to get a pattern to match files in this collection?
		const root_stat = await Deno.lstat(this.config.root);

		if (!root_stat.isDirectory) {
			console.warn(`missing root directory for fsdb collection:`);
			console.dir(this.config);
			return results;
		}

		for await (
			const entry of walk(this.config.root, {
				filter: (entry: WALK_ENTRY<T>): boolean => {
					const extension = path.extname(entry.path);
					if (extension.toLowerCase() !== '.json') {
						return false;
					}

					if (entry.info.isDirectory || entry.info.isSymlink) {
						return false;
					}

					const filename = path.basename(entry.path);
					if (filename === '.fsdb.collection.json') {
						return false;
					}

					return filter ? filter(entry) : true;
				},
				sort
			})
		) {
			if (counter < offset) {
				++counter;
				continue;
			}

			results.push(entry);
			++counter;

			if (counter >= (offset + limit)) {
				break;
			}
		}

		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
		if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));

		this.emit('all', {
			options: {
				limit,
				offset,
				filter,
				sort
			},
			results
		});

		return results;
	}

	/**
	 * Use indexes to search for matching items.
	 *
	 * @param	{Record<string, any>}		criteria		The criteria an item should match, usually a field like `{ email: 'someone@somewhere.com` }`
	 * @param	{FSDB_SEARCH_OPTIONS<T>}	options
	 * @returns	{Promise<WALK_ENTRY<T>[]>}			Returns an array of `WALK_ENTRY` items for items given the input options.
	 */
	async find(criteria: Record<string, any>, options?: FSDB_SEARCH_OPTIONS<T>): Promise<WALK_ENTRY<T>[]> {
		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');

		const find_options: FSDB_SEARCH_OPTIONS<T> = {
			...{
				limit: 100,
				offset: 0
			},
			...(options ?? {})
		};

		const results: WALK_ENTRY<T>[] = [];
		const item_paths: string[] = [];

		for (const search_key of Object.keys(criteria)) {
			const indexer_for_search_key: FSDB_INDEXER<T> | undefined = this.INDEX[search_key];
			const value: string = criteria[search_key];
			if (indexer_for_search_key) {
				item_paths.push(...await indexer_for_search_key.lookup(value, find_options));
			}
		}

		const limit = find_options.limit ?? 100;
		const offset = find_options.offset ?? 0;
		let counter = 0;

		for await (const item_path of item_paths) {
			if (counter < offset) {
				++counter;
				continue;
			}

			const info: Deno.FileInfo = await Deno.lstat(item_path);
			results.push({
				path: item_path,
				info,
				depth: -1,
				load: function () {
					return JSON.parse(Deno.readTextFileSync(this.path)) as T;
				}
			});

			++counter;

			if (counter >= (offset + limit)) {
				break;
			}
		}

		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
		if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));

		this.emit('find', {
			criteria,
			options: find_options,
			results
		});

		return results;
	}

	/**
	 * Add an event listener.
	 *
	 * @param	{string}					event		The event to listen for.
	 * @param	{(event_data: any) => void}	handler		The handler for the event.
	 */
	public on(event: string, handler: (event_data: any) => void) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
		if (!listeners.includes(handler)) {
			listeners.push(handler);
		}

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				on: {
					event,
					handler
				},
				listeners
			});
		}
	}

	/**
	 * Remove an event listener.
	 *
	 * @param	{string}					event		The event that was listened to.
	 * @param	{(event_data: any) => void}	handler		The handler that was registered that should be removed.
	 */
	public off(event: string, handler: (event_data: any) => void) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
		if (listeners.includes(handler)) {
			listeners.splice(listeners.indexOf(handler), 1);
		}

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				off: {
					event: event,
					handler
				},
				listeners
			});
		}
	}

	private emit(event_name: string, event_data: any) {
		const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? [];

		if (Deno.env.get('FSDB_LOG_EVENTS')) {
			console.dir({
				emitting: {
					event_name,
					event_data,
					listeners
				}
			});
		}

		for (const listener of listeners) {
			listener(event_data);
		}
	}

	public sorts = {
		newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
			((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
				(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? ''
			),

		oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
			((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
				(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? ''
			),

		latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
			((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare((a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''),

		stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
			((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '')
	};
}