import * as fs from '@std/fs'; import * as path from '@std/path'; import by_lurid from './organizers/by_lurid.ts'; import { Optional } from './utils/optional.ts'; import { walk, WALK_ENTRY } from './utils/walk.ts'; export type { WALK_ENTRY }; export type FSDB_COLLECTION_CONFIG = { name: string; id_field: string; indexers?: Record>; organize: (id: string) => string[]; root: string; }; export type FSDB_COLLECTION_CONFIG_INPUT = Optional; export type FSDB_SEARCH_OPTIONS = { limit?: number; offset?: number; filter?: (entry: WALK_ENTRY) => boolean; sort?: (a: WALK_ENTRY, b: WALK_ENTRY) => number; }; export interface FSDB_INDEXER { set_fsdb_root(root: string): void; index(item: T, authoritative_path: string): Promise; remove(item: T, authoritative_path: string): Promise; lookup(value: string, options?: FSDB_SEARCH_OPTIONS): Promise; } /** Represents a collection of like items within the database on disk. */ export class FSDB_COLLECTION> { private config: FSDB_COLLECTION_CONFIG; public INDEX: Record>; private event_listeners: Record; constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) { this.config = { ...{ id_field: 'id', organize: by_lurid, root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}` }, ...(input_config ?? {}) }; this.event_listeners = {}; this.INDEX = this.config.indexers ?? {}; for (const indexer of Object.values(this.INDEX)) { indexer.set_fsdb_root(this.config.root); } let existing_collection_info: any = undefined; try { const existing_collection_info_content: string = Deno.readTextFileSync( path.resolve(path.join(this.config.root), '.fsdb.collection.json') ); existing_collection_info = JSON.parse(existing_collection_info_content); } catch (error) { if (!(error instanceof Deno.errors.NotFound)) { throw error; } } if (existing_collection_info) { if (this.config.name !== existing_collection_info.name) { console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.'); } if (this.config.root !== existing_collection_info.root) { console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.'); } if (this.config.id_field !== existing_collection_info.id_field) { console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.'); } if ( Object.keys(this.config.indexers ?? {}).sort().join('|') !== Object.keys(existing_collection_info.indexers ?? {}).sort().join('|') ) { console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.'); } } const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json')); const collection_info_json: string = JSON.stringify(this.config, null, '\t'); Deno.mkdirSync(path.dirname(collection_info_file_path), { recursive: true }); Deno.writeTextFileSync(collection_info_file_path, collection_info_json); } /** Get the "organized" path for the given item within the database. */ public get_organized_item_path(item: any, id_field?: string): string { const id: string = item[id_field ?? this.config.id_field]; const path_elements: string[] = this.config.organize(id); const resolved_item_path = path.resolve(path.join(this.config.root, ...path_elements)); return resolved_item_path; } /** Get the "organized" path for the given id within the database. */ public get_organized_id_path(id: string): string { return this.get_organized_item_path({ id }, 'id'); } private async ensure_item_path(item: any, id_field?: string): Promise { const organized_item_path: string = this.get_organized_item_path(item, id_field); const organized_item_dir: string = path.dirname(organized_item_path); await fs.ensureDir(organized_item_dir); return organized_item_path; } private async write_item(item: T, override_path?: string): Promise { const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field); Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t')); this.emit('write', { item, item_path }); if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).index(item, item_path); this.emit('index', { item, item_path, indexer }); } } } /** Get an item from the collection given its id. */ async get(id: string): Promise { const item_path: string = this.get_organized_id_path(id); const item_exists: boolean = await fs.exists(item_path); if (!item_exists) { return null; } const content: string = await Deno.readTextFile(item_path); const item: T = JSON.parse(content); this.emit('get', { item, item_path }); return item; } /** Create an item in the collection. */ async create(item: T): Promise { const item_path: string = this.get_organized_item_path(item); const item_exists: boolean = await fs.exists(item_path); if (item_exists) { throw new Error('item already exists', { cause: 'item_exists' }); } await this.write_item(item); this.emit('create', { item, item_path }); return item; } /** Update the given item in the collection, requiring the id to be stable. */ async update(item: T): Promise { const item_path: string = this.get_organized_item_path(item); const id: string = item[this.config.id_field]; const previous: T | null = await this.get(id); if (!previous) { throw new Error('item does not exist', { cause: 'item_does_not_exist' }); } await this.write_item(item, item_path); this.emit('update', { item, previous, item_path }); return item; } /** Delete the given item from the collection. */ async delete(item: T): Promise { const item_path = this.get_organized_item_path(item); const item_exists = await fs.exists(item_path); if (!item_exists) { return null; } if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).remove(item, item_path); } } await Deno.remove(item_path); let dir = path.dirname(item_path); do { const files = Deno.readDirSync(dir); let has_files = false; for (const _ of files) { has_files = true; break; } if (has_files) { dir = ''; break; } await Deno.remove(dir); dir = path.dirname(dir); } while (dir.length); this.emit('delete', { item }); return item; } /** Iterate through the items. */ async all({ limit = 100, offset = 0, filter = undefined, sort = undefined }: FSDB_SEARCH_OPTIONS = {}): Promise[]> { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin'); const results: WALK_ENTRY[] = []; let counter = 0; // TODO: better way to get a pattern to match files in this collection? const root_stat = await Deno.lstat(this.config.root); if (!root_stat.isDirectory) { console.warn(`missing root directory for fsdb collection:`); console.dir(this.config); return results; } for await ( const entry of walk(this.config.root, { filter: (entry: WALK_ENTRY): boolean => { const extension = path.extname(entry.path); if (extension.toLowerCase() !== '.json') { return false; } if (entry.info.isDirectory || entry.info.isSymlink) { return false; } const filename = path.basename(entry.path); if (filename === '.fsdb.collection.json') { return false; } return filter ? filter(entry) : true; }, sort }) ) { if (counter < offset) { ++counter; continue; } results.push(entry); ++counter; if (counter >= (offset + limit)) { break; } } if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')); this.emit('all', { options: { limit, offset, filter, sort }, results }); return results; } /** Use indexes to search for matching items. */ async find(criteria: Record, input_options?: FSDB_SEARCH_OPTIONS): Promise[]> { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin'); const options: FSDB_SEARCH_OPTIONS = { ...{ limit: 100, offset: 0 }, ...(input_options ?? {}) }; const results: WALK_ENTRY[] = []; const item_paths: string[] = []; for (const search_key of Object.keys(criteria)) { const indexer_for_search_key: FSDB_INDEXER | undefined = this.INDEX[search_key]; const value: string = criteria[search_key]; if (indexer_for_search_key) { item_paths.push(...await indexer_for_search_key.lookup(value, input_options)); } } const limit = options?.limit ?? 100; const offset = options?.offset ?? 0; let counter = 0; for await (const item_path of item_paths) { if (counter < offset) { ++counter; continue; } const info: Deno.FileInfo = await Deno.lstat(item_path); results.push({ path: item_path, info, depth: -1, load: function () { return JSON.parse(Deno.readTextFileSync(this.path)) as T; } }); ++counter; if (counter >= (offset + limit)) { break; } } if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end')); this.emit('find', { criteria, options, results }); return results; } public on(event: string, handler: (event_data: any) => void) { const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; if (!listeners.includes(handler)) { listeners.push(handler); } if (Deno.env.get('FSDB_LOG_EVENTS')) { console.dir({ on: { event, handler }, listeners }); } } public off(event: string, handler: (event_data: any) => void) { const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; if (listeners.includes(handler)) { listeners.splice(listeners.indexOf(handler), 1); } if (Deno.env.get('FSDB_LOG_EVENTS')) { console.dir({ off: { event: event, handler }, listeners }); } } private emit(event_name: string, event_data: any) { const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? []; if (Deno.env.get('FSDB_LOG_EVENTS')) { console.dir({ emitting: { event_name, event_data, listeners } }); } for (const listener of listeners) { listener(event_data); } } public sorts = { newest: (a: WALK_ENTRY, b: WALK_ENTRY): number => ((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare( (a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '' ), oldest: (a: WALK_ENTRY, b: WALK_ENTRY): number => ((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare( (b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '' ), latest: (a: WALK_ENTRY, b: WALK_ENTRY): number => ((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare((a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''), stalest: (a: WALK_ENTRY, b: WALK_ENTRY): number => ((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '') }; }