import * as fs from '@std/fs'; import * as path from '@std/path'; import by_lurid from './organizers/by_lurid.ts'; import { Optional } from './utils/optional.ts'; export type FSDB_COLLECTION_CONFIG = { name: string; id_field: string; indexers?: Record>; organize: (id: string) => string[]; root: string; }; export type FSDB_COLLECTION_CONFIG_INPUT = Optional; export type FSDB_SEARCH_OPTIONS = { limit: number; offset?: number; before?: string; after?: string; modified_before?: string; modified_after?: string; id_before?: string; id_after?: string; }; export interface FSDB_INDEXER { set_fsdb_root(root: string): void; index(item: T, authoritative_path: string): Promise; remove(item: T, authoritative_path: string): Promise; lookup(value: string, options?: FSDB_SEARCH_OPTIONS): Promise; } /** Represents a collection of like items within the database on disk. */ export class FSDB_COLLECTION> { private config: FSDB_COLLECTION_CONFIG; public INDEX: Record>; constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) { this.config = { ...{ id_field: 'id', organize: by_lurid, root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}` }, ...(input_config ?? {}) }; this.INDEX = this.config.indexers ?? {}; for (const indexer of Object.values(this.INDEX)) { indexer.set_fsdb_root(this.config.root); } let existing_collection_info: any = undefined; try { const existing_collection_info_content: string = Deno.readTextFileSync( path.resolve(path.join(this.config.root), '.fsdb.collection.json') ); existing_collection_info = JSON.parse(existing_collection_info_content); } catch (error) { if (!(error instanceof Deno.errors.NotFound)) { throw error; } } if (existing_collection_info) { if (this.config.name !== existing_collection_info.name) { console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.'); } if (this.config.root !== existing_collection_info.root) { console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.'); } if (this.config.id_field !== existing_collection_info.id_field) { console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.'); } if ( Object.keys(this.config.indexers ?? {}).sort().join('|') !== Object.keys(existing_collection_info.indexers ?? {}).sort().join('|') ) { console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.'); } } const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json')); const collection_info_json: string = JSON.stringify(this.config, null, 4); Deno.mkdirSync(path.dirname(collection_info_file_path), { recursive: true }); Deno.writeTextFileSync(collection_info_file_path, collection_info_json); } /** Get the "organized" path for the given item within the database. */ public get_organized_item_path(item: any, id_field?: string): string { const id: string = item[id_field ?? this.config.id_field]; const path_elements: string[] = this.config.organize(id); const resolved_item_path = path.resolve(path.join(this.config.root, ...path_elements)); return resolved_item_path; } /** Get the "organized" path for the given id within the database. */ public get_organized_id_path(id: string): string { return this.get_organized_item_path({ id }, 'id'); } private async ensure_item_path(item: any, id_field?: string): Promise { const organized_item_path: string = this.get_organized_item_path(item, id_field); const organized_item_dir: string = path.dirname(organized_item_path); await fs.ensureDir(organized_item_dir); return organized_item_path; } private async write_item(item: T, override_path?: string): Promise { const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field); Deno.writeTextFileSync(item_path, JSON.stringify(item, null, 1)); if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).index(item, item_path); } } } /** Get an item from the collection given its id. */ async get(id: string): Promise { const id_path: string = this.get_organized_id_path(id); const item_exists: boolean = await fs.exists(id_path); if (!item_exists) { return null; } const content: string = await Deno.readTextFile(id_path); return JSON.parse(content); } /** Create an item in the collection. */ async create(item: T): Promise { const item_path: string = this.get_organized_item_path(item); const item_exists: boolean = await fs.exists(item_path); if (item_exists) { throw new Error('item already exists', { cause: 'item_exists' }); } await this.write_item(item); return item; } /** Update the given item in the collection, requiring the id to be stable. */ async update(item: T): Promise { const item_path: string = this.get_organized_item_path(item); const item_exists: boolean = await fs.exists(item_path); if (!item_exists) { throw new Error('item does not exist', { cause: 'item_does_not_exist' }); } await this.write_item(item, item_path); return item; } /** Delete the given item from the collection. */ async delete(item: T): Promise { const item_path = this.get_organized_item_path(item); const item_exists = await fs.exists(item_path); if (!item_exists) { return null; } if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).remove(item, item_path); } } await Deno.remove(item_path); let dir = path.dirname(item_path); do { const files = Deno.readDirSync(dir); let has_files = false; for (const _ of files) { has_files = true; break; } if (has_files) { dir = ''; break; } await Deno.remove(dir); dir = path.dirname(dir); } while (dir.length); return item; } /** Iterate through the items. */ async all(input_options?: FSDB_SEARCH_OPTIONS): Promise { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin'); const options: FSDB_SEARCH_OPTIONS = { ...{ limit: 100, offset: 0 }, ...(input_options ?? {}) }; const results: T[] = []; const limit = options?.limit ?? 100; const offset = options?.offset ?? 0; let counter = 0; // TODO: better way to get a pattern to match files in this collection? for await ( const entry of fs.walk(this.config.root, { includeDirs: false, includeSymlinks: false, skip: [/\.fsdb\.collection\.json$/], exts: ['json'] }) ) { let item_stat = null; if (options.before) { item_stat = item_stat ?? await Deno.lstat(entry.path); const birthtime = (item_stat.birthtime ?? new Date(0)).toISOString(); if (birthtime > options.before) { continue; } } if (options.after) { item_stat = item_stat ?? await Deno.lstat(entry.path); if ((item_stat.birthtime ?? new Date(0)).toISOString() < options.after) { continue; } } if (options.modified_before) { item_stat = item_stat ?? await Deno.lstat(entry.path); if ((item_stat.mtime ?? new Date(0)).toISOString() > options.modified_before) { continue; } } if (options.modified_after) { item_stat = item_stat ?? await Deno.lstat(entry.path); if ((item_stat.mtime ?? new Date(0)).toISOString() < options.modified_after) { continue; } } let item_id = null; if (options.id_before) { item_id = item_id ?? entry.name.replace(/\.json$/, ''); if (item_id >= options.id_before) { continue; } } if (options.id_after) { item_id = item_id ?? entry.name.replace(/\.json$/, ''); if (item_id <= options.id_after) { continue; } } if (counter < offset) { ++counter; continue; } const content = await Deno.readTextFile(entry.path); results.push(JSON.parse(content)); ++counter; if (counter >= (offset + limit)) { break; } } if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')); return results; } /** Use indexes to search for matching items. */ async find(criteria: Record, input_options?: FSDB_SEARCH_OPTIONS): Promise { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin'); const options: FSDB_SEARCH_OPTIONS = { ...{ limit: 100, offset: 0 }, ...(input_options ?? {}) }; const results: T[] = []; const item_paths: string[] = []; for (const search_key of Object.keys(criteria)) { const indexer_for_search_key: FSDB_INDEXER | undefined = this.INDEX[search_key]; const value: string = criteria[search_key]; if (indexer_for_search_key) { item_paths.push(...await indexer_for_search_key.lookup(value, input_options)); } } const limit = options?.limit ?? 100; const offset = options?.offset ?? 0; let counter = 0; for await (const item_path of item_paths) { if (counter < offset) { ++counter; continue; } const content = await Deno.readTextFile(item_path); results.push(JSON.parse(content)); ++counter; if (counter >= (offset + limit)) { break; } } if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end')); return results; } }