diff --git a/deno.json b/deno.json index c05c8bb..e7b8f2b 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@andyburke/fsdb", - "version": "1.0.4", + "version": "1.1.0", "license": "MIT", "exports": { ".": "./fsdb.ts", diff --git a/fsdb.ts b/fsdb.ts index db70716..8d63e21 100644 --- a/fsdb.ts +++ b/fsdb.ts @@ -53,11 +53,11 @@ * @module */ -import * as fs from '@std/fs'; -import * as path from '@std/path'; -import by_lurid from './organizers/by_lurid.ts'; -import { Optional } from './utils/optional.ts'; -import { walk, WALK_ENTRY } from './utils/walk.ts'; +import * as fs from "@std/fs"; +import * as path from "@std/path"; +import by_lurid from "./organizers/by_lurid.ts"; +import { Optional } from "./utils/optional.ts"; +import { walk, WALK_ENTRY } from "./utils/walk.ts"; export type { WALK_ENTRY }; @@ -68,7 +68,10 @@ export type FSDB_COLLECTION_CONFIG = { organize: (id: string) => string[]; root: string; }; -export type FSDB_COLLECTION_CONFIG_INPUT = Optional; +export type FSDB_COLLECTION_CONFIG_INPUT = Optional< + FSDB_COLLECTION_CONFIG, + "id_field" | "organize" | "root" +>; export type FSDB_SEARCH_OPTIONS = { limit?: number; @@ -95,11 +98,11 @@ export class FSDB_COLLECTION> { constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) { this.config = { ...{ - id_field: 'id', + id_field: "id", organize: by_lurid, - root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}` + root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`, }, - ...(input_config ?? {}) + ...(input_config ?? {}), }; this.event_listeners = {}; @@ -112,7 +115,7 @@ export class FSDB_COLLECTION> { let existing_collection_info: any = undefined; try { const existing_collection_info_content: string = Deno.readTextFileSync( - path.resolve(path.join(this.config.root), '.fsdb.collection.json') + path.resolve(path.join(this.config.root), ".fsdb.collection.json"), ); existing_collection_info = JSON.parse(existing_collection_info_content); } catch (error) { @@ -123,29 +126,43 @@ export class FSDB_COLLECTION> { if (existing_collection_info) { if (this.config.name !== existing_collection_info.name) { - console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.'); + console.warn( + "Mismatching collection name, maybe the collection was renamed? Be cautious.", + ); } if (this.config.root !== existing_collection_info.root) { - console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.'); + console.warn( + "Mismatching collection root, maybe the collection was moved on disk? Be cautious.", + ); } if (this.config.id_field !== existing_collection_info.id_field) { - console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.'); + console.warn( + "Mismatching collection id field, maybe the data format has changed? Be cautious.", + ); } if ( - Object.keys(this.config.indexers ?? {}).sort().join('|') !== - Object.keys(existing_collection_info.indexers ?? {}).sort().join('|') + Object.keys(this.config.indexers ?? {}) + .sort() + .join("|") !== + Object.keys(existing_collection_info.indexers ?? {}) + .sort() + .join("|") ) { - console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.'); + console.warn( + "Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.", + ); } } - const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json')); - const collection_info_json: string = JSON.stringify(this.config, null, '\t'); + const collection_info_file_path: string = path.resolve( + path.join(this.config.root, ".fsdb.collection.json"), + ); + const collection_info_json: string = JSON.stringify(this.config, null, "\t"); Deno.mkdirSync(path.dirname(collection_info_file_path), { - recursive: true + recursive: true, }); Deno.writeTextFileSync(collection_info_file_path, collection_info_json); } @@ -173,7 +190,7 @@ export class FSDB_COLLECTION> { * @returns {string} An organized, resolved path for the id. */ public get_organized_id_path(id: string): string { - return this.get_organized_item_path({ id }, 'id'); + return this.get_organized_item_path({ id }, "id"); } /** @@ -192,21 +209,22 @@ export class FSDB_COLLECTION> { } private async write_item(item: T, override_path?: string): Promise { - const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field); - Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t')); + const item_path: string = + override_path ?? (await this.ensure_item_path(item, this.config.id_field)); + Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t")); - this.emit('write', { + this.emit("write", { item, - item_path + item_path, }); if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).index(item, item_path); - this.emit('index', { + this.emit("index", { item, item_path, - indexer + indexer, }); } } @@ -230,9 +248,9 @@ export class FSDB_COLLECTION> { const content: string = await Deno.readTextFile(item_path); const item: T = JSON.parse(content); - this.emit('get', { + this.emit("get", { item, - item_path + item_path, }); return item; @@ -250,16 +268,16 @@ export class FSDB_COLLECTION> { const item_exists: boolean = await fs.exists(item_path); if (item_exists) { - throw new Error('item already exists', { - cause: 'item_exists' + throw new Error("item already exists", { + cause: "item_exists", }); } await this.write_item(item); - this.emit('create', { + this.emit("create", { item, - item_path + item_path, }); return item; @@ -278,17 +296,17 @@ export class FSDB_COLLECTION> { const previous: T | null = await this.get(id); if (!previous) { - throw new Error('item does not exist', { - cause: 'item_does_not_exist' + throw new Error("item does not exist", { + cause: "item_does_not_exist", }); } await this.write_item(item, item_path); - this.emit('update', { + this.emit("update", { item, previous, - item_path + item_path, }); return item; @@ -327,7 +345,7 @@ export class FSDB_COLLECTION> { } if (has_files) { - dir = ''; + dir = ""; break; } @@ -335,8 +353,8 @@ export class FSDB_COLLECTION> { dir = path.dirname(dir); } while (dir.length); - this.emit('delete', { - item + this.emit("delete", { + item, }); return item; @@ -353,9 +371,9 @@ export class FSDB_COLLECTION> { limit = 100, offset = 0, filter = undefined, - sort = undefined + sort = undefined, }: FSDB_SEARCH_OPTIONS = {}): Promise[]> { - if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin'); + if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin"); const results: WALK_ENTRY[] = []; @@ -370,28 +388,26 @@ export class FSDB_COLLECTION> { return results; } - for await ( - const entry of walk(this.config.root, { - filter: (entry: WALK_ENTRY): boolean => { - const extension = path.extname(entry.path); - if (extension.toLowerCase() !== '.json') { - return false; - } + for await (const entry of walk(this.config.root, { + filter: (entry: WALK_ENTRY): boolean => { + const extension = path.extname(entry.path); + if (extension.toLowerCase() !== ".json") { + return false; + } - if (entry.info.isDirectory || entry.info.isSymlink) { - return false; - } + if (entry.info.isDirectory || entry.info.isSymlink) { + return false; + } - const filename = path.basename(entry.path); - if (filename === '.fsdb.collection.json') { - return false; - } + const filename = path.basename(entry.path); + if (filename === ".fsdb.collection.json") { + return false; + } - return filter ? filter(entry) : true; - }, - sort - }) - ) { + return filter ? filter(entry) : true; + }, + sort, + })) { if (counter < offset) { ++counter; continue; @@ -400,22 +416,25 @@ export class FSDB_COLLECTION> { results.push(entry); ++counter; - if (counter >= (offset + limit)) { + if (counter >= offset + limit) { break; } } - if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); - if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')); + if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end"); + if (Deno.env.get("FSDB_PERF")) + console.dir( + performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"), + ); - this.emit('all', { + this.emit("all", { options: { limit, offset, filter, - sort + sort, }, - results + results, }); return results; @@ -428,33 +447,47 @@ export class FSDB_COLLECTION> { * @param {FSDB_SEARCH_OPTIONS} options * @returns {Promise[]>} Returns an array of `WALK_ENTRY` items for items given the input options. */ - async find(criteria: Record, options?: FSDB_SEARCH_OPTIONS): Promise[]> { - if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin'); + async find( + criteria: Record, + options?: FSDB_SEARCH_OPTIONS, + ): Promise[]> { + if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin"); const find_options: FSDB_SEARCH_OPTIONS = { ...{ limit: 100, - offset: 0 + offset: 0, }, - ...(options ?? {}) + ...(options ?? {}), }; const results: WALK_ENTRY[] = []; - const item_paths: string[] = []; + const item_path_criteria_match_count: Record = {}; + const criteria_keys: string[] = Object.keys(criteria); + const score_needed = criteria_keys.length; - for (const search_key of Object.keys(criteria)) { + for (const search_key of criteria_keys) { const indexer_for_search_key: FSDB_INDEXER | undefined = this.INDEX[search_key]; const value: string = criteria[search_key]; if (indexer_for_search_key) { - item_paths.push(...await indexer_for_search_key.lookup(value, find_options)); + const matched_items = await indexer_for_search_key.lookup(value, find_options); + for (const matched_item_path of matched_items) { + item_path_criteria_match_count[matched_item_path] = + item_path_criteria_match_count[matched_item_path] ?? 0; + item_path_criteria_match_count[matched_item_path]++; + } } } + const matching_items = Object.keys(item_path_criteria_match_count).filter( + (item_path) => item_path_criteria_match_count[item_path] === score_needed, + ); + const limit = find_options.limit ?? 100; const offset = find_options.offset ?? 0; let counter = 0; - for await (const item_path of item_paths) { + for await (const item_path of matching_items) { if (counter < offset) { ++counter; continue; @@ -467,23 +500,24 @@ export class FSDB_COLLECTION> { depth: -1, load: function () { return JSON.parse(Deno.readTextFileSync(this.path)) as T; - } + }, }); ++counter; - if (counter >= (offset + limit)) { + if (counter >= offset + limit) { break; } } - if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end'); - if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end')); + if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end"); + if (Deno.env.get("FSDB_PERF")) + console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end")); - this.emit('find', { + this.emit("find", { criteria, options: find_options, - results + results, }); return results; @@ -496,18 +530,19 @@ export class FSDB_COLLECTION> { * @param {(event_data: any) => void} handler The handler for the event. */ public on(event: string, handler: (event_data: any) => void) { - const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; + const listeners: ((event: any) => void)[] = (this.event_listeners[event] = + this.event_listeners[event] ?? []); if (!listeners.includes(handler)) { listeners.push(handler); } - if (Deno.env.get('FSDB_LOG_EVENTS')) { + if (Deno.env.get("FSDB_LOG_EVENTS")) { console.dir({ on: { event, - handler + handler, }, - listeners + listeners, }); } } @@ -519,32 +554,34 @@ export class FSDB_COLLECTION> { * @param {(event_data: any) => void} handler The handler that was registered that should be removed. */ public off(event: string, handler: (event_data: any) => void) { - const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; + const listeners: ((event: any) => void)[] = (this.event_listeners[event] = + this.event_listeners[event] ?? []); if (listeners.includes(handler)) { listeners.splice(listeners.indexOf(handler), 1); } - if (Deno.env.get('FSDB_LOG_EVENTS')) { + if (Deno.env.get("FSDB_LOG_EVENTS")) { console.dir({ off: { event: event, - handler + handler, }, - listeners + listeners, }); } } private emit(event_name: string, event_data: any) { - const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? []; + const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] = + this.event_listeners[event_name] ?? []); - if (Deno.env.get('FSDB_LOG_EVENTS')) { + if (Deno.env.get("FSDB_LOG_EVENTS")) { console.dir({ emitting: { event_name, event_data, - listeners - } + listeners, + }, }); } @@ -555,19 +592,23 @@ export class FSDB_COLLECTION> { public sorts = { newest: (a: WALK_ENTRY, b: WALK_ENTRY): number => - ((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare( - (a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '' + ((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare( + (a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "", ), oldest: (a: WALK_ENTRY, b: WALK_ENTRY): number => - ((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare( - (b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '' + ((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare( + (b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "", ), latest: (a: WALK_ENTRY, b: WALK_ENTRY): number => - ((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare((a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''), + ((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare( + (a.info.mtime ?? a.info.ctime)?.toISOString() ?? "", + ), stalest: (a: WALK_ENTRY, b: WALK_ENTRY): number => - ((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '') + ((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare( + (b.info.mtime ?? b.info.ctime)?.toISOString() ?? "", + ), }; } diff --git a/tests/04_indexing_sanity_checks.test.ts b/tests/04_indexing_sanity_checks.test.ts index ad79e3e..4019d1c 100644 --- a/tests/04_indexing_sanity_checks.test.ts +++ b/tests/04_indexing_sanity_checks.test.ts @@ -1,57 +1,66 @@ -import * as asserts from '@std/assert'; -import * as fsdb from '../fsdb.ts'; -import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts'; -import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts'; -import lurid from '@andyburke/lurid'; -import by_email from '../organizers/by_email.ts'; -import by_character from '../organizers/by_character.ts'; -import by_phone from '../organizers/by_phone.ts'; -import { sentence } from 'jsr:@ndaidong/txtgen'; +import * as asserts from "@std/assert"; +import * as fsdb from "../fsdb.ts"; +import { FSDB_INDEXER_SYMLINKS } from "../indexers.ts"; +import { get_data_dir, random_email_address, random_phone_number } from "./helpers.ts"; +import lurid from "@andyburke/lurid"; +import by_email from "../organizers/by_email.ts"; +import by_character from "../organizers/by_character.ts"; +import by_phone from "../organizers/by_phone.ts"; +import { sentence } from "jsr:@ndaidong/txtgen"; Deno.test({ - name: 'index some items', + name: "index some items", permissions: { env: true, // https://github.com/denoland/deno/discussions/17258 read: true, - write: true + write: true, }, fn: async () => { type ITEM = { id: string; email: string; phone: string; + stable: string; value: string; }; const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ - name: 'test-04-items', - root: get_data_dir() + '/test-04-items', + name: "test-04-items", + root: get_data_dir() + "/test-04-items", indexers: { email: new FSDB_INDEXER_SYMLINKS({ - name: 'email', - field: 'email', - organize: by_email + name: "email", + field: "email", + organize: by_email, }), phone: new FSDB_INDEXER_SYMLINKS({ - name: 'phone', - field: 'phone', - organize: by_phone + name: "phone", + field: "phone", + organize: by_phone, + }), + stable: new FSDB_INDEXER_SYMLINKS({ + name: "stable", + field: "stable", + to_many: true, + organize: by_character, }), by_character_test: new FSDB_INDEXER_SYMLINKS({ - name: 'by_character_test', + name: "by_character_test", organize: by_character, - get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), - to_many: true + get_values_to_index: (item: ITEM) => + item.value.split(/\W/).filter((word) => word.length > 3), + to_many: true, }), by_possibly_undefined: new FSDB_INDEXER_SYMLINKS({ - name: 'by_possibly_undefined', + name: "by_possibly_undefined", organize: by_character, - get_values_to_index: (item: ITEM) => item.email.indexOf('.com') > 0 ? [item.email] : [], - to_many: true - }) - } + get_values_to_index: (item: ITEM) => + item.email.indexOf(".com") > 0 ? [item.email] : [], + to_many: true, + }), + }, }); asserts.assert(item_collection); @@ -62,7 +71,8 @@ Deno.test({ id: lurid(), email: random_email_address(), phone: random_phone_number(), - value: sentence() + stable: "stable", + value: sentence(), }; items.push(item); @@ -73,24 +83,58 @@ Deno.test({ } for (const item of items) { - const fetched_by_email: ITEM[] = (await item_collection.find({ email: item.email })).map((entry) => entry.load()); + const fetched_by_email: ITEM[] = ( + await item_collection.find({ email: item.email }) + ).map((entry) => entry.load()); asserts.assertLess(fetched_by_email.length, items.length); asserts.assertGreater(fetched_by_email.length, 0); asserts.assert(fetched_by_email.find((email_item) => email_item.id === item.id)); - const fetched_by_phone: ITEM[] = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load()); + const fetched_by_phone: ITEM[] = ( + await item_collection.find({ phone: item.phone }) + ).map((entry) => entry.load()); asserts.assertLess(fetched_by_phone.length, items.length); asserts.assertGreater(fetched_by_phone.length, 0); asserts.assert(fetched_by_phone.find((phone_item) => phone_item.id === item.id)); - const words_in_value: string[] = item.value.split(/\W/).filter((word) => word.length > 3); - const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)]; - const fetched_by_word_in_value: ITEM[] = (await item_collection.find({ by_character_test: random_word_in_value })).map(( - entry - ) => entry.load()); + const words_in_value: string[] = item.value + .split(/\W/) + .filter((word) => word.length > 3); + const random_word_in_value: string = + words_in_value[Math.floor(Math.random() * words_in_value.length)]; + const fetched_by_word_in_value: ITEM[] = ( + await item_collection.find({ by_character_test: random_word_in_value }) + ).map((entry) => entry.load()); asserts.assertLess(fetched_by_word_in_value.length, items.length); asserts.assertGreater(fetched_by_word_in_value.length, 0); - asserts.assert(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id)); + asserts.assert( + fetched_by_word_in_value.find( + (word_in_value_item) => word_in_value_item.id === item.id, + ), + ); + } + + for (let i = 0; i < 10; ++i) { + const random_item = items[Math.floor(Math.random() * items.length)]; + asserts.assert(random_item); + + const criteria: Record = { + stable: "stable", + }; + + if (Math.random() < 0.5) { + criteria.email = random_item.email; + } else { + criteria.phone = random_item.phone; + } + + const found_entries = await item_collection.find(criteria); + asserts.assertEquals(found_entries.length, 1); + + const found: ITEM = found_entries[0].load(); + + asserts.assert(found); + asserts.assertEquals(found, random_item); } // leave one item behind so the whole db for this test doesn't get cleaned up so I can hand-review it @@ -110,5 +154,5 @@ Deno.test({ // ) => entry.load()); // asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id)); // } - } + }, });