From 3214d17b809f3e265ede68cdb0f268a89703ed9f Mon Sep 17 00:00:00 2001 From: Andy Burke Date: Tue, 1 Jul 2025 19:14:18 -0700 Subject: [PATCH] feature: emit events --- README.md | 25 +- cli.ts | 8 +- deno.json | 4 +- fsdb.ts | 116 ++++++- tests/05_test_events.test.ts | 319 ++++++++++++++++++ ...5_test_all.test.ts => 06_test_all.test.ts} | 4 +- tests/helpers.ts | 2 +- 7 files changed, 460 insertions(+), 18 deletions(-) create mode 100644 tests/05_test_events.test.ts rename tests/{05_test_all.test.ts => 06_test_all.test.ts} (98%) diff --git a/README.md b/README.md index e2e9e97..7b27462 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ optimization to the filesystem layer. `collection.delete(T)` - removes an object from the system and the disk `collection.all([options])` - iterate over all objects `collection.find(criteria[, options])` - find all objects that match the criteria *that have an index* + `collection.on(event, callback)` - set a callback for the given event (create,update,get,delete,write,index,all,find) + `collection.off(event, callback)` - remove a callback for the given event ### Example @@ -157,6 +159,27 @@ for browsing the data as a human. TODO: index everything into a sqlite setup as well? would give a way to run SQL against data still stored on disk in a nicely human browsable format. +## Environment Variables + + | variable | description | + | --------------------------- | ----------------------------------------------- | + | FSDB_ROOT | controls the root directory, default: ./.fsdb | + | FSDB_PERF | set to true for performance tracking | + | FSDB_LOG_EVENTS | set to true to log the events system | + ## TODO - - [ ] make all()/find() return something like { file_info, entry: { private data = undefined; load() => { data = data ?? await Deno.readTextFile(this.file_info.path); return data; } } } + - [ ] make all()/find() return something like + +``` +{ + file_info, + entry: { + private data = undefined; + load() => { + data = data ?? await Deno.readTextFile(this.file_info.path); + return data; + } + } +} +``` diff --git a/cli.ts b/cli.ts index 4e71b5b..ec12591 100644 --- a/cli.ts +++ b/cli.ts @@ -91,7 +91,7 @@ switch (command) { Deno.exit(1); } - console.log(JSON.stringify(item, null, 4)); + console.log(JSON.stringify(item, null, '\t')); break; } case 'create': { @@ -109,7 +109,7 @@ switch (command) { const item: any = JSON.parse(item_json); const created_item: any = await collection.create(item); - console.log('created: ' + JSON.stringify(created_item, null, 4)); + console.log('created: ' + JSON.stringify(created_item, null, '\t')); break; } case 'update': { @@ -127,7 +127,7 @@ switch (command) { const item: any = JSON.parse(item_json); const updated_item: any = await collection.update(item); - console.log('updated: ' + JSON.stringify(updated_item, null, 4)); + console.log('updated: ' + JSON.stringify(updated_item, null, '\t')); break; } case 'delete': { @@ -145,7 +145,7 @@ switch (command) { const item: any = JSON.parse(item_json); const deleted_item: any = await collection.delete(item); - console.log('deleted: ' + JSON.stringify(deleted_item, null, 4)); + console.log('deleted: ' + JSON.stringify(deleted_item, null, '\t')); break; } case 'find': diff --git a/deno.json b/deno.json index 2edb6f1..1502e0a 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@andyburke/fsdb", - "version": "0.6.1", + "version": "0.7.0", "license": "MIT", "exports": { ".": "./fsdb.ts", @@ -12,7 +12,7 @@ "tasks": { "lint": "deno lint", "fmt": "deno fmt", - "test": "cd tests && DENO_ENV=test TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./", + "test": "cd tests && DENO_ENV=test FSDB_TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./", "fsdb": "deno run --allow-env --allow-read --allow-write cli.ts" }, diff --git a/fsdb.ts b/fsdb.ts index 2513bef..fed59ec 100644 --- a/fsdb.ts +++ b/fsdb.ts @@ -34,6 +34,7 @@ export interface FSDB_INDEXER { export class FSDB_COLLECTION> { private config: FSDB_COLLECTION_CONFIG; public INDEX: Record>; + private event_listeners: Record; constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) { this.config = { @@ -45,6 +46,8 @@ export class FSDB_COLLECTION> { ...(input_config ?? {}) }; + this.event_listeners = {}; + this.INDEX = this.config.indexers ?? {}; for (const indexer of Object.values(this.INDEX)) { indexer.set_fsdb_root(this.config.root); @@ -84,7 +87,7 @@ export class FSDB_COLLECTION> { } const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json')); - const collection_info_json: string = JSON.stringify(this.config, null, 4); + const collection_info_json: string = JSON.stringify(this.config, null, '\t'); Deno.mkdirSync(path.dirname(collection_info_file_path), { recursive: true }); @@ -114,26 +117,43 @@ export class FSDB_COLLECTION> { private async write_item(item: T, override_path?: string): Promise { const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field); - Deno.writeTextFileSync(item_path, JSON.stringify(item, null, 1)); + Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t')); + + this.emit('write', { + item, + item_path + }); if (this.config.indexers) { for (const indexer of Object.values(this.config.indexers)) { await (indexer as FSDB_INDEXER).index(item, item_path); + this.emit('index', { + item, + item_path, + indexer + }); } } } /** Get an item from the collection given its id. */ async get(id: string): Promise { - const id_path: string = this.get_organized_id_path(id); - const item_exists: boolean = await fs.exists(id_path); + const item_path: string = this.get_organized_id_path(id); + const item_exists: boolean = await fs.exists(item_path); if (!item_exists) { return null; } - const content: string = await Deno.readTextFile(id_path); - return JSON.parse(content); + const content: string = await Deno.readTextFile(item_path); + const item: T = JSON.parse(content); + + this.emit('get', { + item, + item_path + }); + + return item; } /** Create an item in the collection. */ @@ -149,15 +169,21 @@ export class FSDB_COLLECTION> { await this.write_item(item); + this.emit('create', { + item, + item_path + }); + return item; } /** Update the given item in the collection, requiring the id to be stable. */ async update(item: T): Promise { const item_path: string = this.get_organized_item_path(item); - const item_exists: boolean = await fs.exists(item_path); + const id: string = item[this.config.id_field]; + const previous: T | null = await this.get(id); - if (!item_exists) { + if (!previous) { throw new Error('item does not exist', { cause: 'item_does_not_exist' }); @@ -165,6 +191,12 @@ export class FSDB_COLLECTION> { await this.write_item(item, item_path); + this.emit('update', { + item, + previous, + item_path + }); + return item; } @@ -202,6 +234,11 @@ export class FSDB_COLLECTION> { await Deno.remove(dir); dir = path.dirname(dir); } while (dir.length); + + this.emit('delete', { + item + }); + return item; } @@ -302,6 +339,11 @@ export class FSDB_COLLECTION> { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')); + this.emit('all', { + options, + results + }); + return results; } @@ -350,6 +392,64 @@ export class FSDB_COLLECTION> { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end'); if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end')); + this.emit('find', { + criteria, + options, + results + }); + return results; } + + public on(event: string, handler: (event_data: any) => void) { + const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; + if (!listeners.includes(handler)) { + listeners.push(handler); + } + + if (Deno.env.get('FSDB_LOG_EVENTS')) { + console.dir({ + on: { + event, + handler + }, + listeners + }); + } + } + + public off(event: string, handler: (event_data: any) => void) { + const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? []; + if (listeners.includes(handler)) { + listeners.splice(listeners.indexOf(handler), 1); + } + + if (Deno.env.get('FSDB_LOG_EVENTS')) { + console.dir({ + off: { + event: event, + handler + }, + listeners + }); + } + } + + private emit(event_name: string, event_data: any) { + const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? []; + + if (Deno.env.get('FSDB_LOG_EVENTS')) { + console.dir({ + emitting: { + event_name, + event_data, + listeners + } + }); + } + + for (const listener of listeners) { + listener(event_data); + } + } } diff --git a/tests/05_test_events.test.ts b/tests/05_test_events.test.ts new file mode 100644 index 0000000..15f5ba2 --- /dev/null +++ b/tests/05_test_events.test.ts @@ -0,0 +1,319 @@ +import * as asserts from '@std/assert'; +import * as fsdb from '../fsdb.ts'; +import { get_data_dir } from './helpers.ts'; +import lurid from '@andyburke/lurid'; +import { FSDB_INDEXER_SYMLINKS } from '../indexers/symlinks.ts'; + +type ITEM = { + id: string; + value: string; + created: string; +}; + +const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ + name: 'test-05-items', + root: get_data_dir() + '/test-05-items', + indexers: { + email: new FSDB_INDEXER_SYMLINKS({ + name: 'created', + field: 'created', + organize: (value) => { + const date = new Date(value); + return [`${date.getFullYear()}`, `${date.getMonth()}`, `${date.getDate()}`, `${value}.json`]; + } + }) + } +}); + +Deno.test({ + name: 'events - create', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let create_event: any = null; + item_collection.on('create', (event) => { + create_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + + asserts.assert(create_event); + asserts.assertEquals(create_event?.item, item); + asserts.assert(create_event?.item_path); + } +}); + +Deno.test({ + name: 'events - delete', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let delete_event: any = null; + item_collection.on('delete', (event) => { + delete_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + await item_collection.delete(item); + + asserts.assert(delete_event); + asserts.assertEquals(delete_event?.item, item); + } +}); + +Deno.test({ + name: 'events - write', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let create_write_event: any = null; + function set_create_event(event: any): void { + create_write_event = event; + } + item_collection.on('write', set_create_event); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + + item_collection.off('write', set_create_event); + + asserts.assert(create_write_event); + asserts.assertEquals(create_write_event?.item, item); + asserts.assert(create_write_event.item_path); + + const updated_item = { ...item }; + updated_item.value = 'different'; + + let update_write_event: any = null; + function set_update_event(event: any): void { + update_write_event = event; + } + item_collection.on('write', set_update_event); + + await item_collection.update(updated_item); + + item_collection.off('write', set_update_event); + + asserts.assert(update_write_event); + asserts.assertEquals(update_write_event?.item, updated_item); + asserts.assert(update_write_event.item_path); + } +}); + +Deno.test({ + name: 'events - get', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let get_event: any = null; + item_collection.on('get', (event) => { + get_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + const fetched_item: ITEM | null = await item_collection.get(item.id); + + asserts.assertEquals(fetched_item, item); + + asserts.assert(get_event); + asserts.assertEquals(get_event?.item, fetched_item); + asserts.assert(get_event?.item_path); + } +}); + +Deno.test({ + name: 'events - index', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let index_event: any = null; + item_collection.on('index', (event) => { + index_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + + asserts.assert(index_event); + asserts.assertEquals(index_event?.item, item); + asserts.assert(index_event?.indexer); + } +}); + +Deno.test({ + name: 'events - update', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let update_event: any = null; + item_collection.on('update', (event) => { + update_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + + const updated_item = { ...item }; + updated_item.created = new Date().toISOString(); + + await item_collection.update(updated_item); + + asserts.assert(update_event); + asserts.assertEquals(update_event?.item, updated_item); + asserts.assertEquals(update_event.previous, item); + } +}); + +Deno.test({ + name: 'events - find', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let find_event: any = null; + item_collection.on('find', (event) => { + find_event = event; + }); + + const now = new Date().toISOString(); + const item = { + id: lurid(), + value: 'test', + created: now + }; + + await item_collection.create(item); + + const criteria = { + created: now + }; + + const options = { + limit: 3 + }; + const results = await item_collection.find(criteria, options); + + asserts.assert(find_event); + asserts.assertEquals(find_event.criteria, criteria); + asserts.assertEquals(find_event.options?.limit, options.limit); + asserts.assertEquals(find_event.results, results); + } +}); + +Deno.test({ + name: 'events - all', + permissions: { + env: true, + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + asserts.assert(item_collection); + + let all_event: any = null; + item_collection.on('all', (event) => { + all_event = event; + }); + + for (let i = 0; i < 5; ++i) { + const item = { + id: lurid(), + value: 'test ' + i, + created: new Date().toISOString() + }; + + await item_collection.create(item); + } + + const options = { + limit: 2 + }; + const results = await item_collection.all(options); + + asserts.assert(all_event); + asserts.assertEquals(results.length, options.limit); + asserts.assertEquals(all_event.options?.limit, options.limit); + asserts.assertEquals(all_event.results, results); + } +}); diff --git a/tests/05_test_all.test.ts b/tests/06_test_all.test.ts similarity index 98% rename from tests/05_test_all.test.ts rename to tests/06_test_all.test.ts index abe25c2..d69f908 100644 --- a/tests/05_test_all.test.ts +++ b/tests/06_test_all.test.ts @@ -22,8 +22,8 @@ Deno.test({ }; const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ - name: 'test-05-items', - root: get_data_dir() + '/test-05-items' + name: 'test-06-items', + root: get_data_dir() + '/test-06-items' }); asserts.assert(item_collection); diff --git a/tests/helpers.ts b/tests/helpers.ts index 1ac87e8..3552b83 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -49,5 +49,5 @@ export function random_phone_number(): string { const DATA_DIR = lurid(); export function get_data_dir(): string { - return Deno.env.get('TEST_DATA_STORAGE_ROOT') ?? DATA_DIR; + return Deno.env.get('FSDB_TEST_DATA_STORAGE_ROOT') ?? DATA_DIR; }