diff --git a/README.md b/README.md index da60835..e5de73e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ optimization to the filesystem layer. `collection.get(id) : T` - gets an object of type T based on the id field configured `collection.update(T)` - updates an object of type T, saving it to the disk `collection.delete(T)` - removes an object from the system and the disk - `collection.find(criteria)` - find all objects that match the criteria *that have an index* + `collection.all([options])` - iterate over all objects + `collection.find(criteria[, options])` - find all objects that match the criteria *that have an index* ### Example diff --git a/deno.json b/deno.json index 814170e..73650ee 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@andyburke/fsdb", - "version": "0.4.0", + "version": "0.5.0", "license": "MIT", "exports": { ".": "./fsdb.ts", diff --git a/fsdb.ts b/fsdb.ts index b396716..900c6ee 100644 --- a/fsdb.ts +++ b/fsdb.ts @@ -199,6 +199,53 @@ export class FSDB_COLLECTION> { return item; } + /** Iterate through the items. */ + async all(input_options?: FSDB_SEARCH_OPTIONS): Promise { + if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin'); + + const options: FSDB_SEARCH_OPTIONS = { + ...{ + limit: 100, + offset: 0 + }, + ...(input_options ?? {}) + }; + + const results: T[] = []; + + const limit = options?.limit ?? 100; + const offset = options?.offset ?? 0; + let counter = 0; + + // TODO: better way to get a pattern to match files in this collection? + for await ( + const entry of fs.walk(this.config.root, { + includeDirs: false, + includeSymlinks: false, + skip: [/\.fsdb\.collection\.json$/], + exts: ['json'] + }) + ) { + if (counter < offset) { + ++counter; + continue; + } + + const content = await Deno.readTextFile(entry.path); + results.push(JSON.parse(content)); + ++counter; + + if (counter >= (offset + limit)) { + break; + } + } + + if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); + if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')); + + return results; + } + /** Use indexes to search for matching items. */ async find(criteria: Record, input_options?: FSDB_SEARCH_OPTIONS): Promise { if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin'); diff --git a/tests/05_test_all.test.ts b/tests/05_test_all.test.ts new file mode 100644 index 0000000..ac221d2 --- /dev/null +++ b/tests/05_test_all.test.ts @@ -0,0 +1,94 @@ +import * as asserts from '@std/assert'; +import * as fsdb from '../fsdb.ts'; +import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts'; +import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts'; +import lurid from '@andyburke/lurid'; +import by_email from '../organizers/by_email.ts'; +import by_character from '../organizers/by_character.ts'; +import by_phone from '../organizers/by_phone.ts'; +import { sentence } from 'jsr:@ndaidong/txtgen'; + +Deno.test({ + name: 'iterate over all items', + permissions: { + env: true, + + // https://github.com/denoland/deno/discussions/17258 + read: true, + write: true + }, + fn: async () => { + type ITEM = { + id: string; + email: string; + phone: string; + value: string; + }; + + const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ + name: 'test-05-items', + root: get_data_dir() + '/test-05-items', + indexers: { + email: new FSDB_INDEXER_SYMLINKS({ + name: 'email', + field: 'email', + organize: by_email + }), + phone: new FSDB_INDEXER_SYMLINKS({ + name: 'phone', + field: 'phone', + organize: by_phone + }), + by_character_test: new FSDB_INDEXER_SYMLINKS({ + name: 'by_character_test', + organize: by_character, + get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), + to_many: true + }) + } + }); + + asserts.assert(item_collection); + + const items: ITEM[] = []; + for (let i = 0; i < 500; ++i) { + const item = { + id: lurid(), + email: random_email_address(), + phone: random_phone_number(), + value: sentence() + }; + + items.push(item); + + const stored_item: ITEM = await item_collection.create(item); + + asserts.assertObjectMatch(stored_item, item); + } + + const LIMIT_MIN = 11; + const LIMIT_MAX = 333; + + let offset = 0; + const fetched = []; + let more_to_fetch = true; + do { + // fuzz the limit + const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; + + const fetched_items = await item_collection.all({ + limit, + offset + }); + + fetched.push(...fetched_items); + offset += fetched_items.length; + more_to_fetch = fetched_items.length === limit; + } while (more_to_fetch); + + const sorted_items = items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)); + const sorted_fetched = fetched.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)); + + asserts.assertEquals(sorted_fetched, sorted_items); + } +}); diff --git a/tests/helpers.ts b/tests/helpers.ts index 3533287..1ac87e8 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -14,10 +14,10 @@ const TLDs: string[] = [ const random_byte_buffer: Uint8Array = new Uint8Array(3); export function random_email_address(): string { - crypto.getRandomValues(random_byte_buffer); + for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256); const name = convert_to_words(random_byte_buffer).join('-'); - crypto.getRandomValues(random_byte_buffer); + for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256); const domain = convert_to_words(random_byte_buffer).join('-'); const tld = TLDs[Math.floor(Math.random() * TLDs.length)]; @@ -25,7 +25,7 @@ export function random_email_address(): string { } export function random_username(): string { - crypto.getRandomValues(random_byte_buffer); + for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256); return convert_to_words(random_byte_buffer).join('-'); }