diff --git a/README.md b/README.md index e2e9e97..e5de73e 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,3 @@ for browsing the data as a human. TODO: index everything into a sqlite setup as well? would give a way to run SQL against data still stored on disk in a nicely human browsable format. - -## TODO - - - [ ] make all()/find() return something like { file_info, entry: { private data = undefined; load() => { data = data ?? await Deno.readTextFile(this.file_info.path); return data; } } } diff --git a/deno.json b/deno.json index 3533911..73650ee 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@andyburke/fsdb", - "version": "0.6.0", + "version": "0.5.0", "license": "MIT", "exports": { ".": "./fsdb.ts", diff --git a/fsdb.ts b/fsdb.ts index 5377926..900c6ee 100644 --- a/fsdb.ts +++ b/fsdb.ts @@ -15,12 +15,6 @@ export type FSDB_COLLECTION_CONFIG_INPUT = Optional { @@ -232,51 +226,6 @@ export class FSDB_COLLECTION> { exts: ['json'] }) ) { - let item_stat = null; - if (options.before) { - item_stat = item_stat ?? await Deno.lstat(entry.path); - const birthtime = (item_stat.birthtime ?? new Date(0)).toISOString(); - if (birthtime > options.before) { - continue; - } - } - - if (options.after) { - item_stat = item_stat ?? await Deno.lstat(entry.path); - if ((item_stat.birthtime ?? new Date(0)).toISOString() < options.after) { - continue; - } - } - - if (options.modified_before) { - item_stat = item_stat ?? await Deno.lstat(entry.path); - if ((item_stat.mtime ?? new Date(0)).toISOString() > options.modified_before) { - continue; - } - } - - if (options.modified_after) { - item_stat = item_stat ?? await Deno.lstat(entry.path); - if ((item_stat.mtime ?? new Date(0)).toISOString() < options.modified_after) { - continue; - } - } - - let item_id = null; - if (options.id_before) { - item_id = item_id ?? entry.name.replace(/\.json$/, ''); - if (item_id >= options.id_before) { - continue; - } - } - - if (options.id_after) { - item_id = item_id ?? entry.name.replace(/\.json$/, ''); - if (item_id <= options.id_after) { - continue; - } - } - if (counter < offset) { ++counter; continue; diff --git a/tests/05_test_all.test.ts b/tests/05_test_all.test.ts index abe25c2..ac221d2 100644 --- a/tests/05_test_all.test.ts +++ b/tests/05_test_all.test.ts @@ -1,7 +1,12 @@ import * as asserts from '@std/assert'; import * as fsdb from '../fsdb.ts'; +import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts'; import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts'; import lurid from '@andyburke/lurid'; +import by_email from '../organizers/by_email.ts'; +import by_character from '../organizers/by_character.ts'; +import by_phone from '../organizers/by_phone.ts'; +import { sentence } from 'jsr:@ndaidong/txtgen'; Deno.test({ name: 'iterate over all items', @@ -17,206 +22,73 @@ Deno.test({ id: string; email: string; phone: string; - created: string; - written_by_time?: string; + value: string; }; const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ name: 'test-05-items', - root: get_data_dir() + '/test-05-items' + root: get_data_dir() + '/test-05-items', + indexers: { + email: new FSDB_INDEXER_SYMLINKS({ + name: 'email', + field: 'email', + organize: by_email + }), + phone: new FSDB_INDEXER_SYMLINKS({ + name: 'phone', + field: 'phone', + organize: by_phone + }), + by_character_test: new FSDB_INDEXER_SYMLINKS({ + name: 'by_character_test', + organize: by_character, + get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), + to_many: true + }) + } }); asserts.assert(item_collection); const items: ITEM[] = []; - const item_count: number = 500; - const midpoint: number = Math.floor(item_count / 2); - let first_id = null; - let time_mid = null; - let mid_id = null; - let time_end = null; - let last_id = null; - for (let i = 0; i < item_count; ++i) { - const item: ITEM = { + for (let i = 0; i < 500; ++i) { + const item = { id: lurid(), email: random_email_address(), phone: random_phone_number(), - created: new Date().toISOString() + value: sentence() }; - first_id = first_id ?? item.id; - items.push(item); const stored_item: ITEM = await item_collection.create(item); - item.written_by_time = new Date().toISOString(); asserts.assertObjectMatch(stored_item, item); - - if (i === midpoint) { - time_mid = item.written_by_time; - mid_id = item.id; - } - - time_end = item.written_by_time; - last_id = item.id; } const LIMIT_MIN = 11; const LIMIT_MAX = 333; - let fetch_for_sort_offset = 0; - const fetched_for_sort = []; - let more_to_fetch_for_sorting = true; + let offset = 0; + const fetched = []; + let more_to_fetch = true; do { // fuzz the limit const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; const fetched_items = await item_collection.all({ limit, - offset: fetch_for_sort_offset + offset }); - fetched_for_sort.push(...fetched_items); - fetch_for_sort_offset += fetched_items.length; - more_to_fetch_for_sorting = fetched_items.length === limit; - } while (more_to_fetch_for_sorting); + fetched.push(...fetched_items); + offset += fetched_items.length; + more_to_fetch = fetched_items.length === limit; + } while (more_to_fetch); - const sorted_items = items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)).map((item) => { - return { id: item.id, email: item.email, phone: item.phone, created: item.created }; - }); - const sorted_fetched = fetched_for_sort.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)); + const sorted_items = items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)); + const sorted_fetched = fetched.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)); asserts.assertEquals(sorted_fetched, sorted_items); - - asserts.assert(time_mid); - asserts.assert(time_end); - - asserts.assert(mid_id); - asserts.assert(last_id); - - // test before - let fetch_for_before_offset = 0; - const fetched_for_before = []; - let more_to_fetch_for_before = true; - do { - // fuzz the limit - const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; - - const fetched_items = await item_collection.all({ - limit, - offset: fetch_for_before_offset, - before: time_mid - }); - - fetched_for_before.push(...fetched_items); - fetch_for_before_offset += fetched_items.length; - more_to_fetch_for_before = fetched_items.length === limit; - } while (more_to_fetch_for_before); - - let newest = new Date(0).toISOString(); - asserts.assert(newest); - for (const item of fetched_for_before) { - const original_item = items.find((_) => _.id === item.id); - asserts.assert(original_item); - asserts.assert(original_item.written_by_time); - if (original_item.written_by_time > newest) { - newest = original_item.written_by_time; - } - } - - // note: we use less or equal because we don't have the actual file write time - asserts.assertLessOrEqual(newest, time_mid); - - // test id_before - let fetch_for_id_before_offset = 0; - const fetched_for_id_before = []; - let more_to_fetch_for_id_before = true; - do { - // fuzz the limit - const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; - - const fetched_items = await item_collection.all({ - limit, - offset: fetch_for_id_before_offset, - id_before: mid_id - }); - - fetched_for_id_before.push(...fetched_items); - fetch_for_id_before_offset += fetched_items.length; - more_to_fetch_for_id_before = fetched_items.length === limit; - } while (more_to_fetch_for_id_before); - - let newest_id = first_id; - asserts.assert(newest_id); - for (const item of fetched_for_id_before) { - if (item.id > newest_id) { - newest_id = item.id; - } - } - - asserts.assertLess(newest_id, mid_id); - - // test after - let fetch_for_after_offset = 0; - const fetched_for_after = []; - let more_to_fetch_for_after = true; - do { - // fuzz the limit - const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; - - const fetched_items = await item_collection.all({ - limit, - offset: fetch_for_after_offset, - after: time_mid - }); - - fetched_for_after.push(...fetched_items); - fetch_for_after_offset += fetched_items.length; - more_to_fetch_for_after = fetched_items.length === limit; - } while (more_to_fetch_for_after); - - let oldest = new Date().toISOString(); - asserts.assert(oldest); - for (const item of fetched_for_after) { - const original_item = items.find((_) => _.id === item.id); - asserts.assert(original_item); - asserts.assert(original_item.written_by_time); - if (original_item.written_by_time < oldest) { - oldest = original_item.written_by_time; - } - } - - // again with the file write time slop - asserts.assertGreaterOrEqual(oldest, time_mid); - - // test id_after - let fetch_for_id_after_offset = 0; - const fetched_for_id_after = []; - let more_to_fetch_for_id_after = true; - do { - // fuzz the limit - const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN; - - const fetched_items = await item_collection.all({ - limit, - offset: fetch_for_id_after_offset, - id_after: mid_id - }); - - fetched_for_id_after.push(...fetched_items); - fetch_for_id_after_offset += fetched_items.length; - more_to_fetch_for_id_after = fetched_items.length === limit; - } while (more_to_fetch_for_id_after); - - let oldest_id = last_id; - asserts.assert(oldest_id); - for (const item of fetched_for_id_after) { - if (item.id < oldest_id) { - oldest_id = item.id; - } - } - - asserts.assertGreater(oldest_id, mid_id); } });