andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: make all() take filter and sort options

Browse source 
refactor: return unload item entries from all()/find()
This commit is contained in:
Andy Burke 2025-07-02 17:46:04 -07:00
parent 3214d17b80
commit 05178c924f
8 changed files with 308 additions and 206 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tests/03_index_items.test.ts
View file

@ -65,19 +65,21 @@ Deno.test({
}
for (const item of items) {
const fetched_by_email: ITEM[] = await item_collection.find({ email: item.email });
const fetched_by_email: ITEM[] = (await item_collection.find({ email: item.email })).map((entry) => entry.load());
asserts.assertLess(fetched_by_email.length, items.length);
asserts.assertGreater(fetched_by_email.length, 0);
asserts.assert(fetched_by_email.find((email_item) => email_item.id === item.id));
const fetched_by_phone: ITEM[] = await item_collection.find({ phone: item.phone });
const fetched_by_phone: ITEM[] = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load());
asserts.assertLess(fetched_by_phone.length, items.length);
asserts.assertGreater(fetched_by_phone.length, 0);
asserts.assert(fetched_by_phone.find((phone_item) => phone_item.id === item.id));
const words_in_value: string[] = item.value.split(/\W/).filter((word) => word.length > 3);
const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = await item_collection.find({ by_character_test: random_word_in_value });
const fetched_by_word_in_value: ITEM[] = (await item_collection.find({ by_character_test: random_word_in_value })).map((
entry
) => entry.load());
asserts.assertLess(fetched_by_word_in_value.length, items.length);
asserts.assertGreater(fetched_by_word_in_value.length, 0);
asserts.assert(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));

16
tests/04_indexing_sanity_checks.test.ts
View file

@ -67,19 +67,21 @@ Deno.test({
}
for (const item of items) {
const fetched_by_email: ITEM[] = await item_collection.find({ email: item.email });
const fetched_by_email: ITEM[] = (await item_collection.find({ email: item.email })).map((entry) => entry.load());
asserts.assertLess(fetched_by_email.length, items.length);
asserts.assertGreater(fetched_by_email.length, 0);
asserts.assert(fetched_by_email.find((email_item) => email_item.id === item.id));
const fetched_by_phone: ITEM[] = await item_collection.find({ phone: item.phone });
const fetched_by_phone: ITEM[] = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load());
asserts.assertLess(fetched_by_phone.length, items.length);
asserts.assertGreater(fetched_by_phone.length, 0);
asserts.assert(fetched_by_phone.find((phone_item) => phone_item.id === item.id));
const words_in_value: string[] = item.value.split(/\W/).filter((word) => word.length > 3);
const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = await item_collection.find({ by_character_test: random_word_in_value });
const fetched_by_word_in_value: ITEM[] = (await item_collection.find({ by_character_test: random_word_in_value })).map((
entry
) => entry.load());
asserts.assertLess(fetched_by_word_in_value.length, items.length);
asserts.assertGreater(fetched_by_word_in_value.length, 0);
asserts.assert(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));
@ -89,15 +91,17 @@ Deno.test({
for (const item of items.slice(1)) {
await item_collection.delete(item);
const fetched_by_email: ITEM[] = await item_collection.find({ email: item.email });
const fetched_by_email: ITEM[] = (await item_collection.find({ email: item.email })).map((entry) => entry.load());
asserts.assertFalse(fetched_by_email.find((email_item) => email_item.id === item.id));
const fetched_by_phone: ITEM[] = await item_collection.find({ phone: item.phone });
const fetched_by_phone: ITEM[] = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load());
asserts.assertFalse(fetched_by_phone.find((phone_item) => phone_item.id === item.id));
const words_in_value: string[] = item.value.split(/\W/).filter((word) => word.length > 3);
const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = await item_collection.find({ by_character_test: random_word_in_value });
const fetched_by_word_in_value: ITEM[] = (await item_collection.find({ by_character_test: random_word_in_value })).map((
entry
) => entry.load());
asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));
}
}

276
tests/06_test_all.test.ts
View file

@ -1,93 +1,148 @@
import * as asserts from '@std/assert';
import * as fsdb from '../fsdb.ts';
import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts';
import * as path from '@std/path';
import { get_data_dir } from './helpers.ts';
import lurid from '@andyburke/lurid';
type ITEM = {
id: string;
value: string;
created: string;
written_by_time?: string;
};
const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
name: 'test-06-items',
root: get_data_dir() + '/test-06-items'
});
const items: ITEM[] = [];
const item_count: number = 1_000;
const midpoint: number = Math.floor(item_count / 2);
let first_id = null;
let time_mid = null;
let mid_id = null;
let time_end = null;
let last_id = null;
for (let i = 0; i < item_count; ++i) {
const item: ITEM = {
id: lurid(),
value: `${Math.random() * 10_000_000}`,
created: new Date().toISOString()
};
first_id = first_id ?? item.id;
items.push(item);
const stored_item: ITEM = await item_collection.create(item);
item.written_by_time = new Date().toISOString();
asserts.assertObjectMatch(stored_item, item);
if (i === midpoint) {
time_mid = item.written_by_time;
mid_id = item.id;
}
time_end = item.written_by_time;
last_id = item.id;
// wait a ms between items to help with sort stuff later
await new Promise((resolve) => setTimeout(resolve, 1));
}
const sorted_items = {
by_id: items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)).map((item) => {
return { id: item.id, value: item.value, created: item.created };
}),
by_created: items.sort((lhs, rhs) => lhs.created.localeCompare(rhs.created)).map((item) => {
return { id: item.id, value: item.value, created: item.created };
})
};
const LIMIT_MIN = 11;
const LIMIT_MAX = 333;
Deno.test({
name: 'iterate over all items',
name: 'sort all() by id',
permissions: {
env: true,
// https://github.com/denoland/deno/discussions/17258
read: true,
write: true
},
fn: async () => {
type ITEM = {
id: string;
email: string;
phone: string;
created: string;
written_by_time?: string;
};
const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
name: 'test-06-items',
root: get_data_dir() + '/test-06-items'
});
asserts.assert(item_collection);
const items: ITEM[] = [];
const item_count: number = 500;
const midpoint: number = Math.floor(item_count / 2);
let first_id = null;
let time_mid = null;
let mid_id = null;
let time_end = null;
let last_id = null;
for (let i = 0; i < item_count; ++i) {
const item: ITEM = {
id: lurid(),
email: random_email_address(),
phone: random_phone_number(),
created: new Date().toISOString()
};
first_id = first_id ?? item.id;
items.push(item);
const stored_item: ITEM = await item_collection.create(item);
item.written_by_time = new Date().toISOString();
asserts.assertObjectMatch(stored_item, item);
if (i === midpoint) {
time_mid = item.written_by_time;
mid_id = item.id;
}
time_end = item.written_by_time;
last_id = item.id;
}
const LIMIT_MIN = 11;
const LIMIT_MAX = 333;
let fetch_for_sort_offset = 0;
const fetched_for_sort = [];
let more_to_fetch_for_sorting = true;
let offset = 0;
const fetched = [];
let more = true;
do {
// fuzz the limit
const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN;
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_sort_offset
offset: offset,
sort: (a: fsdb.WALK_ENTRY<ITEM>, b: fsdb.WALK_ENTRY<ITEM>) =>
path.basename(a.path).replace(/\.json$/i, '').localeCompare(path.basename(b.path).replace(/\.json/i, ''))
});
fetched_for_sort.push(...fetched_items);
fetch_for_sort_offset += fetched_items.length;
more_to_fetch_for_sorting = fetched_items.length === limit;
} while (more_to_fetch_for_sorting);
fetched.push(...(fetched_items.map((item) => item.load())));
offset += fetched_items.length;
more = fetched_items.length === limit;
} while (more);
const sorted_items = items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id)).map((item) => {
return { id: item.id, email: item.email, phone: item.phone, created: item.created };
});
const sorted_fetched = fetched_for_sort.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id));
asserts.assertEquals(fetched, sorted_items.by_id);
}
});
asserts.assertEquals(sorted_fetched, sorted_items);
Deno.test({
name: 'sort all() by birthtime',
permissions: {
env: true,
// https://github.com/denoland/deno/discussions/17258
read: true,
write: true
},
fn: async () => {
asserts.assert(item_collection);
let offset = 0;
const fetched = [];
let more = true;
do {
// fuzz the limit
const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN;
const fetched_items = await item_collection.all({
limit,
offset: offset,
sort: (a: fsdb.WALK_ENTRY<ITEM>, b: fsdb.WALK_ENTRY<ITEM>) =>
(a.info.birthtime?.toISOString() ?? '').localeCompare(b.info.birthtime?.toISOString() ?? '')
});
fetched.push(...(fetched_items.map((item) => item.load())));
offset += fetched_items.length;
more = fetched_items.length === limit;
} while (more);
asserts.assertEquals(fetched, sorted_items.by_created);
}
});
Deno.test({
name: 'filter all() by birthtime',
permissions: {
env: true,
// https://github.com/denoland/deno/discussions/17258
read: true,
write: true
},
fn: async () => {
asserts.assert(item_collection);
asserts.assert(time_mid);
asserts.assert(time_end);
@ -106,10 +161,10 @@ Deno.test({
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_before_offset,
before: time_mid
filter: (entry) => (entry.info.birthtime?.toISOString() ?? '') < time_mid
});
fetched_for_before.push(...fetched_items);
fetched_for_before.push(...(fetched_items.map((item) => item.load())));
fetch_for_before_offset += fetched_items.length;
more_to_fetch_for_before = fetched_items.length === limit;
} while (more_to_fetch_for_before);
@ -128,35 +183,6 @@ Deno.test({
// note: we use less or equal because we don't have the actual file write time
asserts.assertLessOrEqual(newest, time_mid);
// test id_before
let fetch_for_id_before_offset = 0;
const fetched_for_id_before = [];
let more_to_fetch_for_id_before = true;
do {
// fuzz the limit
const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN;
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_id_before_offset,
id_before: mid_id
});
fetched_for_id_before.push(...fetched_items);
fetch_for_id_before_offset += fetched_items.length;
more_to_fetch_for_id_before = fetched_items.length === limit;
} while (more_to_fetch_for_id_before);
let newest_id = first_id;
asserts.assert(newest_id);
for (const item of fetched_for_id_before) {
if (item.id > newest_id) {
newest_id = item.id;
}
}
asserts.assertLess(newest_id, mid_id);
// test after
let fetch_for_after_offset = 0;
const fetched_for_after = [];
@ -168,10 +194,10 @@ Deno.test({
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_after_offset,
after: time_mid
filter: (entry) => (entry.info.birthtime?.toISOString() ?? '') > time_mid
});
fetched_for_after.push(...fetched_items);
fetched_for_after.push(...(fetched_items.map((item) => item.load())));
fetch_for_after_offset += fetched_items.length;
more_to_fetch_for_after = fetched_items.length === limit;
} while (more_to_fetch_for_after);
@ -189,6 +215,54 @@ Deno.test({
// again with the file write time slop
asserts.assertGreaterOrEqual(oldest, time_mid);
}
});
Deno.test({
name: 'filter all() by id',
permissions: {
env: true,
// https://github.com/denoland/deno/discussions/17258
read: true,
write: true
},
fn: async () => {
asserts.assert(item_collection);
asserts.assert(time_mid);
asserts.assert(time_end);
asserts.assert(mid_id);
asserts.assert(last_id);
// test id_before
let fetch_for_id_before_offset = 0;
const fetched_for_id_before = [];
let more_to_fetch_for_id_before = true;
do {
// fuzz the limit
const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN;
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_id_before_offset,
filter: (entry) => path.basename(entry.path).replace(/\.json$/i, '') < mid_id
});
fetched_for_id_before.push(...(fetched_items.map((item) => item.load())));
fetch_for_id_before_offset += fetched_items.length;
more_to_fetch_for_id_before = fetched_items.length === limit;
} while (more_to_fetch_for_id_before);
let newest_id = first_id;
asserts.assert(newest_id);
for (const item of fetched_for_id_before) {
if (item.id > newest_id) {
newest_id = item.id;
}
}
asserts.assertLess(newest_id, mid_id);
// test id_after
let fetch_for_id_after_offset = 0;
@ -201,10 +275,10 @@ Deno.test({
const fetched_items = await item_collection.all({
limit,
offset: fetch_for_id_after_offset,
id_after: mid_id
filter: (entry) => path.basename(entry.path).replace(/\.json$/i, '') > mid_id
});
fetched_for_id_after.push(...fetched_items);
fetched_for_id_after.push(...(fetched_items.map((item) => item.load())));
fetch_for_id_after_offset += fetched_items.length;
more_to_fetch_for_id_after = fetched_items.length === limit;
} while (more_to_fetch_for_id_after);