andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: ensure find() matches all criteria

Browse source
This commit is contained in:
Andy Burke 2025-10-21 21:52:06 -07:00
parent 54d284e597
commit 2c77227fca
3 changed files with 221 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

2
deno.json
View file

@ -1,6 +1,6 @@
{
"name": "@andyburke/fsdb",
"version": "1.0.4",
"version": "1.1.0",
"license": "MIT",
"exports": {
".": "./fsdb.ts",

213
fsdb.ts
View file

@ -53,11 +53,11 @@
* @module
*/
import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';
import { walk, WALK_ENTRY } from './utils/walk.ts';
import * as fs from "@std/fs";
import * as path from "@std/path";
import by_lurid from "./organizers/by_lurid.ts";
import { Optional } from "./utils/optional.ts";
import { walk, WALK_ENTRY } from "./utils/walk.ts";
export type { WALK_ENTRY };
@ -68,7 +68,10 @@ export type FSDB_COLLECTION_CONFIG = {
organize: (id: string) => string[];
root: string;
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<FSDB_COLLECTION_CONFIG, 'id_field' | 'organize' | 'root'>;
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<
FSDB_COLLECTION_CONFIG,
"id_field" | "organize" | "root"
>;
export type FSDB_SEARCH_OPTIONS<T> = {
limit?: number;
@ -95,11 +98,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
this.config = {
...{
id_field: 'id',
id_field: "id",
organize: by_lurid,
root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`,
},
...(input_config ?? {})
...(input_config ?? {}),
};
this.event_listeners = {};
@ -112,7 +115,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
let existing_collection_info: any = undefined;
try {
const existing_collection_info_content: string = Deno.readTextFileSync(
path.resolve(path.join(this.config.root), '.fsdb.collection.json')
path.resolve(path.join(this.config.root), ".fsdb.collection.json"),
);
existing_collection_info = JSON.parse(existing_collection_info_content);
} catch (error) {
@ -123,29 +126,43 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (existing_collection_info) {
if (this.config.name !== existing_collection_info.name) {
console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.');
console.warn(
"Mismatching collection name, maybe the collection was renamed? Be cautious.",
);
}
if (this.config.root !== existing_collection_info.root) {
console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.');
console.warn(
"Mismatching collection root, maybe the collection was moved on disk? Be cautious.",
);
}
if (this.config.id_field !== existing_collection_info.id_field) {
console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.');
console.warn(
"Mismatching collection id field, maybe the data format has changed? Be cautious.",
);
}
if (
Object.keys(this.config.indexers ?? {}).sort().join('|') !==
Object.keys(existing_collection_info.indexers ?? {}).sort().join('|')
Object.keys(this.config.indexers ?? {})
.sort()
.join("|") !==
Object.keys(existing_collection_info.indexers ?? {})
.sort()
.join("|")
) {
console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.');
console.warn(
"Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.",
);
}
}
const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json'));
const collection_info_json: string = JSON.stringify(this.config, null, '\t');
const collection_info_file_path: string = path.resolve(
path.join(this.config.root, ".fsdb.collection.json"),
);
const collection_info_json: string = JSON.stringify(this.config, null, "\t");
Deno.mkdirSync(path.dirname(collection_info_file_path), {
recursive: true
recursive: true,
});
Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
}
@ -173,7 +190,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @returns {string} An organized, resolved path for the id.
*/
public get_organized_id_path(id: string): string {
return this.get_organized_item_path({ id }, 'id');
return this.get_organized_item_path({ id }, "id");
}
/**
@ -192,21 +209,22 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
private async write_item(item: T, override_path?: string): Promise<void> {
const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field);
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));
const item_path: string =
override_path ?? (await this.ensure_item_path(item, this.config.id_field));
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t"));
this.emit('write', {
this.emit("write", {
item,
item_path
item_path,
});
if (this.config.indexers) {
for (const indexer of Object.values(this.config.indexers)) {
await (indexer as FSDB_INDEXER<T>).index(item, item_path);
this.emit('index', {
this.emit("index", {
item,
item_path,
indexer
indexer,
});
}
}
@ -230,9 +248,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const content: string = await Deno.readTextFile(item_path);
const item: T = JSON.parse(content);
this.emit('get', {
this.emit("get", {
item,
item_path
item_path,
});
return item;
@ -250,16 +268,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const item_exists: boolean = await fs.exists(item_path);
if (item_exists) {
throw new Error('item already exists', {
cause: 'item_exists'
throw new Error("item already exists", {
cause: "item_exists",
});
}
await this.write_item(item);
this.emit('create', {
this.emit("create", {
item,
item_path
item_path,
});
return item;
@ -278,17 +296,17 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const previous: T | null = await this.get(id);
if (!previous) {
throw new Error('item does not exist', {
cause: 'item_does_not_exist'
throw new Error("item does not exist", {
cause: "item_does_not_exist",
});
}
await this.write_item(item, item_path);
this.emit('update', {
this.emit("update", {
item,
previous,
item_path
item_path,
});
return item;
@ -327,7 +345,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
if (has_files) {
dir = '';
dir = "";
break;
}
@ -335,8 +353,8 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
dir = path.dirname(dir);
} while (dir.length);
this.emit('delete', {
item
this.emit("delete", {
item,
});
return item;
@ -353,9 +371,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
limit = 100,
offset = 0,
filter = undefined,
sort = undefined
sort = undefined,
}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin");
const results: WALK_ENTRY<T>[] = [];
@ -370,11 +388,10 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
return results;
}
for await (
const entry of walk(this.config.root, {
for await (const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== '.json') {
if (extension.toLowerCase() !== ".json") {
return false;
}
@ -383,15 +400,14 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
const filename = path.basename(entry.path);
if (filename === '.fsdb.collection.json') {
if (filename === ".fsdb.collection.json") {
return false;
}
return filter ? filter(entry) : true;
},
sort
})
) {
sort,
})) {
if (counter < offset) {
++counter;
continue;
@ -400,22 +416,25 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
results.push(entry);
++counter;
if (counter >= (offset + limit)) {
if (counter >= offset + limit) {
break;
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(
performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"),
);
this.emit('all', {
this.emit("all", {
options: {
limit,
offset,
filter,
sort
sort,
},
results
results,
});
return results;
@ -428,33 +447,47 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {FSDB_SEARCH_OPTIONS<T>} options
* @returns {Promise<WALK_ENTRY<T>[]>} Returns an array of `WALK_ENTRY` items for items given the input options.
*/
async find(criteria: Record<string, any>, options?: FSDB_SEARCH_OPTIONS<T>): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');
async find(
criteria: Record<string, any>,
options?: FSDB_SEARCH_OPTIONS<T>,
): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin");
const find_options: FSDB_SEARCH_OPTIONS<T> = {
...{
limit: 100,
offset: 0
offset: 0,
},
...(options ?? {})
...(options ?? {}),
};
const results: WALK_ENTRY<T>[] = [];
const item_paths: string[] = [];
const item_path_criteria_match_count: Record<string, number> = {};
const criteria_keys: string[] = Object.keys(criteria);
const score_needed = criteria_keys.length;
for (const search_key of Object.keys(criteria)) {
for (const search_key of criteria_keys) {
const indexer_for_search_key: FSDB_INDEXER<T> | undefined = this.INDEX[search_key];
const value: string = criteria[search_key];
if (indexer_for_search_key) {
item_paths.push(...await indexer_for_search_key.lookup(value, find_options));
const matched_items = await indexer_for_search_key.lookup(value, find_options);
for (const matched_item_path of matched_items) {
item_path_criteria_match_count[matched_item_path] =
item_path_criteria_match_count[matched_item_path] ?? 0;
item_path_criteria_match_count[matched_item_path]++;
}
}
}
const matching_items = Object.keys(item_path_criteria_match_count).filter(
(item_path) => item_path_criteria_match_count[item_path] === score_needed,
);
const limit = find_options.limit ?? 100;
const offset = find_options.offset ?? 0;
let counter = 0;
for await (const item_path of item_paths) {
for await (const item_path of matching_items) {
if (counter < offset) {
++counter;
continue;
@ -467,23 +500,24 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
depth: -1,
load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T;
}
},
});
++counter;
if (counter >= (offset + limit)) {
if (counter >= offset + limit) {
break;
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end"));
this.emit('find', {
this.emit("find", {
criteria,
options: find_options,
results
results,
});
return results;
@ -496,18 +530,19 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler for the event.
*/
public on(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (!listeners.includes(handler)) {
listeners.push(handler);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
on: {
event,
handler
handler,
},
listeners
listeners,
});
}
}
@ -519,32 +554,34 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler that was registered that should be removed.
*/
public off(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (listeners.includes(handler)) {
listeners.splice(listeners.indexOf(handler), 1);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
off: {
event: event,
handler
handler,
},
listeners
listeners,
});
}
}
private emit(event_name: string, event_data: any) {
const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] =
this.event_listeners[event_name] ?? []);
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
emitting: {
event_name,
event_data,
listeners
}
listeners,
},
});
}
@ -555,19 +592,23 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
public sorts = {
newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? ''
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "",
),
oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? ''
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "",
),
latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare((a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''),
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.mtime ?? a.info.ctime)?.toISOString() ?? "",
),
stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '')
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.mtime ?? b.info.ctime)?.toISOString() ?? "",
),
};
}

118
tests/04_indexing_sanity_checks.test.ts
View file

@ -1,57 +1,66 @@
import * as asserts from '@std/assert';
import * as fsdb from '../fsdb.ts';
import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts';
import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts';
import lurid from '@andyburke/lurid';
import by_email from '../organizers/by_email.ts';
import by_character from '../organizers/by_character.ts';
import by_phone from '../organizers/by_phone.ts';
import { sentence } from 'jsr:@ndaidong/txtgen';
import * as asserts from "@std/assert";
import * as fsdb from "../fsdb.ts";
import { FSDB_INDEXER_SYMLINKS } from "../indexers.ts";
import { get_data_dir, random_email_address, random_phone_number } from "./helpers.ts";
import lurid from "@andyburke/lurid";
import by_email from "../organizers/by_email.ts";
import by_character from "../organizers/by_character.ts";
import by_phone from "../organizers/by_phone.ts";
import { sentence } from "jsr:@ndaidong/txtgen";
Deno.test({
name: 'index some items',
name: "index some items",
permissions: {
env: true,
// https://github.com/denoland/deno/discussions/17258
read: true,
write: true
write: true,
},
fn: async () => {
type ITEM = {
id: string;
email: string;
phone: string;
stable: string;
value: string;
};
const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
name: 'test-04-items',
root: get_data_dir() + '/test-04-items',
name: "test-04-items",
root: get_data_dir() + "/test-04-items",
indexers: {
email: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'email',
field: 'email',
organize: by_email
name: "email",
field: "email",
organize: by_email,
}),
phone: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'phone',
field: 'phone',
organize: by_phone
name: "phone",
field: "phone",
organize: by_phone,
}),
stable: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: "stable",
field: "stable",
to_many: true,
organize: by_character,
}),
by_character_test: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'by_character_test',
name: "by_character_test",
organize: by_character,
get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3),
to_many: true
get_values_to_index: (item: ITEM) =>
item.value.split(/\W/).filter((word) => word.length > 3),
to_many: true,
}),
by_possibly_undefined: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'by_possibly_undefined',
name: "by_possibly_undefined",
organize: by_character,
get_values_to_index: (item: ITEM) => item.email.indexOf('.com') > 0 ? [item.email] : [],
to_many: true
})
}
get_values_to_index: (item: ITEM) =>
item.email.indexOf(".com") > 0 ? [item.email] : [],
to_many: true,
}),
},
});
asserts.assert(item_collection);
@ -62,7 +71,8 @@ Deno.test({
id: lurid(),
email: random_email_address(),
phone: random_phone_number(),
value: sentence()
stable: "stable",
value: sentence(),
};
items.push(item);
@ -73,24 +83,58 @@ Deno.test({
}
for (const item of items) {
const fetched_by_email: ITEM[] = (await item_collection.find({ email: item.email })).map((entry) => entry.load());
const fetched_by_email: ITEM[] = (
await item_collection.find({ email: item.email })
).map((entry) => entry.load());
asserts.assertLess(fetched_by_email.length, items.length);
asserts.assertGreater(fetched_by_email.length, 0);
asserts.assert(fetched_by_email.find((email_item) => email_item.id === item.id));
const fetched_by_phone: ITEM[] = (await item_collection.find({ phone: item.phone })).map((entry) => entry.load());
const fetched_by_phone: ITEM[] = (
await item_collection.find({ phone: item.phone })
).map((entry) => entry.load());
asserts.assertLess(fetched_by_phone.length, items.length);
asserts.assertGreater(fetched_by_phone.length, 0);
asserts.assert(fetched_by_phone.find((phone_item) => phone_item.id === item.id));
const words_in_value: string[] = item.value.split(/\W/).filter((word) => word.length > 3);
const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = (await item_collection.find({ by_character_test: random_word_in_value })).map((
entry
) => entry.load());
const words_in_value: string[] = item.value
.split(/\W/)
.filter((word) => word.length > 3);
const random_word_in_value: string =
words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = (
await item_collection.find({ by_character_test: random_word_in_value })
).map((entry) => entry.load());
asserts.assertLess(fetched_by_word_in_value.length, items.length);
asserts.assertGreater(fetched_by_word_in_value.length, 0);
asserts.assert(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));
asserts.assert(
fetched_by_word_in_value.find(
(word_in_value_item) => word_in_value_item.id === item.id,
),
);
}
for (let i = 0; i < 10; ++i) {
const random_item = items[Math.floor(Math.random() * items.length)];
asserts.assert(random_item);
const criteria: Record<string, string> = {
stable: "stable",
};
if (Math.random() < 0.5) {
criteria.email = random_item.email;
} else {
criteria.phone = random_item.phone;
}
const found_entries = await item_collection.find(criteria);
asserts.assertEquals(found_entries.length, 1);
const found: ITEM = found_entries[0].load();
asserts.assert(found);
asserts.assertEquals(found, random_item);
}
// leave one item behind so the whole db for this test doesn't get cleaned up so I can hand-review it
@ -110,5 +154,5 @@ Deno.test({
// ) => entry.load());
// asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));
// }
}
},
});