andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: ensure find() matches all criteria

Browse source
This commit is contained in:
Andy Burke 2025-10-21 21:52:06 -07:00
parent 54d284e597
commit 2c77227fca
3 changed files with 221 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

237
fsdb.ts
View file

@ -53,11 +53,11 @@
* @module
*/
import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';
import { walk, WALK_ENTRY } from './utils/walk.ts';
import * as fs from "@std/fs";
import * as path from "@std/path";
import by_lurid from "./organizers/by_lurid.ts";
import { Optional } from "./utils/optional.ts";
import { walk, WALK_ENTRY } from "./utils/walk.ts";
export type { WALK_ENTRY };
@ -68,7 +68,10 @@ export type FSDB_COLLECTION_CONFIG = {
organize: (id: string) => string[];
root: string;
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<FSDB_COLLECTION_CONFIG, 'id_field' | 'organize' | 'root'>;
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<
FSDB_COLLECTION_CONFIG,
"id_field" | "organize" | "root"
>;
export type FSDB_SEARCH_OPTIONS<T> = {
limit?: number;
@ -95,11 +98,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
this.config = {
...{
id_field: 'id',
id_field: "id",
organize: by_lurid,
root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`,
},
...(input_config ?? {})
...(input_config ?? {}),
};
this.event_listeners = {};
@ -112,7 +115,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
let existing_collection_info: any = undefined;
try {
const existing_collection_info_content: string = Deno.readTextFileSync(
path.resolve(path.join(this.config.root), '.fsdb.collection.json')
path.resolve(path.join(this.config.root), ".fsdb.collection.json"),
);
existing_collection_info = JSON.parse(existing_collection_info_content);
} catch (error) {
@ -123,29 +126,43 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (existing_collection_info) {
if (this.config.name !== existing_collection_info.name) {
console.warn('Mismatching collection name, maybe the collection was renamed? Be cautious.');
console.warn(
"Mismatching collection name, maybe the collection was renamed? Be cautious.",
);
}
if (this.config.root !== existing_collection_info.root) {
console.warn('Mismatching collection root, maybe the collection was moved on disk? Be cautious.');
console.warn(
"Mismatching collection root, maybe the collection was moved on disk? Be cautious.",
);
}
if (this.config.id_field !== existing_collection_info.id_field) {
console.warn('Mismatching collection id field, maybe the data format has changed? Be cautious.');
console.warn(
"Mismatching collection id field, maybe the data format has changed? Be cautious.",
);
}
if (
Object.keys(this.config.indexers ?? {}).sort().join('|') !==
Object.keys(existing_collection_info.indexers ?? {}).sort().join('|')
Object.keys(this.config.indexers ?? {})
.sort()
.join("|") !==
Object.keys(existing_collection_info.indexers ?? {})
.sort()
.join("|")
) {
console.warn('Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.');
console.warn(
"Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.",
);
}
}
const collection_info_file_path: string = path.resolve(path.join(this.config.root, '.fsdb.collection.json'));
const collection_info_json: string = JSON.stringify(this.config, null, '\t');
const collection_info_file_path: string = path.resolve(
path.join(this.config.root, ".fsdb.collection.json"),
);
const collection_info_json: string = JSON.stringify(this.config, null, "\t");
Deno.mkdirSync(path.dirname(collection_info_file_path), {
recursive: true
recursive: true,
});
Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
}
@ -173,7 +190,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @returns {string} An organized, resolved path for the id.
*/
public get_organized_id_path(id: string): string {
return this.get_organized_item_path({ id }, 'id');
return this.get_organized_item_path({ id }, "id");
}
/**
@ -192,21 +209,22 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
private async write_item(item: T, override_path?: string): Promise<void> {
const item_path: string = override_path ?? await this.ensure_item_path(item, this.config.id_field);
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));
const item_path: string =
override_path ?? (await this.ensure_item_path(item, this.config.id_field));
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t"));
this.emit('write', {
this.emit("write", {
item,
item_path
item_path,
});
if (this.config.indexers) {
for (const indexer of Object.values(this.config.indexers)) {
await (indexer as FSDB_INDEXER<T>).index(item, item_path);
this.emit('index', {
this.emit("index", {
item,
item_path,
indexer
indexer,
});
}
}
@ -230,9 +248,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const content: string = await Deno.readTextFile(item_path);
const item: T = JSON.parse(content);
this.emit('get', {
this.emit("get", {
item,
item_path
item_path,
});
return item;
@ -250,16 +268,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const item_exists: boolean = await fs.exists(item_path);
if (item_exists) {
throw new Error('item already exists', {
cause: 'item_exists'
throw new Error("item already exists", {
cause: "item_exists",
});
}
await this.write_item(item);
this.emit('create', {
this.emit("create", {
item,
item_path
item_path,
});
return item;
@ -278,17 +296,17 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const previous: T | null = await this.get(id);
if (!previous) {
throw new Error('item does not exist', {
cause: 'item_does_not_exist'
throw new Error("item does not exist", {
cause: "item_does_not_exist",
});
}
await this.write_item(item, item_path);
this.emit('update', {
this.emit("update", {
item,
previous,
item_path
item_path,
});
return item;
@ -327,7 +345,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
if (has_files) {
dir = '';
dir = "";
break;
}
@ -335,8 +353,8 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
dir = path.dirname(dir);
} while (dir.length);
this.emit('delete', {
item
this.emit("delete", {
item,
});
return item;
@ -353,9 +371,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
limit = 100,
offset = 0,
filter = undefined,
sort = undefined
sort = undefined,
}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin");
const results: WALK_ENTRY<T>[] = [];
@ -370,28 +388,26 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
return results;
}
for await (
const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== '.json') {
return false;
}
for await (const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== ".json") {
return false;
}
if (entry.info.isDirectory || entry.info.isSymlink) {
return false;
}
if (entry.info.isDirectory || entry.info.isSymlink) {
return false;
}
const filename = path.basename(entry.path);
if (filename === '.fsdb.collection.json') {
return false;
}
const filename = path.basename(entry.path);
if (filename === ".fsdb.collection.json") {
return false;
}
return filter ? filter(entry) : true;
},
sort
})
) {
return filter ? filter(entry) : true;
},
sort,
})) {
if (counter < offset) {
++counter;
continue;
@ -400,22 +416,25 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
results.push(entry);
++counter;
if (counter >= (offset + limit)) {
if (counter >= offset + limit) {
break;
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(
performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"),
);
this.emit('all', {
this.emit("all", {
options: {
limit,
offset,
filter,
sort
sort,
},
results
results,
});
return results;
@ -428,33 +447,47 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {FSDB_SEARCH_OPTIONS<T>} options
* @returns {Promise<WALK_ENTRY<T>[]>} Returns an array of `WALK_ENTRY` items for items given the input options.
*/
async find(criteria: Record<string, any>, options?: FSDB_SEARCH_OPTIONS<T>): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');
async find(
criteria: Record<string, any>,
options?: FSDB_SEARCH_OPTIONS<T>,
): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin");
const find_options: FSDB_SEARCH_OPTIONS<T> = {
...{
limit: 100,
offset: 0
offset: 0,
},
...(options ?? {})
...(options ?? {}),
};
const results: WALK_ENTRY<T>[] = [];
const item_paths: string[] = [];
const item_path_criteria_match_count: Record<string, number> = {};
const criteria_keys: string[] = Object.keys(criteria);
const score_needed = criteria_keys.length;
for (const search_key of Object.keys(criteria)) {
for (const search_key of criteria_keys) {
const indexer_for_search_key: FSDB_INDEXER<T> | undefined = this.INDEX[search_key];
const value: string = criteria[search_key];
if (indexer_for_search_key) {
item_paths.push(...await indexer_for_search_key.lookup(value, find_options));
const matched_items = await indexer_for_search_key.lookup(value, find_options);
for (const matched_item_path of matched_items) {
item_path_criteria_match_count[matched_item_path] =
item_path_criteria_match_count[matched_item_path] ?? 0;
item_path_criteria_match_count[matched_item_path]++;
}
}
}
const matching_items = Object.keys(item_path_criteria_match_count).filter(
(item_path) => item_path_criteria_match_count[item_path] === score_needed,
);
const limit = find_options.limit ?? 100;
const offset = find_options.offset ?? 0;
let counter = 0;
for await (const item_path of item_paths) {
for await (const item_path of matching_items) {
if (counter < offset) {
++counter;
continue;
@ -467,23 +500,24 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
depth: -1,
load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T;
}
},
});
++counter;
if (counter >= (offset + limit)) {
if (counter >= offset + limit) {
break;
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end"));
this.emit('find', {
this.emit("find", {
criteria,
options: find_options,
results
results,
});
return results;
@ -496,18 +530,19 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler for the event.
*/
public on(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (!listeners.includes(handler)) {
listeners.push(handler);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
on: {
event,
handler
handler,
},
listeners
listeners,
});
}
}
@ -519,32 +554,34 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler that was registered that should be removed.
*/
public off(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = this.event_listeners[event] = this.event_listeners[event] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (listeners.includes(handler)) {
listeners.splice(listeners.indexOf(handler), 1);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
off: {
event: event,
handler
handler,
},
listeners
listeners,
});
}
}
private emit(event_name: string, event_data: any) {
const listeners: ((event: any) => void)[] = this.event_listeners[event_name] = this.event_listeners[event_name] ?? [];
const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] =
this.event_listeners[event_name] ?? []);
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
emitting: {
event_name,
event_data,
listeners
}
listeners,
},
});
}
@ -555,19 +592,23 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
public sorts = {
newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? ''
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "",
),
oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? ''
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "",
),
latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare((a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''),
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.mtime ?? a.info.ctime)?.toISOString() ?? "",
),
stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '')
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.mtime ?? b.info.ctime)?.toISOString() ?? "",
),
};
}