andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: make all() take filter and sort options

Browse source 
refactor: return unload item entries from all()/find()
This commit is contained in:
Andy Burke 2025-07-02 17:46:04 -07:00
parent 3214d17b80
commit 05178c924f
8 changed files with 308 additions and 206 deletions
Download patch file Download diff file Expand all files Collapse all files

133
fsdb.ts
View file

@ -2,6 +2,9 @@ import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';
import { walk, WALK_ENTRY } from './utils/walk.ts';
export type { WALK_ENTRY };
export type FSDB_COLLECTION_CONFIG = {
name: string;
@ -12,22 +15,18 @@ export type FSDB_COLLECTION_CONFIG = {
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<FSDB_COLLECTION_CONFIG, 'id_field' | 'organize' | 'root'>;
export type FSDB_SEARCH_OPTIONS = {
limit: number;
export type FSDB_SEARCH_OPTIONS<T> = {
limit?: number;
offset?: number;
before?: string;
after?: string;
modified_before?: string;
modified_after?: string;
id_before?: string;
id_after?: string;
filter?: (entry: WALK_ENTRY<T>) => boolean;
sort?: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>) => number;
};
export interface FSDB_INDEXER<T> {
set_fsdb_root(root: string): void;
index(item: T, authoritative_path: string): Promise<string[]>;
remove(item: T, authoritative_path: string): Promise<string[]>;
lookup(value: string, options?: FSDB_SEARCH_OPTIONS): Promise<string[]>;
lookup(value: string, options?: FSDB_SEARCH_OPTIONS<T>): Promise<string[]>;
}
/** Represents a collection of like items within the database on disk. */
@ -243,21 +242,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
/** Iterate through the items. */
async all(input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
async all({
limit = 100,
offset = 0,
filter = undefined,
sort = undefined
}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
const options: FSDB_SEARCH_OPTIONS = {
...{
limit: 100,
offset: 0
},
...(input_options ?? {})
};
const results: WALK_ENTRY<T>[] = [];
const results: T[] = [];
const limit = options?.limit ?? 100;
const offset = options?.offset ?? 0;
let counter = 0;
// TODO: better way to get a pattern to match files in this collection?
@ -270,65 +264,33 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
for await (
const entry of fs.walk(this.config.root, {
includeDirs: false,
includeSymlinks: false,
skip: [/\.fsdb\.collection\.json$/],
exts: ['json']
const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== '.json') {
return false;
}
if (entry.info.isDirectory || entry.info.isSymlink) {
return false;
}
const filename = path.basename(entry.path);
if (filename === '.fsdb.collection.json') {
return false;
}
return filter ? filter(entry) : true;
},
sort
})
) {
let item_stat = null;
if (options.before) {
item_stat = item_stat ?? await Deno.lstat(entry.path);
const birthtime = (item_stat.birthtime ?? new Date(0)).toISOString();
if (birthtime > options.before) {
continue;
}
}
if (options.after) {
item_stat = item_stat ?? await Deno.lstat(entry.path);
if ((item_stat.birthtime ?? new Date(0)).toISOString() < options.after) {
continue;
}
}
if (options.modified_before) {
item_stat = item_stat ?? await Deno.lstat(entry.path);
if ((item_stat.mtime ?? new Date(0)).toISOString() > options.modified_before) {
continue;
}
}
if (options.modified_after) {
item_stat = item_stat ?? await Deno.lstat(entry.path);
if ((item_stat.mtime ?? new Date(0)).toISOString() < options.modified_after) {
continue;
}
}
let item_id = null;
if (options.id_before) {
item_id = item_id ?? entry.name.replace(/\.json$/, '');
if (item_id >= options.id_before) {
continue;
}
}
if (options.id_after) {
item_id = item_id ?? entry.name.replace(/\.json$/, '');
if (item_id <= options.id_after) {
continue;
}
}
if (counter < offset) {
++counter;
continue;
}
const content = await Deno.readTextFile(entry.path);
results.push(JSON.parse(content));
results.push(entry);
++counter;
if (counter >= (offset + limit)) {
@ -340,7 +302,12 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));
this.emit('all', {
options,
options: {
limit,
offset,
filter,
sort
},
results
});
@ -348,10 +315,10 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
/** Use indexes to search for matching items. */
async find(criteria: Record<string, any>, input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
async find(criteria: Record<string, any>, input_options?: FSDB_SEARCH_OPTIONS<T>): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');
const options: FSDB_SEARCH_OPTIONS = {
const options: FSDB_SEARCH_OPTIONS<T> = {
...{
limit: 100,
offset: 0
@ -359,7 +326,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
...(input_options ?? {})
};
const results: T[] = [];
const results: WALK_ENTRY<T>[] = [];
const item_paths: string[] = [];
for (const search_key of Object.keys(criteria)) {
@ -380,8 +347,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
continue;
}
const content = await Deno.readTextFile(item_path);
results.push(JSON.parse(content));
const info: Deno.FileInfo = await Deno.lstat(item_path);
results.push({
path: item_path,
info,
depth: -1,
load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T;
}
});
++counter;
if (counter >= (offset + limit)) {