andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: andyburke:77765e9e60d6f84f1bed902367efb661a16fb9df
Branches Tags
andyburke:dev
..
compare: andyburke:2c77227fcadd5dbeb894af9d17a97d47d180bfd5
Branches Tags
andyburke:dev

No commits in common. "77765e9e60d6f84f1bed902367efb661a16fb9df" and "2c77227fcadd5dbeb894af9d17a97d47d180bfd5" have entirely different histories.
77765e9e60 ... 2c77227fca

4 changed files with 112 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

23
.zed/settings.json
View file

@ -1,23 +0,0 @@
{
"lsp": {
"deno": {
"settings": {
"deno": {
"enable": true
}
}
}
},
"languages": {
"TypeScript": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
},
"TSX": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
},
"JavaScript": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
}
},
"formatter": "language_server"
}

20
deno.json
View file

@ -1,6 +1,6 @@
{ {
"name": "@andyburke/fsdb", "name": "@andyburke/fsdb",
"version": "1.2.0", "version": "1.1.0",
"license": "MIT", "license": "MIT",
"exports": { "exports": {
".": "./fsdb.ts", ".": "./fsdb.ts",
@ -8,16 +8,16 @@
"./indexers": "./indexers.ts", "./indexers": "./indexers.ts",
"./organizers": "./organizers.ts" "./organizers": "./organizers.ts"
}, },
"tasks": { "tasks": {
"lint": "deno lint", "lint": "deno lint",
"fmt": "deno fmt", "fmt": "deno fmt",
"test": "cd tests && DENO_ENV=test FSDB_TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./", "test": "cd tests && DENO_ENV=test FSDB_TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./",
"fsdb": "deno run --allow-env --allow-read --allow-write cli.ts" "fsdb": "deno run --allow-env --allow-read --allow-write cli.ts"
}, },
"fmt": { "fmt": {
"include": [ "include": ["**/*.ts"],
"**/*.ts"
],
"options": { "options": {
"useTabs": true, "useTabs": true,
"lineWidth": 140, "lineWidth": 140,
@ -28,16 +28,10 @@
} }
}, },
"lint": { "lint": {
"include": [ "include": ["**/*.ts"],
"**/*.ts"
],
"rules": { "rules": {
"tags": [ "tags": ["recommended"],
"recommended" "exclude": ["no-explicit-any"]
],
"exclude": [
"no-explicit-any"
]
} }
}, },
"imports": { "imports": {

207
fsdb.ts
View file

@ -53,11 +53,11 @@
* @module * @module
*/ */
import * as fs from '@std/fs'; import * as fs from "@std/fs";
import * as path from '@std/path'; import * as path from "@std/path";
import by_lurid from './organizers/by_lurid.ts'; import by_lurid from "./organizers/by_lurid.ts";
import { Optional } from './utils/optional.ts'; import { Optional } from "./utils/optional.ts";
import { walk, WALK_ENTRY } from './utils/walk.ts'; import { walk, WALK_ENTRY } from "./utils/walk.ts";
export type { WALK_ENTRY }; export type { WALK_ENTRY };
@ -70,7 +70,7 @@ export type FSDB_COLLECTION_CONFIG = {
}; };
export type FSDB_COLLECTION_CONFIG_INPUT = Optional< export type FSDB_COLLECTION_CONFIG_INPUT = Optional<
FSDB_COLLECTION_CONFIG, FSDB_COLLECTION_CONFIG,
'id_field' | 'organize' | 'root' "id_field" | "organize" | "root"
>; >;
export type FSDB_SEARCH_OPTIONS<T> = { export type FSDB_SEARCH_OPTIONS<T> = {
@ -98,11 +98,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) { constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
this.config = { this.config = {
...{ ...{
id_field: 'id', id_field: "id",
organize: by_lurid, organize: by_lurid,
root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}` root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`,
}, },
...(input_config ?? {}) ...(input_config ?? {}),
}; };
this.event_listeners = {}; this.event_listeners = {};
@ -115,7 +115,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
let existing_collection_info: any = undefined; let existing_collection_info: any = undefined;
try { try {
const existing_collection_info_content: string = Deno.readTextFileSync( const existing_collection_info_content: string = Deno.readTextFileSync(
path.resolve(path.join(this.config.root), '.fsdb.collection.json') path.resolve(path.join(this.config.root), ".fsdb.collection.json"),
); );
existing_collection_info = JSON.parse(existing_collection_info_content); existing_collection_info = JSON.parse(existing_collection_info_content);
} catch (error) { } catch (error) {
@ -127,42 +127,42 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (existing_collection_info) { if (existing_collection_info) {
if (this.config.name !== existing_collection_info.name) { if (this.config.name !== existing_collection_info.name) {
console.warn( console.warn(
'Mismatching collection name, maybe the collection was renamed? Be cautious.' "Mismatching collection name, maybe the collection was renamed? Be cautious.",
); );
} }
if (this.config.root !== existing_collection_info.root) { if (this.config.root !== existing_collection_info.root) {
console.warn( console.warn(
'Mismatching collection root, maybe the collection was moved on disk? Be cautious.' "Mismatching collection root, maybe the collection was moved on disk? Be cautious.",
); );
} }
if (this.config.id_field !== existing_collection_info.id_field) { if (this.config.id_field !== existing_collection_info.id_field) {
console.warn( console.warn(
'Mismatching collection id field, maybe the data format has changed? Be cautious.' "Mismatching collection id field, maybe the data format has changed? Be cautious.",
); );
} }
if ( if (
Object.keys(this.config.indexers ?? {}) Object.keys(this.config.indexers ?? {})
.sort() .sort()
.join('|') !== .join("|") !==
Object.keys(existing_collection_info.indexers ?? {}) Object.keys(existing_collection_info.indexers ?? {})
.sort() .sort()
.join('|') .join("|")
) { ) {
console.warn( console.warn(
'Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.' "Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.",
); );
} }
} }
const collection_info_file_path: string = path.resolve( const collection_info_file_path: string = path.resolve(
path.join(this.config.root, '.fsdb.collection.json') path.join(this.config.root, ".fsdb.collection.json"),
); );
const collection_info_json: string = JSON.stringify(this.config, null, '\t'); const collection_info_json: string = JSON.stringify(this.config, null, "\t");
Deno.mkdirSync(path.dirname(collection_info_file_path), { Deno.mkdirSync(path.dirname(collection_info_file_path), {
recursive: true recursive: true,
}); });
Deno.writeTextFileSync(collection_info_file_path, collection_info_json); Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
} }
@ -190,7 +190,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @returns {string} An organized, resolved path for the id. * @returns {string} An organized, resolved path for the id.
*/ */
public get_organized_id_path(id: string): string { public get_organized_id_path(id: string): string {
return this.get_organized_item_path({ id }, 'id'); return this.get_organized_item_path({ id }, "id");
} }
/** /**
@ -209,21 +209,22 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
} }
private async write_item(item: T, override_path?: string): Promise<void> { private async write_item(item: T, override_path?: string): Promise<void> {
const item_path: string = override_path ?? (await this.ensure_item_path(item, this.config.id_field)); const item_path: string =
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t')); override_path ?? (await this.ensure_item_path(item, this.config.id_field));
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t"));
this.emit('write', { this.emit("write", {
item, item,
item_path item_path,
}); });
if (this.config.indexers) { if (this.config.indexers) {
for (const indexer of Object.values(this.config.indexers)) { for (const indexer of Object.values(this.config.indexers)) {
await (indexer as FSDB_INDEXER<T>).index(item, item_path); await (indexer as FSDB_INDEXER<T>).index(item, item_path);
this.emit('index', { this.emit("index", {
item, item,
item_path, item_path,
indexer indexer,
}); });
} }
} }
@ -247,9 +248,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const content: string = await Deno.readTextFile(item_path); const content: string = await Deno.readTextFile(item_path);
const item: T = JSON.parse(content); const item: T = JSON.parse(content);
this.emit('get', { this.emit("get", {
item, item,
item_path item_path,
}); });
return item; return item;
@ -267,16 +268,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const item_exists: boolean = await fs.exists(item_path); const item_exists: boolean = await fs.exists(item_path);
if (item_exists) { if (item_exists) {
throw new Error('item already exists', { throw new Error("item already exists", {
cause: 'item_exists' cause: "item_exists",
}); });
} }
await this.write_item(item); await this.write_item(item);
this.emit('create', { this.emit("create", {
item, item,
item_path item_path,
}); });
return item; return item;
@ -295,17 +296,17 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const previous: T | null = await this.get(id); const previous: T | null = await this.get(id);
if (!previous) { if (!previous) {
throw new Error('item does not exist', { throw new Error("item does not exist", {
cause: 'item_does_not_exist' cause: "item_does_not_exist",
}); });
} }
await this.write_item(item, item_path); await this.write_item(item, item_path);
this.emit('update', { this.emit("update", {
item, item,
previous, previous,
item_path item_path,
}); });
return item; return item;
@ -344,7 +345,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
} }
if (has_files) { if (has_files) {
dir = ''; dir = "";
break; break;
} }
@ -352,8 +353,8 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
dir = path.dirname(dir); dir = path.dirname(dir);
} while (dir.length); } while (dir.length);
this.emit('delete', { this.emit("delete", {
item item,
}); });
return item; return item;
@ -370,9 +371,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
limit = 100, limit = 100,
offset = 0, offset = 0,
filter = undefined, filter = undefined,
sort = undefined sort = undefined,
}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> { }: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin'); if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin");
const results: WALK_ENTRY<T>[] = []; const results: WALK_ENTRY<T>[] = [];
@ -387,28 +388,26 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
return results; return results;
} }
for await ( for await (const entry of walk(this.config.root, {
const entry of walk(this.config.root, { filter: (entry: WALK_ENTRY<T>): boolean => {
filter: (entry: WALK_ENTRY<T>): boolean => { const extension = path.extname(entry.path);
const extension = path.extname(entry.path); if (extension.toLowerCase() !== ".json") {
if (extension.toLowerCase() !== '.json') { return false;
return false; }
}
if (entry.info.isDirectory || entry.info.isSymlink) { if (entry.info.isDirectory || entry.info.isSymlink) {
return false; return false;
} }
const filename = path.basename(entry.path); const filename = path.basename(entry.path);
if (filename === '.fsdb.collection.json') { if (filename === ".fsdb.collection.json") {
return false; return false;
} }
return filter ? filter(entry) : true; return filter ? filter(entry) : true;
}, },
sort sort,
}) })) {
) {
if (counter < offset) { if (counter < offset) {
++counter; ++counter;
continue; continue;
@ -422,21 +421,20 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
} }
} }
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end'); if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end");
if (Deno.env.get('FSDB_PERF')) { if (Deno.env.get("FSDB_PERF"))
console.dir( console.dir(
performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end') performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"),
); );
}
this.emit('all', { this.emit("all", {
options: { options: {
limit, limit,
offset, offset,
filter, filter,
sort sort,
}, },
results results,
}); });
return results; return results;
@ -451,16 +449,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
*/ */
async find( async find(
criteria: Record<string, any>, criteria: Record<string, any>,
options?: FSDB_SEARCH_OPTIONS<T> options?: FSDB_SEARCH_OPTIONS<T>,
): Promise<WALK_ENTRY<T>[]> { ): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin'); if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin");
const find_options: FSDB_SEARCH_OPTIONS<T> = { const find_options: FSDB_SEARCH_OPTIONS<T> = {
...{ ...{
limit: 100, limit: 100,
offset: 0 offset: 0,
}, },
...(options ?? {}) ...(options ?? {}),
}; };
const results: WALK_ENTRY<T>[] = []; const results: WALK_ENTRY<T>[] = [];
@ -474,14 +472,15 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (indexer_for_search_key) { if (indexer_for_search_key) {
const matched_items = await indexer_for_search_key.lookup(value, find_options); const matched_items = await indexer_for_search_key.lookup(value, find_options);
for (const matched_item_path of matched_items) { for (const matched_item_path of matched_items) {
item_path_criteria_match_count[matched_item_path] = item_path_criteria_match_count[matched_item_path] ?? 0; item_path_criteria_match_count[matched_item_path] =
item_path_criteria_match_count[matched_item_path] ?? 0;
item_path_criteria_match_count[matched_item_path]++; item_path_criteria_match_count[matched_item_path]++;
} }
} }
} }
const matching_items = Object.keys(item_path_criteria_match_count).filter( const matching_items = Object.keys(item_path_criteria_match_count).filter(
(item_path) => item_path_criteria_match_count[item_path] === score_needed (item_path) => item_path_criteria_match_count[item_path] === score_needed,
); );
const limit = find_options.limit ?? 100; const limit = find_options.limit ?? 100;
@ -501,7 +500,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
depth: -1, depth: -1,
load: function () { load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T; return JSON.parse(Deno.readTextFileSync(this.path)) as T;
} },
}); });
++counter; ++counter;
@ -511,15 +510,14 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
} }
} }
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end'); if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end");
if (Deno.env.get('FSDB_PERF')) { if (Deno.env.get("FSDB_PERF"))
console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end')); console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end"));
}
this.emit('find', { this.emit("find", {
criteria, criteria,
options: find_options, options: find_options,
results results,
}); });
return results; return results;
@ -532,18 +530,19 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler for the event. * @param {(event_data: any) => void} handler The handler for the event.
*/ */
public on(event: string, handler: (event_data: any) => void) { public on(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []); const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (!listeners.includes(handler)) { if (!listeners.includes(handler)) {
listeners.push(handler); listeners.push(handler);
} }
if (Deno.env.get('FSDB_LOG_EVENTS')) { if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({ console.dir({
on: { on: {
event, event,
handler handler,
}, },
listeners listeners,
}); });
} }
} }
@ -555,32 +554,34 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler that was registered that should be removed. * @param {(event_data: any) => void} handler The handler that was registered that should be removed.
*/ */
public off(event: string, handler: (event_data: any) => void) { public off(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []); const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (listeners.includes(handler)) { if (listeners.includes(handler)) {
listeners.splice(listeners.indexOf(handler), 1); listeners.splice(listeners.indexOf(handler), 1);
} }
if (Deno.env.get('FSDB_LOG_EVENTS')) { if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({ console.dir({
off: { off: {
event: event, event: event,
handler handler,
}, },
listeners listeners,
}); });
} }
} }
private emit(event_name: string, event_data: any) { private emit(event_name: string, event_data: any) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] = this.event_listeners[event_name] ?? []); const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] =
this.event_listeners[event_name] ?? []);
if (Deno.env.get('FSDB_LOG_EVENTS')) { if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({ console.dir({
emitting: { emitting: {
event_name, event_name,
event_data, event_data,
listeners listeners,
} },
}); });
} }
@ -591,23 +592,23 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
public sorts = { public sorts = {
newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number => newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare( ((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '' (a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "",
), ),
oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number => oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare( ((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '' (b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "",
), ),
latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number => latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare( ((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.mtime ?? a.info.ctime)?.toISOString() ?? '' (a.info.mtime ?? a.info.ctime)?.toISOString() ?? "",
), ),
stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number => stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare( ((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.mtime ?? b.info.ctime)?.toISOString() ?? '' (b.info.mtime ?? b.info.ctime)?.toISOString() ?? "",
) ),
}; };
} }

7
indexers/symlinks.ts
View file

@ -14,7 +14,6 @@ interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
id_field?: string; id_field?: string;
to_many?: boolean; to_many?: boolean;
organize?: (value: string) => string[]; organize?: (value: string) => string[];
organize_id?: (value: string) => string[];
} }
interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
@ -192,11 +191,7 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
continue; continue;
} }
if (this.config.organize_id) { organized_paths.push(`${item_id}.json`);
organized_paths.push(...this.config.organize_id(item_id));
} else {
organized_paths.push(`${item_id}.json`);
}
} }
const symlink_path = path.resolve(path.join(this.config.root, ...organized_paths)); const symlink_path = path.resolve(path.join(this.config.root, ...organized_paths));