andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: andyburke:77765e9e60d6f84f1bed902367efb661a16fb9df
Branches Tags
andyburke:dev
..
compare: andyburke:2c77227fcadd5dbeb894af9d17a97d47d180bfd5
Branches Tags
andyburke:dev

No commits in common. "77765e9e60d6f84f1bed902367efb661a16fb9df" and "2c77227fcadd5dbeb894af9d17a97d47d180bfd5" have entirely different histories.
77765e9e60 ... 2c77227fca

4 changed files with 112 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

23
.zed/settings.json
View file

@ -1,23 +0,0 @@
{
"lsp": {
"deno": {
"settings": {
"deno": {
"enable": true
}
}
}
},
"languages": {
"TypeScript": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
},
"TSX": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
},
"JavaScript": {
"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
}
},
"formatter": "language_server"
}

20
deno.json
View file

@ -1,6 +1,6 @@
{
"name": "@andyburke/fsdb",
"version": "1.2.0",
"version": "1.1.0",
"license": "MIT",
"exports": {
".": "./fsdb.ts",
@ -8,16 +8,16 @@
"./indexers": "./indexers.ts",
"./organizers": "./organizers.ts"
},
"tasks": {
"lint": "deno lint",
"fmt": "deno fmt",
"test": "cd tests && DENO_ENV=test FSDB_TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./",
"fsdb": "deno run --allow-env --allow-read --allow-write cli.ts"
},
"fmt": {
"include": [
"**/*.ts"
],
"include": ["**/*.ts"],
"options": {
"useTabs": true,
"lineWidth": 140,
@ -28,16 +28,10 @@
}
},
"lint": {
"include": [
"**/*.ts"
],
"include": ["**/*.ts"],
"rules": {
"tags": [
"recommended"
],
"exclude": [
"no-explicit-any"
]
"tags": ["recommended"],
"exclude": ["no-explicit-any"]
}
},
"imports": {

207
fsdb.ts
View file

@ -53,11 +53,11 @@
* @module
*/
import * as fs from '@std/fs';
import * as path from '@std/path';
import by_lurid from './organizers/by_lurid.ts';
import { Optional } from './utils/optional.ts';
import { walk, WALK_ENTRY } from './utils/walk.ts';
import * as fs from "@std/fs";
import * as path from "@std/path";
import by_lurid from "./organizers/by_lurid.ts";
import { Optional } from "./utils/optional.ts";
import { walk, WALK_ENTRY } from "./utils/walk.ts";
export type { WALK_ENTRY };
@ -70,7 +70,7 @@ export type FSDB_COLLECTION_CONFIG = {
};
export type FSDB_COLLECTION_CONFIG_INPUT = Optional<
FSDB_COLLECTION_CONFIG,
'id_field' | 'organize' | 'root'
"id_field" | "organize" | "root"
>;
export type FSDB_SEARCH_OPTIONS<T> = {
@ -98,11 +98,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
this.config = {
...{
id_field: 'id',
id_field: "id",
organize: by_lurid,
root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`,
},
...(input_config ?? {})
...(input_config ?? {}),
};
this.event_listeners = {};
@ -115,7 +115,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
let existing_collection_info: any = undefined;
try {
const existing_collection_info_content: string = Deno.readTextFileSync(
path.resolve(path.join(this.config.root), '.fsdb.collection.json')
path.resolve(path.join(this.config.root), ".fsdb.collection.json"),
);
existing_collection_info = JSON.parse(existing_collection_info_content);
} catch (error) {
@ -127,42 +127,42 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (existing_collection_info) {
if (this.config.name !== existing_collection_info.name) {
console.warn(
'Mismatching collection name, maybe the collection was renamed? Be cautious.'
"Mismatching collection name, maybe the collection was renamed? Be cautious.",
);
}
if (this.config.root !== existing_collection_info.root) {
console.warn(
'Mismatching collection root, maybe the collection was moved on disk? Be cautious.'
"Mismatching collection root, maybe the collection was moved on disk? Be cautious.",
);
}
if (this.config.id_field !== existing_collection_info.id_field) {
console.warn(
'Mismatching collection id field, maybe the data format has changed? Be cautious.'
"Mismatching collection id field, maybe the data format has changed? Be cautious.",
);
}
if (
Object.keys(this.config.indexers ?? {})
.sort()
.join('|') !==
Object.keys(existing_collection_info.indexers ?? {})
.sort()
.join('|')
.join("|") !==
Object.keys(existing_collection_info.indexers ?? {})
.sort()
.join("|")
) {
console.warn(
'Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.'
"Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.",
);
}
}
const collection_info_file_path: string = path.resolve(
path.join(this.config.root, '.fsdb.collection.json')
path.join(this.config.root, ".fsdb.collection.json"),
);
const collection_info_json: string = JSON.stringify(this.config, null, '\t');
const collection_info_json: string = JSON.stringify(this.config, null, "\t");
Deno.mkdirSync(path.dirname(collection_info_file_path), {
recursive: true
recursive: true,
});
Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
}
@ -190,7 +190,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @returns {string} An organized, resolved path for the id.
*/
public get_organized_id_path(id: string): string {
return this.get_organized_item_path({ id }, 'id');
return this.get_organized_item_path({ id }, "id");
}
/**
@ -209,21 +209,22 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
private async write_item(item: T, override_path?: string): Promise<void> {
const item_path: string = override_path ?? (await this.ensure_item_path(item, this.config.id_field));
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));
const item_path: string =
override_path ?? (await this.ensure_item_path(item, this.config.id_field));
Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t"));
this.emit('write', {
this.emit("write", {
item,
item_path
item_path,
});
if (this.config.indexers) {
for (const indexer of Object.values(this.config.indexers)) {
await (indexer as FSDB_INDEXER<T>).index(item, item_path);
this.emit('index', {
this.emit("index", {
item,
item_path,
indexer
indexer,
});
}
}
@ -247,9 +248,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const content: string = await Deno.readTextFile(item_path);
const item: T = JSON.parse(content);
this.emit('get', {
this.emit("get", {
item,
item_path
item_path,
});
return item;
@ -267,16 +268,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const item_exists: boolean = await fs.exists(item_path);
if (item_exists) {
throw new Error('item already exists', {
cause: 'item_exists'
throw new Error("item already exists", {
cause: "item_exists",
});
}
await this.write_item(item);
this.emit('create', {
this.emit("create", {
item,
item_path
item_path,
});
return item;
@ -295,17 +296,17 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
const previous: T | null = await this.get(id);
if (!previous) {
throw new Error('item does not exist', {
cause: 'item_does_not_exist'
throw new Error("item does not exist", {
cause: "item_does_not_exist",
});
}
await this.write_item(item, item_path);
this.emit('update', {
this.emit("update", {
item,
previous,
item_path
item_path,
});
return item;
@ -344,7 +345,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
if (has_files) {
dir = '';
dir = "";
break;
}
@ -352,8 +353,8 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
dir = path.dirname(dir);
} while (dir.length);
this.emit('delete', {
item
this.emit("delete", {
item,
});
return item;
@ -370,9 +371,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
limit = 100,
offset = 0,
filter = undefined,
sort = undefined
sort = undefined,
}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin");
const results: WALK_ENTRY<T>[] = [];
@ -387,28 +388,26 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
return results;
}
for await (
const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== '.json') {
return false;
}
for await (const entry of walk(this.config.root, {
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== ".json") {
return false;
}
if (entry.info.isDirectory || entry.info.isSymlink) {
return false;
}
if (entry.info.isDirectory || entry.info.isSymlink) {
return false;
}
const filename = path.basename(entry.path);
if (filename === '.fsdb.collection.json') {
return false;
}
const filename = path.basename(entry.path);
if (filename === ".fsdb.collection.json") {
return false;
}
return filter ? filter(entry) : true;
},
sort
})
) {
return filter ? filter(entry) : true;
},
sort,
})) {
if (counter < offset) {
++counter;
continue;
@ -422,21 +421,20 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
if (Deno.env.get('FSDB_PERF')) {
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(
performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')
performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"),
);
}
this.emit('all', {
this.emit("all", {
options: {
limit,
offset,
filter,
sort
sort,
},
results
results,
});
return results;
@ -451,16 +449,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
*/
async find(
criteria: Record<string, any>,
options?: FSDB_SEARCH_OPTIONS<T>
options?: FSDB_SEARCH_OPTIONS<T>,
): Promise<WALK_ENTRY<T>[]> {
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin");
const find_options: FSDB_SEARCH_OPTIONS<T> = {
...{
limit: 100,
offset: 0
offset: 0,
},
...(options ?? {})
...(options ?? {}),
};
const results: WALK_ENTRY<T>[] = [];
@ -474,14 +472,15 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
if (indexer_for_search_key) {
const matched_items = await indexer_for_search_key.lookup(value, find_options);
for (const matched_item_path of matched_items) {
item_path_criteria_match_count[matched_item_path] = item_path_criteria_match_count[matched_item_path] ?? 0;
item_path_criteria_match_count[matched_item_path] =
item_path_criteria_match_count[matched_item_path] ?? 0;
item_path_criteria_match_count[matched_item_path]++;
}
}
}
const matching_items = Object.keys(item_path_criteria_match_count).filter(
(item_path) => item_path_criteria_match_count[item_path] === score_needed
(item_path) => item_path_criteria_match_count[item_path] === score_needed,
);
const limit = find_options.limit ?? 100;
@ -501,7 +500,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
depth: -1,
load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T;
}
},
});
++counter;
@ -511,15 +510,14 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
}
}
if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
if (Deno.env.get('FSDB_PERF')) {
console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));
}
if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end");
if (Deno.env.get("FSDB_PERF"))
console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end"));
this.emit('find', {
this.emit("find", {
criteria,
options: find_options,
results
results,
});
return results;
@ -532,18 +530,19 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler for the event.
*/
public on(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []);
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (!listeners.includes(handler)) {
listeners.push(handler);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
on: {
event,
handler
handler,
},
listeners
listeners,
});
}
}
@ -555,32 +554,34 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
* @param {(event_data: any) => void} handler The handler that was registered that should be removed.
*/
public off(event: string, handler: (event_data: any) => void) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []);
const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
this.event_listeners[event] ?? []);
if (listeners.includes(handler)) {
listeners.splice(listeners.indexOf(handler), 1);
}
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
off: {
event: event,
handler
handler,
},
listeners
listeners,
});
}
}
private emit(event_name: string, event_data: any) {
const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] = this.event_listeners[event_name] ?? []);
const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] =
this.event_listeners[event_name] ?? []);
if (Deno.env.get('FSDB_LOG_EVENTS')) {
if (Deno.env.get("FSDB_LOG_EVENTS")) {
console.dir({
emitting: {
event_name,
event_data,
listeners
}
listeners,
},
});
}
@ -591,23 +592,23 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
public sorts = {
newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? ''
((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "",
),
oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? ''
((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "",
),
latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
(a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''
((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
(a.info.mtime ?? a.info.ctime)?.toISOString() ?? "",
),
stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
(b.info.mtime ?? b.info.ctime)?.toISOString() ?? ''
)
((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
(b.info.mtime ?? b.info.ctime)?.toISOString() ?? "",
),
};
}

7
indexers/symlinks.ts
View file

@ -14,7 +14,6 @@ interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
id_field?: string;
to_many?: boolean;
organize?: (value: string) => string[];
organize_id?: (value: string) => string[];
}
interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
@ -192,11 +191,7 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
continue;
}
if (this.config.organize_id) {
organized_paths.push(...this.config.organize_id(item_id));
} else {
organized_paths.push(`${item_id}.json`);
}
organized_paths.push(`${item_id}.json`);
}
const symlink_path = path.resolve(path.join(this.config.root, ...organized_paths));