feature: allow organizing by id within symlink indexes

.zed/settings.json

@@ -0,0 +1,23 @@
+{
+	"lsp": {
+		"deno": {
+			"settings": {
+				"deno": {
+					"enable": true
+				}
+			}
+		}
+	},
+	"languages": {
+		"TypeScript": {
+			"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
+		},
+		"TSX": {
+			"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
+		},
+		"JavaScript": {
+			"language_servers": ["deno", "!typescript-language-server", "!vtsls", "!eslint", "..."]
+		}
+	},
+	"formatter": "language_server"
+}

deno.json

@@ -1,6 +1,6 @@
 {
 	"name": "@andyburke/fsdb",
-	"version": "1.1.0",
+	"version": "1.2.0",
 	"license": "MIT",
 	"exports": {
 		".": "./fsdb.ts",
@@ -8,16 +8,16 @@
 		"./indexers": "./indexers.ts",
 		"./organizers": "./organizers.ts"
 	},
-
 	"tasks": {
 		"lint": "deno lint",
 		"fmt": "deno fmt",
 		"test": "cd tests && DENO_ENV=test FSDB_TEST_DATA_STORAGE_ROOT=./data/$(date --iso-8601=seconds) deno test --allow-env --allow-read --allow-write --fail-fast --trace-leaks ./",
 		"fsdb": "deno run --allow-env --allow-read --allow-write cli.ts"
 	},
-
 	"fmt": {
-		"include": ["**/*.ts"],
+		"include": [
+			"**/*.ts"
+		],
 		"options": {
 			"useTabs": true,
 			"lineWidth": 140,
@@ -28,10 +28,16 @@
 		}
 	},
 	"lint": {
-		"include": ["**/*.ts"],
+		"include": [
+			"**/*.ts"
+		],
 		"rules": {
-			"tags": ["recommended"],
-			"exclude": ["no-explicit-any"]
+			"tags": [
+				"recommended"
+			],
+			"exclude": [
+				"no-explicit-any"
+			]
 		}
 	},
 	"imports": {

fsdb.ts

@@ -53,11 +53,11 @@
  * @module
  */
 
-import * as fs from "@std/fs";
-import * as path from "@std/path";
-import by_lurid from "./organizers/by_lurid.ts";
-import { Optional } from "./utils/optional.ts";
-import { walk, WALK_ENTRY } from "./utils/walk.ts";
+import * as fs from '@std/fs';
+import * as path from '@std/path';
+import by_lurid from './organizers/by_lurid.ts';
+import { Optional } from './utils/optional.ts';
+import { walk, WALK_ENTRY } from './utils/walk.ts';
 
 export type { WALK_ENTRY };
 
@@ -70,7 +70,7 @@ export type FSDB_COLLECTION_CONFIG = {
 };
 export type FSDB_COLLECTION_CONFIG_INPUT = Optional<
 	FSDB_COLLECTION_CONFIG,
-	"id_field" | "organize" | "root"
+	'id_field' | 'organize' | 'root'
 >;
 
 export type FSDB_SEARCH_OPTIONS<T> = {
@@ -98,11 +98,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	constructor(input_config: FSDB_COLLECTION_CONFIG_INPUT) {
 		this.config = {
 			...{
-				id_field: "id",
+				id_field: 'id',
 				organize: by_lurid,
-				root: `${Deno.env.get("FSDB_ROOT") ?? "./fsdb"}/${input_config?.name ?? "unknown"}`,
+				root: `${Deno.env.get('FSDB_ROOT') ?? './fsdb'}/${input_config?.name ?? 'unknown'}`
 			},
-			...(input_config ?? {}),
+			...(input_config ?? {})
 		};
 
 		this.event_listeners = {};
@@ -115,7 +115,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		let existing_collection_info: any = undefined;
 		try {
 			const existing_collection_info_content: string = Deno.readTextFileSync(
-				path.resolve(path.join(this.config.root), ".fsdb.collection.json"),
+				path.resolve(path.join(this.config.root), '.fsdb.collection.json')
 			);
 			existing_collection_info = JSON.parse(existing_collection_info_content);
 		} catch (error) {
@@ -127,42 +127,42 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		if (existing_collection_info) {
 			if (this.config.name !== existing_collection_info.name) {
 				console.warn(
-					"Mismatching collection name, maybe the collection was renamed? Be cautious.",
+					'Mismatching collection name, maybe the collection was renamed? Be cautious.'
 				);
 			}
 
 			if (this.config.root !== existing_collection_info.root) {
 				console.warn(
-					"Mismatching collection root, maybe the collection was moved on disk? Be cautious.",
+					'Mismatching collection root, maybe the collection was moved on disk? Be cautious.'
 				);
 			}
 
 			if (this.config.id_field !== existing_collection_info.id_field) {
 				console.warn(
-					"Mismatching collection id field, maybe the data format has changed? Be cautious.",
+					'Mismatching collection id field, maybe the data format has changed? Be cautious.'
 				);
 			}
 
 			if (
 				Object.keys(this.config.indexers ?? {})
 					.sort()
-					.join("|") !==
+					.join('|') !==
 					Object.keys(existing_collection_info.indexers ?? {})
 						.sort()
-					.join("|")
+						.join('|')
 			) {
 				console.warn(
-					"Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.",
+					'Mismatching collection indexes, maybe the code was updated to add or drop an index? Be cautious.'
 				);
 			}
 		}
 
 		const collection_info_file_path: string = path.resolve(
-			path.join(this.config.root, ".fsdb.collection.json"),
+			path.join(this.config.root, '.fsdb.collection.json')
 		);
-		const collection_info_json: string = JSON.stringify(this.config, null, "\t");
+		const collection_info_json: string = JSON.stringify(this.config, null, '\t');
 		Deno.mkdirSync(path.dirname(collection_info_file_path), {
-			recursive: true,
+			recursive: true
 		});
 		Deno.writeTextFileSync(collection_info_file_path, collection_info_json);
 	}
@@ -190,7 +190,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	 * @returns	{string}				An organized, resolved path for the id.
 	 */
 	public get_organized_id_path(id: string): string {
-		return this.get_organized_item_path({ id }, "id");
+		return this.get_organized_item_path({ id }, 'id');
 	}
 
 	/**
@@ -209,22 +209,21 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	}
 
 	private async write_item(item: T, override_path?: string): Promise<void> {
-		const item_path: string =
-			override_path ?? (await this.ensure_item_path(item, this.config.id_field));
-		Deno.writeTextFileSync(item_path, JSON.stringify(item, null, "\t"));
+		const item_path: string = override_path ?? (await this.ensure_item_path(item, this.config.id_field));
+		Deno.writeTextFileSync(item_path, JSON.stringify(item, null, '\t'));
 
-		this.emit("write", {
+		this.emit('write', {
 			item,
-			item_path,
+			item_path
 		});
 
 		if (this.config.indexers) {
 			for (const indexer of Object.values(this.config.indexers)) {
 				await (indexer as FSDB_INDEXER<T>).index(item, item_path);
-				this.emit("index", {
+				this.emit('index', {
 					item,
 					item_path,
-					indexer,
+					indexer
 				});
 			}
 		}
@@ -248,9 +247,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		const content: string = await Deno.readTextFile(item_path);
 		const item: T = JSON.parse(content);
 
-		this.emit("get", {
+		this.emit('get', {
 			item,
-			item_path,
+			item_path
 		});
 
 		return item;
@@ -268,16 +267,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		const item_exists: boolean = await fs.exists(item_path);
 
 		if (item_exists) {
-			throw new Error("item already exists", {
-				cause: "item_exists",
+			throw new Error('item already exists', {
+				cause: 'item_exists'
 			});
 		}
 
 		await this.write_item(item);
 
-		this.emit("create", {
+		this.emit('create', {
 			item,
-			item_path,
+			item_path
 		});
 
 		return item;
@@ -296,17 +295,17 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		const previous: T | null = await this.get(id);
 
 		if (!previous) {
-			throw new Error("item does not exist", {
-				cause: "item_does_not_exist",
+			throw new Error('item does not exist', {
+				cause: 'item_does_not_exist'
 			});
 		}
 
 		await this.write_item(item, item_path);
 
-		this.emit("update", {
+		this.emit('update', {
 			item,
 			previous,
-			item_path,
+			item_path
 		});
 
 		return item;
@@ -345,7 +344,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			}
 
 			if (has_files) {
-				dir = "";
+				dir = '';
 				break;
 			}
 
@@ -353,8 +352,8 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			dir = path.dirname(dir);
 		} while (dir.length);
 
-		this.emit("delete", {
-			item,
+		this.emit('delete', {
+			item
 		});
 
 		return item;
@@ -371,9 +370,9 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		limit = 100,
 		offset = 0,
 		filter = undefined,
-		sort = undefined,
+		sort = undefined
 	}: FSDB_SEARCH_OPTIONS<T> = {}): Promise<WALK_ENTRY<T>[]> {
-		if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_begin");
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
 
 		const results: WALK_ENTRY<T>[] = [];
 
@@ -388,10 +387,11 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			return results;
 		}
 
-		for await (const entry of walk(this.config.root, {
+		for await (
+			const entry of walk(this.config.root, {
 				filter: (entry: WALK_ENTRY<T>): boolean => {
 					const extension = path.extname(entry.path);
-				if (extension.toLowerCase() !== ".json") {
+					if (extension.toLowerCase() !== '.json') {
 						return false;
 					}
 
@@ -400,14 +400,15 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 					}
 
 					const filename = path.basename(entry.path);
-				if (filename === ".fsdb.collection.json") {
+					if (filename === '.fsdb.collection.json') {
 						return false;
 					}
 
 					return filter ? filter(entry) : true;
 				},
-			sort,
-		})) {
+				sort
+			})
+		) {
 			if (counter < offset) {
 				++counter;
 				continue;
@@ -421,20 +422,21 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			}
 		}
 
-		if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_all_end");
-		if (Deno.env.get("FSDB_PERF"))
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
+		if (Deno.env.get('FSDB_PERF')) {
 			console.dir(
-				performance.measure("fsdb all items time", "fsdb_all_begin", "fsdb_all_end"),
+				performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end')
 			);
+		}
 
-		this.emit("all", {
+		this.emit('all', {
 			options: {
 				limit,
 				offset,
 				filter,
-				sort,
+				sort
 			},
-			results,
+			results
 		});
 
 		return results;
@@ -449,16 +451,16 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	 */
 	async find(
 		criteria: Record<string, any>,
-		options?: FSDB_SEARCH_OPTIONS<T>,
+		options?: FSDB_SEARCH_OPTIONS<T>
 	): Promise<WALK_ENTRY<T>[]> {
-		if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_begin");
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');
 
 		const find_options: FSDB_SEARCH_OPTIONS<T> = {
 			...{
 				limit: 100,
-				offset: 0,
+				offset: 0
 			},
-			...(options ?? {}),
+			...(options ?? {})
 		};
 
 		const results: WALK_ENTRY<T>[] = [];
@@ -472,15 +474,14 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			if (indexer_for_search_key) {
 				const matched_items = await indexer_for_search_key.lookup(value, find_options);
 				for (const matched_item_path of matched_items) {
-					item_path_criteria_match_count[matched_item_path] =
-						item_path_criteria_match_count[matched_item_path] ?? 0;
+					item_path_criteria_match_count[matched_item_path] = item_path_criteria_match_count[matched_item_path] ?? 0;
 					item_path_criteria_match_count[matched_item_path]++;
 				}
 			}
 		}
 
 		const matching_items = Object.keys(item_path_criteria_match_count).filter(
-			(item_path) => item_path_criteria_match_count[item_path] === score_needed,
+			(item_path) => item_path_criteria_match_count[item_path] === score_needed
 		);
 
 		const limit = find_options.limit ?? 100;
@@ -500,7 +501,7 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 				depth: -1,
 				load: function () {
 					return JSON.parse(Deno.readTextFileSync(this.path)) as T;
-				},
+				}
 			});
 
 			++counter;
@@ -510,14 +511,15 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 			}
 		}
 
-		if (Deno.env.get("FSDB_PERF")) performance.mark("fsdb_find_end");
-		if (Deno.env.get("FSDB_PERF"))
-			console.dir(performance.measure("fsdb find time", "fsdb_find_begin", "fsdb_find_end"));
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_end');
+		if (Deno.env.get('FSDB_PERF')) {
+			console.dir(performance.measure('fsdb find time', 'fsdb_find_begin', 'fsdb_find_end'));
+		}
 
-		this.emit("find", {
+		this.emit('find', {
 			criteria,
 			options: find_options,
-			results,
+			results
 		});
 
 		return results;
@@ -530,19 +532,18 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	 * @param	{(event_data: any) => void}	handler		The handler for the event.
 	 */
 	public on(event: string, handler: (event_data: any) => void) {
-		const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
-			this.event_listeners[event] ?? []);
+		const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []);
 		if (!listeners.includes(handler)) {
 			listeners.push(handler);
 		}
 
-		if (Deno.env.get("FSDB_LOG_EVENTS")) {
+		if (Deno.env.get('FSDB_LOG_EVENTS')) {
 			console.dir({
 				on: {
 					event,
-					handler,
+					handler
 				},
-				listeners,
+				listeners
 			});
 		}
 	}
@@ -554,34 +555,32 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 	 * @param	{(event_data: any) => void}	handler		The handler that was registered that should be removed.
 	 */
 	public off(event: string, handler: (event_data: any) => void) {
-		const listeners: ((event: any) => void)[] = (this.event_listeners[event] =
-			this.event_listeners[event] ?? []);
+		const listeners: ((event: any) => void)[] = (this.event_listeners[event] = this.event_listeners[event] ?? []);
 		if (listeners.includes(handler)) {
 			listeners.splice(listeners.indexOf(handler), 1);
 		}
 
-		if (Deno.env.get("FSDB_LOG_EVENTS")) {
+		if (Deno.env.get('FSDB_LOG_EVENTS')) {
 			console.dir({
 				off: {
 					event: event,
-					handler,
+					handler
 				},
-				listeners,
+				listeners
 			});
 		}
 	}
 
 	private emit(event_name: string, event_data: any) {
-		const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] =
-			this.event_listeners[event_name] ?? []);
+		const listeners: ((event: any) => void)[] = (this.event_listeners[event_name] = this.event_listeners[event_name] ?? []);
 
-		if (Deno.env.get("FSDB_LOG_EVENTS")) {
+		if (Deno.env.get('FSDB_LOG_EVENTS')) {
 			console.dir({
 				emitting: {
 					event_name,
 					event_data,
-					listeners,
-				},
+					listeners
+				}
 			});
 		}
 
@@ -592,23 +591,23 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 
 	public sorts = {
 		newest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
-			((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
-				(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "",
+			((b.info.birthtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
+				(a.info.birthtime ?? a.info.ctime)?.toISOString() ?? ''
 			),
 
 		oldest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
-			((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
-				(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? "",
+			((a.info.birthtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
+				(b.info.birthtime ?? b.info.ctime)?.toISOString() ?? ''
 			),
 
 		latest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
-			((b.info.mtime ?? b.info.ctime)?.toISOString() ?? "").localeCompare(
-				(a.info.mtime ?? a.info.ctime)?.toISOString() ?? "",
+			((b.info.mtime ?? b.info.ctime)?.toISOString() ?? '').localeCompare(
+				(a.info.mtime ?? a.info.ctime)?.toISOString() ?? ''
 			),
 
 		stalest: (a: WALK_ENTRY<T>, b: WALK_ENTRY<T>): number =>
-			((a.info.mtime ?? a.info.ctime)?.toISOString() ?? "").localeCompare(
-				(b.info.mtime ?? b.info.ctime)?.toISOString() ?? "",
-			),
+			((a.info.mtime ?? a.info.ctime)?.toISOString() ?? '').localeCompare(
+				(b.info.mtime ?? b.info.ctime)?.toISOString() ?? ''
+			)
 	};
 }

indexers/symlinks.ts

@@ -14,6 +14,7 @@ interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
 	id_field?: string;
 	to_many?: boolean;
 	organize?: (value: string) => string[];
+	organize_id?: (value: string) => string[];
 }
 
 interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
@@ -191,8 +192,12 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
 					continue;
 				}
 
+				if (this.config.organize_id) {
+					organized_paths.push(...this.config.organize_id(item_id));
+				} else {
 					organized_paths.push(`${item_id}.json`);
 				}
+			}
 
 			const symlink_path = path.resolve(path.join(this.config.root, ...organized_paths));
 			const item_dir = path.dirname(authoritative_path);