diff --git a/deno.json b/deno.json index ead199b..8073d60 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@andyburke/fsdb", - "version": "1.2.2", + "version": "1.2.1", "license": "MIT", "exports": { ".": "./fsdb.ts", diff --git a/indexers/symlinks.ts b/indexers/symlinks.ts index 96db604..7c875b4 100644 --- a/indexers/symlinks.ts +++ b/indexers/symlinks.ts @@ -7,7 +7,6 @@ import * as fs from '@std/fs'; import { FSDB_INDEXER, FSDB_SEARCH_OPTIONS } from '../fsdb.ts'; import * as path from '@std/path'; import sanitize from '../utils/sanitize.ts'; -import { walk, WALK_ENTRY } from '../utils/walk.ts'; interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { name: string; @@ -15,7 +14,7 @@ interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { id_field?: string; to_many?: boolean; organize?: (value: string) => string[]; - organize_id?: (value: string, organized: string) => string[]; + organize_id?: (value: string) => string[]; } interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { @@ -94,6 +93,7 @@ export class FSDB_INDEXER_SYMLINKS implements FSDB_INDEXER { const filename: string = organized_paths.pop() ?? ''; // remove filename const parsed_filename = path.parse(filename); organized_paths.push(parsed_filename.name); // add back filename without extension for a directory + organized_paths.push('*'); // wildcard to get all references } const limit = options?.limit ?? 100; @@ -109,37 +109,20 @@ export class FSDB_INDEXER_SYMLINKS implements FSDB_INDEXER { }); } - const resolved_path = path.resolve(path.join(this.config.root, ...organized_paths)); - for await ( - const entry of walk(resolved_path, { - filter: (entry: WALK_ENTRY): boolean => { - const extension = path.extname(entry.path); - if (extension.toLowerCase() !== '.json') { - return false; - } - - if (entry.info.isDirectory) { - return false; - } - - return true; - }, - sort: options?.sort - }) - ) { - const file_info: Deno.FileInfo = await Deno.lstat(entry.path); + const glob_pattern = path.resolve(path.join(this.config.root, ...organized_paths)); + for await (const item_file of fs.expandGlob(glob_pattern)) { + const file_info: Deno.FileInfo = await Deno.lstat(item_file.path); if (file_info.isSymlink) { if (counter < offset) { ++counter; continue; } - const resolved_item_path = await Deno.readLink(entry.path); - const normalized_path = path.normalize(path.resolve(path.dirname(entry.path), resolved_item_path)); + const resolved_item_path = await Deno.readLink(item_file.path); + const normalized_path = path.normalize(path.resolve(path.dirname(item_file.path), resolved_item_path)); if (Deno.env.get('FSDB_DEBUG')) { console.dir({ - entry, resolved_item_path, normalized_path }); @@ -228,7 +211,7 @@ export class FSDB_INDEXER_SYMLINKS implements FSDB_INDEXER { } if (this.config.organize_id) { - organized_paths.push(...this.config.organize_id(item_id, parsed_filename.name)); + organized_paths.push(...this.config.organize_id(item_id)); } else { organized_paths.push(`${item_id}.json`); } diff --git a/tests/04_indexing_sanity_checks.test.ts b/tests/04_indexing_sanity_checks.test.ts index bbb6240..4019d1c 100644 --- a/tests/04_indexing_sanity_checks.test.ts +++ b/tests/04_indexing_sanity_checks.test.ts @@ -1,22 +1,21 @@ -import * as asserts from '@std/assert'; -import * as fsdb from '../fsdb.ts'; -import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts'; -import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts'; -import lurid from '@andyburke/lurid'; -import by_email from '../organizers/by_email.ts'; -import by_character from '../organizers/by_character.ts'; -import by_phone from '../organizers/by_phone.ts'; -import { sentence } from 'jsr:@ndaidong/txtgen'; -import { by_lurid } from '@andyburke/fsdb/organizers'; +import * as asserts from "@std/assert"; +import * as fsdb from "../fsdb.ts"; +import { FSDB_INDEXER_SYMLINKS } from "../indexers.ts"; +import { get_data_dir, random_email_address, random_phone_number } from "./helpers.ts"; +import lurid from "@andyburke/lurid"; +import by_email from "../organizers/by_email.ts"; +import by_character from "../organizers/by_character.ts"; +import by_phone from "../organizers/by_phone.ts"; +import { sentence } from "jsr:@ndaidong/txtgen"; Deno.test({ - name: 'index some items', + name: "index some items", permissions: { env: true, // https://github.com/denoland/deno/discussions/17258 read: true, - write: true + write: true, }, fn: async () => { type ITEM = { @@ -28,47 +27,40 @@ Deno.test({ }; const item_collection: fsdb.FSDB_COLLECTION = new fsdb.FSDB_COLLECTION({ - name: 'test-04-items', - root: get_data_dir() + '/test-04-items', + name: "test-04-items", + root: get_data_dir() + "/test-04-items", indexers: { email: new FSDB_INDEXER_SYMLINKS({ - name: 'email', - field: 'email', - organize: by_email + name: "email", + field: "email", + organize: by_email, }), phone: new FSDB_INDEXER_SYMLINKS({ - name: 'phone', - field: 'phone', - organize: by_phone + name: "phone", + field: "phone", + organize: by_phone, }), stable: new FSDB_INDEXER_SYMLINKS({ - name: 'stable', - field: 'stable', + name: "stable", + field: "stable", to_many: true, - organize: by_character - }), - custom_organizing_test: new FSDB_INDEXER_SYMLINKS({ - name: 'custom_organizing_test', - organize: (word) => [word], - organize_id: (id: string) => { - return [id.substring(0, 4), ...by_lurid(id)]; - }, - get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), - to_many: true + organize: by_character, }), by_character_test: new FSDB_INDEXER_SYMLINKS({ - name: 'by_character_test', + name: "by_character_test", organize: by_character, - get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), - to_many: true + get_values_to_index: (item: ITEM) => + item.value.split(/\W/).filter((word) => word.length > 3), + to_many: true, }), by_possibly_undefined: new FSDB_INDEXER_SYMLINKS({ - name: 'by_possibly_undefined', + name: "by_possibly_undefined", organize: by_character, - get_values_to_index: (item: ITEM) => item.email.indexOf('.com') > 0 ? [item.email] : [], - to_many: true - }) - } + get_values_to_index: (item: ITEM) => + item.email.indexOf(".com") > 0 ? [item.email] : [], + to_many: true, + }), + }, }); asserts.assert(item_collection); @@ -79,8 +71,8 @@ Deno.test({ id: lurid(), email: random_email_address(), phone: random_phone_number(), - stable: 'stable', - value: sentence() + stable: "stable", + value: sentence(), }; items.push(item); @@ -108,7 +100,8 @@ Deno.test({ const words_in_value: string[] = item.value .split(/\W/) .filter((word) => word.length > 3); - const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)]; + const random_word_in_value: string = + words_in_value[Math.floor(Math.random() * words_in_value.length)]; const fetched_by_word_in_value: ITEM[] = ( await item_collection.find({ by_character_test: random_word_in_value }) ).map((entry) => entry.load()); @@ -116,19 +109,8 @@ Deno.test({ asserts.assertGreater(fetched_by_word_in_value.length, 0); asserts.assert( fetched_by_word_in_value.find( - (word_in_value_item) => word_in_value_item.id === item.id - ) - ); - - const fetched_by_custom_organization_index: ITEM[] = ( - await item_collection.find({ custom_organizing_test: random_word_in_value }) - ).map((entry) => entry.load()); - asserts.assertLess(fetched_by_custom_organization_index.length, items.length); - asserts.assertGreater(fetched_by_custom_organization_index.length, 0); - asserts.assert( - fetched_by_custom_organization_index.find( - (word_in_value_item) => word_in_value_item.id === item.id - ) + (word_in_value_item) => word_in_value_item.id === item.id, + ), ); } @@ -137,7 +119,7 @@ Deno.test({ asserts.assert(random_item); const criteria: Record = { - stable: 'stable' + stable: "stable", }; if (Math.random() < 0.5) { @@ -172,5 +154,5 @@ Deno.test({ // ) => entry.load()); // asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id)); // } - } + }, }); diff --git a/utils/walk.ts b/utils/walk.ts index 537ca1b..e31b401 100644 --- a/utils/walk.ts +++ b/utils/walk.ts @@ -27,11 +27,12 @@ export async function* walk( const entries: WALK_ENTRY[] = []; - const root_info: Deno.FileInfo = await Deno.lstat(root_path); - if (!root_info.isDirectory) { - const entry: WALK_ENTRY = { - path: root_path, - info: root_info, + for await (const dir_entry of Deno.readDir(root_path)) { + const full_path = path.join(root, dir_entry.name); + const info = await Deno.lstat(full_path); + const entry = { + path: full_path, + info, depth, load: function () { return JSON.parse(Deno.readTextFileSync(this.path)) as T; @@ -39,21 +40,6 @@ export async function* walk( }; entries.push(entry); - } else { - for await (const dir_entry of Deno.readDir(root_path)) { - const full_path = path.join(root, dir_entry.name); - const info = await Deno.lstat(full_path); - const entry = { - path: full_path, - info, - depth, - load: function () { - return JSON.parse(Deno.readTextFileSync(this.path)) as T; - } - }; - - entries.push(entry); - } } if (sort) {