andyburke/fsdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: andyburke:57bc0f18a03ba7979d61427a9c1c730c4026aa62
Branches Tags
andyburke:dev
..
compare: andyburke:e46f9cefb73361b9cd4623185cf879dedcc6b51d
Branches Tags
andyburke:dev

No commits in common. "57bc0f18a03ba7979d61427a9c1c730c4026aa62" and "e46f9cefb73361b9cd4623185cf879dedcc6b51d" have entirely different histories.
57bc0f18a0 ... e46f9cefb7

4 changed files with 55 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

2
deno.json
View file

@ -1,6 +1,6 @@
{ {
"name": "@andyburke/fsdb", "name": "@andyburke/fsdb",
"version": "1.2.2", "version": "1.2.1",
"license": "MIT", "license": "MIT",
"exports": { "exports": {
".": "./fsdb.ts", ".": "./fsdb.ts",

33
indexers/symlinks.ts
View file

@ -7,7 +7,6 @@ import * as fs from '@std/fs';
import { FSDB_INDEXER, FSDB_SEARCH_OPTIONS } from '../fsdb.ts'; import { FSDB_INDEXER, FSDB_SEARCH_OPTIONS } from '../fsdb.ts';
import * as path from '@std/path'; import * as path from '@std/path';
import sanitize from '../utils/sanitize.ts'; import sanitize from '../utils/sanitize.ts';
import { walk, WALK_ENTRY } from '../utils/walk.ts';
interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
name: string; name: string;
@ -15,7 +14,7 @@ interface FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
id_field?: string; id_field?: string;
to_many?: boolean; to_many?: boolean;
organize?: (value: string) => string[]; organize?: (value: string) => string[];
organize_id?: (value: string, organized: string) => string[]; organize_id?: (value: string) => string[];
} }
interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED { interface FSDB_INDEXER_SYMLINKS_CONFIG_WITH_FIELD extends FSDB_INDEXER_SYMLINKS_CONFIG_SHARED {
@ -94,6 +93,7 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
const filename: string = organized_paths.pop() ?? ''; // remove filename const filename: string = organized_paths.pop() ?? ''; // remove filename
const parsed_filename = path.parse(filename); const parsed_filename = path.parse(filename);
organized_paths.push(parsed_filename.name); // add back filename without extension for a directory organized_paths.push(parsed_filename.name); // add back filename without extension for a directory
organized_paths.push('*'); // wildcard to get all references
} }
const limit = options?.limit ?? 100; const limit = options?.limit ?? 100;
@ -109,37 +109,20 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
}); });
} }
const resolved_path = path.resolve(path.join(this.config.root, ...organized_paths)); const glob_pattern = path.resolve(path.join(this.config.root, ...organized_paths));
for await ( for await (const item_file of fs.expandGlob(glob_pattern)) {
const entry of walk(resolved_path, { const file_info: Deno.FileInfo = await Deno.lstat(item_file.path);
filter: (entry: WALK_ENTRY<T>): boolean => {
const extension = path.extname(entry.path);
if (extension.toLowerCase() !== '.json') {
return false;
}
if (entry.info.isDirectory) {
return false;
}
return true;
},
sort: options?.sort
})
) {
const file_info: Deno.FileInfo = await Deno.lstat(entry.path);
if (file_info.isSymlink) { if (file_info.isSymlink) {
if (counter < offset) { if (counter < offset) {
++counter; ++counter;
continue; continue;
} }
const resolved_item_path = await Deno.readLink(entry.path); const resolved_item_path = await Deno.readLink(item_file.path);
const normalized_path = path.normalize(path.resolve(path.dirname(entry.path), resolved_item_path)); const normalized_path = path.normalize(path.resolve(path.dirname(item_file.path), resolved_item_path));
if (Deno.env.get('FSDB_DEBUG')) { if (Deno.env.get('FSDB_DEBUG')) {
console.dir({ console.dir({
entry,
resolved_item_path, resolved_item_path,
normalized_path normalized_path
}); });
@ -228,7 +211,7 @@ export class FSDB_INDEXER_SYMLINKS<T> implements FSDB_INDEXER<T> {
} }
if (this.config.organize_id) { if (this.config.organize_id) {
organized_paths.push(...this.config.organize_id(item_id, parsed_filename.name)); organized_paths.push(...this.config.organize_id(item_id));
} else { } else {
organized_paths.push(`${item_id}.json`); organized_paths.push(`${item_id}.json`);
} }

98
tests/04_indexing_sanity_checks.test.ts
View file

@ -1,22 +1,21 @@
import * as asserts from '@std/assert'; import * as asserts from "@std/assert";
import * as fsdb from '../fsdb.ts'; import * as fsdb from "../fsdb.ts";
import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts'; import { FSDB_INDEXER_SYMLINKS } from "../indexers.ts";
import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts'; import { get_data_dir, random_email_address, random_phone_number } from "./helpers.ts";
import lurid from '@andyburke/lurid'; import lurid from "@andyburke/lurid";
import by_email from '../organizers/by_email.ts'; import by_email from "../organizers/by_email.ts";
import by_character from '../organizers/by_character.ts'; import by_character from "../organizers/by_character.ts";
import by_phone from '../organizers/by_phone.ts'; import by_phone from "../organizers/by_phone.ts";
import { sentence } from 'jsr:@ndaidong/txtgen'; import { sentence } from "jsr:@ndaidong/txtgen";
import { by_lurid } from '@andyburke/fsdb/organizers';
Deno.test({ Deno.test({
name: 'index some items', name: "index some items",
permissions: { permissions: {
env: true, env: true,
// https://github.com/denoland/deno/discussions/17258 // https://github.com/denoland/deno/discussions/17258
read: true, read: true,
write: true write: true,
}, },
fn: async () => { fn: async () => {
type ITEM = { type ITEM = {
@ -28,47 +27,40 @@ Deno.test({
}; };
const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({ const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
name: 'test-04-items', name: "test-04-items",
root: get_data_dir() + '/test-04-items', root: get_data_dir() + "/test-04-items",
indexers: { indexers: {
email: new FSDB_INDEXER_SYMLINKS<ITEM>({ email: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'email', name: "email",
field: 'email', field: "email",
organize: by_email organize: by_email,
}), }),
phone: new FSDB_INDEXER_SYMLINKS<ITEM>({ phone: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'phone', name: "phone",
field: 'phone', field: "phone",
organize: by_phone organize: by_phone,
}), }),
stable: new FSDB_INDEXER_SYMLINKS<ITEM>({ stable: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'stable', name: "stable",
field: 'stable', field: "stable",
to_many: true, to_many: true,
organize: by_character organize: by_character,
}),
custom_organizing_test: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'custom_organizing_test',
organize: (word) => [word],
organize_id: (id: string) => {
return [id.substring(0, 4), ...by_lurid(id)];
},
get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3),
to_many: true
}), }),
by_character_test: new FSDB_INDEXER_SYMLINKS<ITEM>({ by_character_test: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'by_character_test', name: "by_character_test",
organize: by_character, organize: by_character,
get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3), get_values_to_index: (item: ITEM) =>
to_many: true item.value.split(/\W/).filter((word) => word.length > 3),
to_many: true,
}), }),
by_possibly_undefined: new FSDB_INDEXER_SYMLINKS<ITEM>({ by_possibly_undefined: new FSDB_INDEXER_SYMLINKS<ITEM>({
name: 'by_possibly_undefined', name: "by_possibly_undefined",
organize: by_character, organize: by_character,
get_values_to_index: (item: ITEM) => item.email.indexOf('.com') > 0 ? [item.email] : [], get_values_to_index: (item: ITEM) =>
to_many: true item.email.indexOf(".com") > 0 ? [item.email] : [],
}) to_many: true,
} }),
},
}); });
asserts.assert(item_collection); asserts.assert(item_collection);
@ -79,8 +71,8 @@ Deno.test({
id: lurid(), id: lurid(),
email: random_email_address(), email: random_email_address(),
phone: random_phone_number(), phone: random_phone_number(),
stable: 'stable', stable: "stable",
value: sentence() value: sentence(),
}; };
items.push(item); items.push(item);
@ -108,7 +100,8 @@ Deno.test({
const words_in_value: string[] = item.value const words_in_value: string[] = item.value
.split(/\W/) .split(/\W/)
.filter((word) => word.length > 3); .filter((word) => word.length > 3);
const random_word_in_value: string = words_in_value[Math.floor(Math.random() * words_in_value.length)]; const random_word_in_value: string =
words_in_value[Math.floor(Math.random() * words_in_value.length)];
const fetched_by_word_in_value: ITEM[] = ( const fetched_by_word_in_value: ITEM[] = (
await item_collection.find({ by_character_test: random_word_in_value }) await item_collection.find({ by_character_test: random_word_in_value })
).map((entry) => entry.load()); ).map((entry) => entry.load());
@ -116,19 +109,8 @@ Deno.test({
asserts.assertGreater(fetched_by_word_in_value.length, 0); asserts.assertGreater(fetched_by_word_in_value.length, 0);
asserts.assert( asserts.assert(
fetched_by_word_in_value.find( fetched_by_word_in_value.find(
(word_in_value_item) => word_in_value_item.id === item.id (word_in_value_item) => word_in_value_item.id === item.id,
) ),
);
const fetched_by_custom_organization_index: ITEM[] = (
await item_collection.find({ custom_organizing_test: random_word_in_value })
).map((entry) => entry.load());
asserts.assertLess(fetched_by_custom_organization_index.length, items.length);
asserts.assertGreater(fetched_by_custom_organization_index.length, 0);
asserts.assert(
fetched_by_custom_organization_index.find(
(word_in_value_item) => word_in_value_item.id === item.id
)
); );
} }
@ -137,7 +119,7 @@ Deno.test({
asserts.assert(random_item); asserts.assert(random_item);
const criteria: Record<string, string> = { const criteria: Record<string, string> = {
stable: 'stable' stable: "stable",
}; };
if (Math.random() < 0.5) { if (Math.random() < 0.5) {
@ -172,5 +154,5 @@ Deno.test({
// ) => entry.load()); // ) => entry.load());
// asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id)); // asserts.assertFalse(fetched_by_word_in_value.find((word_in_value_item) => word_in_value_item.id === item.id));
// } // }
} },
}); });

14
utils/walk.ts
View file

@ -27,19 +27,6 @@ export async function* walk<T>(
const entries: WALK_ENTRY<T>[] = []; const entries: WALK_ENTRY<T>[] = [];
const root_info: Deno.FileInfo = await Deno.lstat(root_path);
if (!root_info.isDirectory) {
const entry: WALK_ENTRY<T> = {
path: root_path,
info: root_info,
depth,
load: function () {
return JSON.parse(Deno.readTextFileSync(this.path)) as T;
}
};
entries.push(entry);
} else {
for await (const dir_entry of Deno.readDir(root_path)) { for await (const dir_entry of Deno.readDir(root_path)) {
const full_path = path.join(root, dir_entry.name); const full_path = path.join(root, dir_entry.name);
const info = await Deno.lstat(full_path); const info = await Deno.lstat(full_path);
@ -54,7 +41,6 @@ export async function* walk<T>(
entries.push(entry); entries.push(entry);
} }
}
if (sort) { if (sort) {
entries.sort(sort); entries.sort(sort);