feature: .all() to iterate over all objects in a collection

README.md

@@ -9,7 +9,8 @@ optimization to the filesystem layer.
 	`collection.get(id) : T` - gets an object of type T based on the id field configured
 	`collection.update(T)` - updates an object of type T, saving it to the disk
 	`collection.delete(T)` - removes an object from the system and the disk
-	`collection.find(criteria)` - find all objects that match the criteria *that have an index*
+	`collection.all([options])` - iterate over all objects
+	`collection.find(criteria[, options])` - find all objects that match the criteria *that have an index*
 
 ### Example
 

deno.json

@@ -1,6 +1,6 @@
 {
 	"name": "@andyburke/fsdb",
-	"version": "0.4.0",
+	"version": "0.5.0",
 	"license": "MIT",
 	"exports": {
 		".": "./fsdb.ts",

fsdb.ts

@@ -199,6 +199,53 @@ export class FSDB_COLLECTION<T extends Record<string, any>> {
 		return item;
 	}
 
+	/** Iterate through the items. */
+	async all(input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_begin');
+
+		const options: FSDB_SEARCH_OPTIONS = {
+			...{
+				limit: 100,
+				offset: 0
+			},
+			...(input_options ?? {})
+		};
+
+		const results: T[] = [];
+
+		const limit = options?.limit ?? 100;
+		const offset = options?.offset ?? 0;
+		let counter = 0;
+
+		// TODO: better way to get a pattern to match files in this collection?
+		for await (
+			const entry of fs.walk(this.config.root, {
+				includeDirs: false,
+				includeSymlinks: false,
+				skip: [/\.fsdb\.collection\.json$/],
+				exts: ['json']
+			})
+		) {
+			if (counter < offset) {
+				++counter;
+				continue;
+			}
+
+			const content = await Deno.readTextFile(entry.path);
+			results.push(JSON.parse(content));
+			++counter;
+
+			if (counter >= (offset + limit)) {
+				break;
+			}
+		}
+
+		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_all_end');
+		if (Deno.env.get('FSDB_PERF')) console.dir(performance.measure('fsdb all items time', 'fsdb_all_begin', 'fsdb_all_end'));
+
+		return results;
+	}
+
 	/** Use indexes to search for matching items. */
 	async find(criteria: Record<string, any>, input_options?: FSDB_SEARCH_OPTIONS): Promise<T[]> {
 		if (Deno.env.get('FSDB_PERF')) performance.mark('fsdb_find_begin');

tests/05_test_all.test.ts

@@ -0,0 +1,94 @@
+import * as asserts from '@std/assert';
+import * as fsdb from '../fsdb.ts';
+import { FSDB_INDEXER_SYMLINKS } from '../indexers.ts';
+import { get_data_dir, random_email_address, random_phone_number } from './helpers.ts';
+import lurid from '@andyburke/lurid';
+import by_email from '../organizers/by_email.ts';
+import by_character from '../organizers/by_character.ts';
+import by_phone from '../organizers/by_phone.ts';
+import { sentence } from 'jsr:@ndaidong/txtgen';
+
+Deno.test({
+	name: 'iterate over all items',
+	permissions: {
+		env: true,
+
+		// https://github.com/denoland/deno/discussions/17258
+		read: true,
+		write: true
+	},
+	fn: async () => {
+		type ITEM = {
+			id: string;
+			email: string;
+			phone: string;
+			value: string;
+		};
+
+		const item_collection: fsdb.FSDB_COLLECTION<ITEM> = new fsdb.FSDB_COLLECTION<ITEM>({
+			name: 'test-05-items',
+			root: get_data_dir() + '/test-05-items',
+			indexers: {
+				email: new FSDB_INDEXER_SYMLINKS<ITEM>({
+					name: 'email',
+					field: 'email',
+					organize: by_email
+				}),
+				phone: new FSDB_INDEXER_SYMLINKS<ITEM>({
+					name: 'phone',
+					field: 'phone',
+					organize: by_phone
+				}),
+				by_character_test: new FSDB_INDEXER_SYMLINKS<ITEM>({
+					name: 'by_character_test',
+					organize: by_character,
+					get_values_to_index: (item: ITEM) => item.value.split(/\W/).filter((word) => word.length > 3),
+					to_many: true
+				})
+			}
+		});
+
+		asserts.assert(item_collection);
+
+		const items: ITEM[] = [];
+		for (let i = 0; i < 500; ++i) {
+			const item = {
+				id: lurid(),
+				email: random_email_address(),
+				phone: random_phone_number(),
+				value: sentence()
+			};
+
+			items.push(item);
+
+			const stored_item: ITEM = await item_collection.create(item);
+
+			asserts.assertObjectMatch(stored_item, item);
+		}
+
+		const LIMIT_MIN = 11;
+		const LIMIT_MAX = 333;
+
+		let offset = 0;
+		const fetched = [];
+		let more_to_fetch = true;
+		do {
+			// fuzz the limit
+			const limit = Math.floor(Math.random() * (LIMIT_MAX - LIMIT_MIN + 1)) + LIMIT_MIN;
+
+			const fetched_items = await item_collection.all({
+				limit,
+				offset
+			});
+
+			fetched.push(...fetched_items);
+			offset += fetched_items.length;
+			more_to_fetch = fetched_items.length === limit;
+		} while (more_to_fetch);
+
+		const sorted_items = items.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id));
+		const sorted_fetched = fetched.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id));
+
+		asserts.assertEquals(sorted_fetched, sorted_items);
+	}
+});

tests/helpers.ts

@@ -14,10 +14,10 @@ const TLDs: string[] = [
 
 const random_byte_buffer: Uint8Array = new Uint8Array(3);
 export function random_email_address(): string {
-	crypto.getRandomValues(random_byte_buffer);
+	for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256);
 	const name = convert_to_words(random_byte_buffer).join('-');
 
-	crypto.getRandomValues(random_byte_buffer);
+	for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256);
 	const domain = convert_to_words(random_byte_buffer).join('-');
 
 	const tld = TLDs[Math.floor(Math.random() * TLDs.length)];
@@ -25,7 +25,7 @@ export function random_email_address(): string {
 }
 
 export function random_username(): string {
-	crypto.getRandomValues(random_byte_buffer);
+	for (let i: number = 0; i < random_byte_buffer.length; ++i) random_byte_buffer[i] = Math.floor(Math.random() * 256);
 	return convert_to_words(random_byte_buffer).join('-');
 }