feat: add deno rpc integration for the extension

deno-src/bucket.ts

@@ -1,42 +1,37 @@
-import { FaissStore } from "@langchain/community/vectorstores/faiss";
+import { FaissStore } from '@langchain/community/vectorstores/faiss';
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { OpenAIEmbeddings } from '@langchain/openai';
-import * as v from "valibot";
+import * as v from 'valibot';
-import * as path from "jsr:@std/path";
+import * as path from 'jsr:@std/path';
-import { existsSync } from "node:fs";
+import { existsSync, readdirSync } from 'node:fs';
-import { Document } from "@langchain/core/documents";
+import { Document } from '@langchain/core/documents';
-import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
-import { DirectoryLoader } from "langchain/document_loaders/fs/directory";
+import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
-import {
+import { JSONLoader, JSONLinesLoader } from 'langchain/document_loaders/fs/json';
-  JSONLoader,
+import { TextLoader } from 'langchain/document_loaders/fs/text';
-  JSONLinesLoader,
+import { computeSha256FromText } from './crypto.ts';
-} from "langchain/document_loaders/fs/json";
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
-import { TextLoader } from "langchain/document_loaders/fs/text";
-import { computeSha256FromText } from "./crypto.ts";
-import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf";
 
 export const embeddings = new OpenAIEmbeddings({
-  model: "text-embedding-3-large",
+	model: 'text-embedding-3-large'
 });
 
 export const MetadataSchema = v.object({
-  filesSha256: v.array(v.string()),
+	filesSha256: v.array(v.string())
 });
 export type Metadata = v.InferOutput<typeof MetadataSchema>;
 
-export async function getDocsFromDirectory(
+export async function getDocsFromDirectory(directoryPath: string): Promise<Document[]> {
-  directoryPath: string
-): Promise<Document[]> {
 	const splitter = new RecursiveCharacterTextSplitter({
 		chunkSize: 1000,
-    chunkOverlap: 200,
+		chunkOverlap: 200
 	});
 
 	const loader = new DirectoryLoader(directoryPath, {
-    ".json": (path) => new JSONLoader(path, "/texts"),
+		'.json': (path) => new JSONLoader(path, '/texts'),
-    ".jsonl": (path) => new JSONLinesLoader(path, "/html"),
+		'.jsonl': (path) => new JSONLinesLoader(path, '/html'),
-    ".txt": (path) => new TextLoader(path),
+		'.txt': (path) => new TextLoader(path),
-    ".md": (path) => new TextLoader(path),
+		'.md': (path) => new TextLoader(path),
-    ".mdx": (path) => new TextLoader(path),
+		'.mdx': (path) => new TextLoader(path)
 	});
 	const docs = await loader.load();
 	const allSplits = await splitter.splitDocuments(docs);
@@ -44,19 +39,19 @@ export async function getDocsFromDirectory(
 }
 
 export class Bucket {
-  private readonly bucketPath: string;
+	readonly bucketPath: string;
-  private readonly faissStorePath: string;
+	readonly faissStorePath: string;
-  private readonly metadataPath: string;
+	readonly metadataPath: string;
 	private _vectorStore: FaissStore | null = null;
 	filesSha256: Set<string> = new Set();
 
 	constructor(
-    private readonly bucketDir: string,
+		readonly bucketDir: string,
-    private readonly bucketName: string
+		readonly bucketName: string
 	) {
 		this.bucketPath = path.join(this.bucketDir, this.bucketName);
-    this.faissStorePath = path.join(this.bucketPath, "faiss-store");
+		this.faissStorePath = path.join(this.bucketPath, 'faiss-store');
-    this.metadataPath = path.join(this.bucketPath, "metadata.json");
+		this.metadataPath = path.join(this.bucketPath, 'metadata.json');
 	}
 
 	async init() {
@@ -69,7 +64,7 @@ export class Bucket {
 			if (parsedMetadata.success) {
 				this.filesSha256 = new Set(parsedMetadata.output.filesSha256);
 			} else {
-        throw new Error("Invalid metadata");
+				throw new Error('Invalid metadata');
 			}
 		}
 		this.updateMetadata();
@@ -81,7 +76,7 @@ export class Bucket {
 
 	updateMetadata() {
 		const metadata: Metadata = {
-      filesSha256: Array.from(this.filesSha256),
+			filesSha256: Array.from(this.filesSha256)
 		};
 		Deno.writeTextFileSync(this.metadataPath, JSON.stringify(metadata));
 	}
@@ -89,12 +84,9 @@ export class Bucket {
 	async getVectorStore() {
 		if (
 			existsSync(this.faissStorePath) &&
-      existsSync(path.join(this.faissStorePath, "docstore.json"))
+			existsSync(path.join(this.faissStorePath, 'docstore.json'))
 		) {
-      const vectorStore = await FaissStore.load(
+			const vectorStore = await FaissStore.load(this.faissStorePath, embeddings);
-        this.faissStorePath,
-        embeddings
-      );
 			return vectorStore;
 		}
 		// const vectorStore = await FaissStore.fromDocuments(docs, embeddings);
@@ -105,14 +97,14 @@ export class Bucket {
 
 	get vectorStore() {
 		if (this._vectorStore === null) {
-      throw new Error("Vector store not initialized");
+			throw new Error('Vector store not initialized');
 		}
 		return this._vectorStore;
 	}
 
 	private async addDocuments(documents: Document[]) {
 		await this.vectorStore.addDocuments(documents);
-    await this.vectorStore.save(this.faissStorePath);
+		// await this.vectorStore.save(this.faissStorePath);
 	}
 
 	private fillSha256(documents: Document[]) {
@@ -123,9 +115,7 @@ export class Bucket {
 	}
 
 	private getFilteredDocs(documents: Document[]) {
-    return documents.filter(
+		return documents.filter((doc) => !this.filesSha256.has(doc.metadata.sha256));
-      (doc) => !this.filesSha256.has(doc.metadata.sha256)
-    );
 	}
 
 	private updateSha256(docs: Document[]) {
@@ -137,12 +127,12 @@ export class Bucket {
 
 	async addDirectory(directoryPath: string) {
 		if (!existsSync(directoryPath)) {
-      throw new Error("Directory does not exist");
+			throw new Error('Directory does not exist');
 		}
 		// check if path is a directory or file
 		const stats = Deno.statSync(directoryPath);
 		if (stats.isFile) {
-      throw new Error("Path is a file");
+			throw new Error('Path is a file');
 		}
 
 		const docs = await getDocsFromDirectory(directoryPath);
@@ -154,7 +144,7 @@ export class Bucket {
 		this.updateMetadata();
 		const splitter = new RecursiveCharacterTextSplitter({
 			chunkSize: 1000,
-      chunkOverlap: 200,
+			chunkOverlap: 200
 		});
 		const allSplits = await splitter.splitDocuments(fileteredDocs);
 		await this.addDocuments(allSplits);
@@ -163,10 +153,27 @@ export class Bucket {
 	async addTextFile(filePath: string) {
 		const loader = new TextLoader(filePath);
 		const docs = await loader.load();
+		console.error('Loaded docs', docs.length);
 		this.fillSha256(docs);
 		const fileteredDocs = this.getFilteredDocs(docs);
+		console.error('Filtered docs', fileteredDocs.length);
 		this.updateSha256(docs);
-    await this.addDocuments(fileteredDocs);
+		console.error('Updated sha256', this.filesSha256.size);
+		// await this.addDocuments(fileteredDocs);
+		return this.vectorStore.addDocuments(fileteredDocs).catch((err) => {
+			console.error('Error adding documents', err);
+		});
+	}
+
+	async save() {
+		console.error('Save Bucket', this.vectorStore.docstore._docs.size);
+		if (existsSync(this.faissStorePath) && readdirSync(this.faissStorePath).length === 0) {
+			Deno.removeSync(this.bucketPath, { recursive: true });
+		}
+		if (this.vectorStore.docstore._docs.size === 0) {
+			throw new Error('No documents to save');
+		}
+		await this.vectorStore.save(this.faissStorePath);
 	}
 
 	async addPDF(filePath: string) {

deno-src/buckets/dev/metadata.json

@@ -1 +0,0 @@
-{"filesSha256":["bcecd6e1f4c1296ebaed37b2686a55485855bb3d04d71a16609297899ef89ba9","f67eb8d401f74305c8a4f21b2ac8986c1db3850aabe2753773ba00255fd4288c","61614a8362e20d8dc96bc8bcd52cd40b9bbf983c862253b9472f1044744bea02","f34a6fbd2e0f3fc1a1a8f3c019935e53e809d38b18fdab67ec742e02e3318a6f","e6929768ff182e0c6714a60157e94420c7942fa2cdad1f201a8302be9e86d064","b19540b04d609f5798e91d6de8ecd0fbbee9f7c3dfb008e2a198247241401c57","324d4f9a84e4849c002ae4091fe7614d2d00cb0bf4989574b307081669f9eb7a","0618796260fd0b8b4cc6daedbac59c06492a74eeebefbddf69fb13b2d2456da8","1a5d9b3ea54168ec8f89e1a93950b1d5c527c831b951070d3ffcce853c6e797f","0cc8092a90aa3f614a87258d372348706bc2f0d33e9235630849cb0aeec4342e","90ef40d14982b96bb2c4c657e17dbf74d8b15f1a36712d2759765c9e5de94995","ec62822e3eeffec14393b16a12d0dcb0920c11ad4fa1a3a6c59252ef39321c57","53f3b51bad507e0799121d30f1a96cb458d6caacda20ee6bd6447aa7a2041772","bc7551da68e709d023b76979f554252467465066285bb4c7aad07e05be82833c","a2fc83c128ebafcfc444c3740a3afcedc2e92b8c10f51724d41da57b8f670546","04c793bf24afcd608ecbbb3606ba1280602f02c678bb885b2e75f423ce394e22","3fe4c596a0bedd833fbe214081467647786a12561e3af0e4cd88a43c2787f35f","95f0df47bfd2b698f43aac7dc22ac13948fe093f838a25ab63c5f6f9b789d942","a9b8caefaa7a959928c587f74399a86549b7680894c1c882a631725a5252881e","ab68d62cb3631acf57b0704996303f2704730dd27fb30fd867092c8af90f4fb5","926688c2cc2bf5214f8ad2f82bdd68a630f0226118f50c54f8d618c2417f4c9f","4e52cee59ad6b3220e31dee7e2c9da499ce2f6c6a6a16305fefc4d05c0d63e08","c28379be7109e76bfccffcf7268b1e767c54e5f279179c773ddb33ba1a83717f","451ecba7690493bf70d26bf127bff2de09d7d5f0b51c16c363f3f372184a2988","f0b6f639135d28f12914f4fd9d8e7f11aa3c67e736cbbfbda36ec97f382b4577","a7516e047d1287385581727d5b019512288207bd2172bd241a0c6255d2b570c0","d67a47978e51dec70271dde449d9d38695a73c9787432379ab13a3b4ef78d7eb","ffbcb29742daf9184167d0e12e191e672b2d0662af402e8dfdea3f33117eb388","c19ae6410ab85759fbba1eb7c0ed49cb4f7a62759288ade5a287714e57f17823","a508c2976b25ed28733d26d2cd7601ee67aff3ad8cc0f1faf221e2ba23f325d4"]}

deno-src/constants.ts

@@ -0,0 +1 @@
+export const txtExts = ['.txt', '.md', '.mdx'];

deno-src/dev.ts

@@ -1,20 +1,77 @@
-import { FaissStore } from "@langchain/community/vectorstores/faiss";
+// import { FaissStore } from '@langchain/community/vectorstores/faiss';
-import { Bucket, embeddings, getDocsFromDirectory } from "./bucket.ts";
+// import { Bucket, embeddings, getDocsFromDirectory } from './bucket.ts';
-import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+// import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
-import { DirectoryLoader } from "langchain/document_loaders/fs/directory";
+// import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
-import {
+// import { JSONLoader, JSONLinesLoader } from 'langchain/document_loaders/fs/json';
-  JSONLoader,
+// import { TextLoader } from 'langchain/document_loaders/fs/text';
-  JSONLinesLoader,
+// import { OpenAIEmbeddings } from '@langchain/openai';
-} from "langchain/document_loaders/fs/json";
+import { existsSync } from 'node:fs';
-import { TextLoader } from "langchain/document_loaders/fs/text";
+// import path from 'path';
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { txtExts } from './constants.ts';
+import { Bucket } from './bucket.ts';
+import path from 'node:path';
 
-const bucket = new Bucket(
+async function indexFiles(bucketName: string, files: string[]): Promise<void> {
-  "/Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/deno-src/buckets",
+	const bucket = new Bucket('./store', bucketName);
-  "dev"
+	// const bucket = new Bucket(extensionSupportPath, bucketName);
-);
+	console.error('bucket path', bucket.bucketPath);
+	console.error('files', files);
 	await bucket.init();
-// await bucket.addDirectory(
+	for (const file of files) {
-//   "/Users/hk/Dev/kunkun-docs/src/content/docs/guides/Extensions/Publish"
+		if (!existsSync(file)) {
+			throw new Error(`File ${file} does not exist`);
+		}
+		console.error('file', file);
+		// check if file is directory
+		const stats = Deno.statSync(file);
+		if (stats.isFile) {
+			const ext = path.extname(file);
+			if (txtExts.includes(ext)) {
+				console.error('Adding text file', file);
+				await bucket.addTextFile(file);
+				console.error('Finished adding text file', file);
+			} else if (ext === '.pdf') {
+				console.error('Adding pdf file', file);
+				await bucket.addPDF(file);
+			} else if (stats.isDirectory) {
+				console.error('Adding directory', file);
+				await bucket.addDirectory(file);
+			} else {
+				throw new Error(`Unsupported file type: ${ext}`);
+			}
+		}
+	}
+	await bucket.save();
+}
+indexFiles('Kunkun Docs', ['/Users/hk/Dev/kunkun-docs/src/content/docs/developer/DX.mdx']);
+
+// const bucket = new Bucket(
+// 	'/Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/extensions_support',
+// 	'Kunkun Docs'
 // );
-// await bucket.addPDF("/Users/hk/Downloads/WACV_2025_Caroline_Huakun__Copy_.pdf");
+// await bucket.init();
+// const files = ['/Users/hk/Dev/kunkun-docs/src/content/docs/developer/manifest.mdx'];
+// for (const file of files) {
+// 	if (!existsSync(file)) {
+// 		throw new Error(`File ${file} does not exist`);
+// 	}
+// 	console.error('file', file);
+// 	// check if file is directory
+// 	const stats = Deno.statSync(file);
+// 	if (stats.isFile) {
+// 		const ext = path.extname(file);
+// 		if (txtExts.includes(ext)) {
+// 			console.error('Adding text file', file);
+// 			await bucket.addTextFile(file);
+// 		} else if (ext === '.pdf') {
+// 			console.error('Adding pdf file', file);
+// 			await bucket.addPDF(file);
+// 		} else if (stats.isDirectory) {
+// 			console.error('Adding directory', file);
+// 			await bucket.addDirectory(file);
+// 		} else {
+// 			throw new Error(`Unsupported file type: ${ext}`);
+// 		}
+// 	}
+// }
+// await bucket.save();

deno-src/index.ts

@@ -1,6 +1,48 @@
-import { expose } from "@kunkun/api/runtime/deno";
+import { expose } from '@kunkun/api/runtime/deno';
+import type { DenoAPI } from '../src/api.types.ts';
+import { Bucket } from './bucket.ts';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { txtExts } from './constants.ts';
+
+export const extensionSupportPath = Deno.env.get('EXTENSION_SUPPORT');
+if (!extensionSupportPath) {
+	throw new Error('EXTENSION_SUPPORT is not set');
+}
 
 expose({
-
+	async indexFiles(bucketName: string, files: string[]): Promise<void> {
-})
+		const cwd = Deno.cwd();
-
+		console.error('cwd', cwd);
+		const bucket = new Bucket(extensionSupportPath, bucketName);
+		// const bucket = new Bucket(extensionSupportPath, bucketName);
+		console.error('bucket path', bucket.bucketPath);
+		console.error('files', files);
+		await bucket.init();
+		for (const file of files) {
+			if (!existsSync(file)) {
+				throw new Error(`File ${file} does not exist`);
+			}
+			console.error('file', file);
+			// check if file is directory
+			const stats = Deno.statSync(file);
+			if (stats.isFile) {
+				const ext = path.extname(file);
+				if (txtExts.includes(ext)) {
+					console.error('Adding text file', file);
+					await bucket.addTextFile(file);
+					console.error('Finished adding text file', file);
+				} else if (ext === '.pdf') {
+					console.error('Adding pdf file', file);
+					await bucket.addPDF(file);
+				} else if (stats.isDirectory) {
+					console.error('Adding directory', file);
+					await bucket.addDirectory(file);
+				} else {
+					throw new Error(`Unsupported file type: ${ext}`);
+				}
+			}
+		}
+		await bucket.save();
+	}
+} satisfies DenoAPI);

package.json

@@ -17,7 +17,50 @@
 		"demoImages": [],
 		"permissions": [
 			"clipboard:read-text",
-			"notification:all"
+			"notification:all",
+			"dialog:all",
+			{
+				"permission": "fs:exists",
+				"allow": [
+					{
+						"path": "$EXTENSION/deno-src/node_modules"
+					}
+				]
+			},
+			{
+				"permission": "shell:execute",
+				"allow": [
+					{
+						"cmd": {
+							"program": "deno",
+							"args": [
+								"install",
+								"--allow-scripts"
+							]
+						}
+					}
+				]
+			},
+			{
+				"permission": "shell:deno:spawn",
+				"allow": [
+					{
+						"path": "$EXTENSION/deno-src/index.ts",
+						"read": "*",
+						"write": "*",
+						"ffi": "*",
+						"sys": [
+							"uid"
+						],
+						"env": "*",
+						"net": [
+							"api.openai.com"
+						]
+					}
+				]
+			},
+			"shell:stdin-write",
+			"shell:kill"
 		],
 		"customUiCmds": [
 			{

pnpm-lock.yaml

@@ -0,0 +1,13 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      '@kksh/api':
+        specifier: ^0.0.54
+        version: link:../../kunkun/packages/api

src/api.types.ts

@@ -0,0 +1,3 @@
+export interface DenoAPI {
+	indexFiles(bucketName: string, files: string[]): Promise<void>;
+}

src/lib/components/DatabaseList.svelte

@@ -24,6 +24,7 @@
 			>
 				Delete
 			</Button>
+			<a href={`/database/${dbInfo.id}`}><Button>Manage Database</Button></a>
 		</Card.Content>
 	</Card.Root>
 {/snippet}

src/lib/deno.ts

@@ -0,0 +1,56 @@
+import { log, fs, shell, path, toast } from '@kksh/api/ui/iframe';
+import type { DenoAPI } from '../api.types';
+
+export async function getRpcAPI(env: { OPENAI_API_KEY: string; EXTENSION_SUPPORT: string }) {
+	await installDenoDeps().catch((err) => {
+		return toast.error(`Failed to install deno dependencies; ${err.message}`);
+	});
+	const cwd = await path.join(await path.extensionDir(), 'deno-src');
+	console.log('cwd', cwd);
+	const { rpcChannel, process, command } = await shell.createDenoRpcChannel<object, DenoAPI>(
+		'$EXTENSION/deno-src/index.ts',
+		[],
+		{
+			cwd,
+			// allowAllEnv: true,
+			allowEnv: ['OPENAI_API_KEY', 'EXTENSION_SUPPORT', 'CWD'],
+			allowAllRead: true,
+			// allowRead: ['$EXTENSION_SUPPORT', '$EXTENSION/deno-src'],
+			// allowWrite: ['$EXTENSION_SUPPORT'],
+			allowAllWrite: true,
+			allowAllFfi: true, // /Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/deno-src/node_modules/.deno/faiss-node@0.5.1/node_modules/faiss-node/build/Release/faiss-node.node
+			// allowAllSys: true, // uid
+			allowSys: ['uid'],
+			allowNet: ['api.openai.com'],
+			env
+		},
+		{}
+	);
+	const api = rpcChannel.getAPI();
+	return {
+		api,
+		rpcChannel,
+		process,
+		command
+	};
+}
+
+export async function installDenoDeps() {
+	const cwd = await path.join(await path.extensionDir(), 'deno-src');
+	const nodeModulesPath = await path.join(cwd, 'node_modules');
+	console.log('nodeModulesPath', nodeModulesPath);
+	console.log('await fs.exists(nodeModulesPath)', await fs.exists(nodeModulesPath));
+
+	if (await fs.exists(nodeModulesPath)) {
+		console.log('Node modules already installed');
+		return;
+	}
+
+	const command = shell.createCommand('deno', ['install', '--allow-scripts'], { cwd });
+	const ret = await command.execute();
+	if (ret.code !== 0) {
+		// log.error(`Failed to install deno dependencies; ${ret.stderr}`);
+		console.error(`Failed to install deno dependencies; ${ret.stderr}`);
+		throw new Error('Failed to install deno dependencies', { cause: ret.stderr });
+	}
+}

src/routes/database/[id]/+page.svelte

@@ -0,0 +1,71 @@
+<script lang="ts">
+	import { dialog, path } from '@kksh/api/ui/iframe';
+	import { dbStore } from '@/stores/db';
+	import { Button } from '@kksh/svelte5';
+	import { getRpcAPI } from '@/deno';
+	import { goto } from '$app/navigation';
+	import { toast } from '@kksh/api/headless';
+	let { data } = $props();
+
+	const selectedDb = $dbStore.find((db) => db.id === data.id);
+	if (!selectedDb) {
+		toast.error('Database not found', { description: 'Name: ' + data.id });
+		goto('/');
+	}
+	let rpc: Awaited<ReturnType<typeof getRpcAPI>> | undefined;
+
+	async function indexFiles(files: string[]) {
+		if (!selectedDb) {
+			toast.error('Database not found', { description: 'Name: ' + data.id });
+			return goto('/');
+		}
+		try {
+			const extSupportDir = await path.extensionSupportDir();
+			rpc = await getRpcAPI({
+				OPENAI_API_KEY: selectedDb.apiKey,
+				EXTENSION_SUPPORT: extSupportDir
+			});
+			rpc.command.stderr.on('data', (data) => {
+				console.warn(data);
+			});
+			console.log('Start indexing files');
+			await rpc.api.indexFiles(selectedDb!.name, files);
+			console.log('Finished indexing files');
+		} catch (error) {
+			console.error('Error indexing files', error);
+			toast.error('Failed to index files');
+		} finally {
+			setTimeout(async () => {
+				await rpc?.process.kill();
+			}, 2_000);
+		}
+	}
+
+	async function addFiles() {
+		dialog
+			.open({
+				multiple: true,
+				directory: false
+			})
+			.then(async (res: string[]) => {
+				await indexFiles(res);
+			});
+	}
+
+	function addDirectory() {
+		dialog
+			.open({
+				multiple: true,
+				directory: true
+			})
+			.then(async (res: string[]) => {
+				await indexFiles(res);
+			});
+	}
+</script>
+
+<div class="container">
+	<h1 class="text-2xl font-bold">Manage Database</h1>
+	<Button onclick={addFiles}>Add Files</Button>
+	<Button onclick={addDirectory}>Add Directory</Button>
+</div>

src/routes/database/[id]/+page.ts

@@ -0,0 +1,5 @@
+import type { PageLoad } from './$types';
+
+export const load: PageLoad = ({ params: { id } }) => {
+	return { id: parseInt(id) };
+};