mirror of
https://github.com/kunkunsh/kunkun-ext-rag.git
synced 2025-07-10 06:21:30 +00:00
feat: add deno rpc integration for the extension
This commit is contained in:
parent
6665dccc2b
commit
19a98d0aaa
@ -1,42 +1,37 @@
|
|||||||
import { FaissStore } from "@langchain/community/vectorstores/faiss";
|
import { FaissStore } from '@langchain/community/vectorstores/faiss';
|
||||||
import { OpenAIEmbeddings } from "@langchain/openai";
|
import { OpenAIEmbeddings } from '@langchain/openai';
|
||||||
import * as v from "valibot";
|
import * as v from 'valibot';
|
||||||
import * as path from "jsr:@std/path";
|
import * as path from 'jsr:@std/path';
|
||||||
import { existsSync } from "node:fs";
|
import { existsSync, readdirSync } from 'node:fs';
|
||||||
import { Document } from "@langchain/core/documents";
|
import { Document } from '@langchain/core/documents';
|
||||||
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
|
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
||||||
import { DirectoryLoader } from "langchain/document_loaders/fs/directory";
|
import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
|
||||||
import {
|
import { JSONLoader, JSONLinesLoader } from 'langchain/document_loaders/fs/json';
|
||||||
JSONLoader,
|
import { TextLoader } from 'langchain/document_loaders/fs/text';
|
||||||
JSONLinesLoader,
|
import { computeSha256FromText } from './crypto.ts';
|
||||||
} from "langchain/document_loaders/fs/json";
|
import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
|
||||||
import { TextLoader } from "langchain/document_loaders/fs/text";
|
|
||||||
import { computeSha256FromText } from "./crypto.ts";
|
|
||||||
import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf";
|
|
||||||
|
|
||||||
export const embeddings = new OpenAIEmbeddings({
|
export const embeddings = new OpenAIEmbeddings({
|
||||||
model: "text-embedding-3-large",
|
model: 'text-embedding-3-large'
|
||||||
});
|
});
|
||||||
|
|
||||||
export const MetadataSchema = v.object({
|
export const MetadataSchema = v.object({
|
||||||
filesSha256: v.array(v.string()),
|
filesSha256: v.array(v.string())
|
||||||
});
|
});
|
||||||
export type Metadata = v.InferOutput<typeof MetadataSchema>;
|
export type Metadata = v.InferOutput<typeof MetadataSchema>;
|
||||||
|
|
||||||
export async function getDocsFromDirectory(
|
export async function getDocsFromDirectory(directoryPath: string): Promise<Document[]> {
|
||||||
directoryPath: string
|
|
||||||
): Promise<Document[]> {
|
|
||||||
const splitter = new RecursiveCharacterTextSplitter({
|
const splitter = new RecursiveCharacterTextSplitter({
|
||||||
chunkSize: 1000,
|
chunkSize: 1000,
|
||||||
chunkOverlap: 200,
|
chunkOverlap: 200
|
||||||
});
|
});
|
||||||
|
|
||||||
const loader = new DirectoryLoader(directoryPath, {
|
const loader = new DirectoryLoader(directoryPath, {
|
||||||
".json": (path) => new JSONLoader(path, "/texts"),
|
'.json': (path) => new JSONLoader(path, '/texts'),
|
||||||
".jsonl": (path) => new JSONLinesLoader(path, "/html"),
|
'.jsonl': (path) => new JSONLinesLoader(path, '/html'),
|
||||||
".txt": (path) => new TextLoader(path),
|
'.txt': (path) => new TextLoader(path),
|
||||||
".md": (path) => new TextLoader(path),
|
'.md': (path) => new TextLoader(path),
|
||||||
".mdx": (path) => new TextLoader(path),
|
'.mdx': (path) => new TextLoader(path)
|
||||||
});
|
});
|
||||||
const docs = await loader.load();
|
const docs = await loader.load();
|
||||||
const allSplits = await splitter.splitDocuments(docs);
|
const allSplits = await splitter.splitDocuments(docs);
|
||||||
@ -44,19 +39,19 @@ export async function getDocsFromDirectory(
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class Bucket {
|
export class Bucket {
|
||||||
private readonly bucketPath: string;
|
readonly bucketPath: string;
|
||||||
private readonly faissStorePath: string;
|
readonly faissStorePath: string;
|
||||||
private readonly metadataPath: string;
|
readonly metadataPath: string;
|
||||||
private _vectorStore: FaissStore | null = null;
|
private _vectorStore: FaissStore | null = null;
|
||||||
filesSha256: Set<string> = new Set();
|
filesSha256: Set<string> = new Set();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly bucketDir: string,
|
readonly bucketDir: string,
|
||||||
private readonly bucketName: string
|
readonly bucketName: string
|
||||||
) {
|
) {
|
||||||
this.bucketPath = path.join(this.bucketDir, this.bucketName);
|
this.bucketPath = path.join(this.bucketDir, this.bucketName);
|
||||||
this.faissStorePath = path.join(this.bucketPath, "faiss-store");
|
this.faissStorePath = path.join(this.bucketPath, 'faiss-store');
|
||||||
this.metadataPath = path.join(this.bucketPath, "metadata.json");
|
this.metadataPath = path.join(this.bucketPath, 'metadata.json');
|
||||||
}
|
}
|
||||||
|
|
||||||
async init() {
|
async init() {
|
||||||
@ -69,7 +64,7 @@ export class Bucket {
|
|||||||
if (parsedMetadata.success) {
|
if (parsedMetadata.success) {
|
||||||
this.filesSha256 = new Set(parsedMetadata.output.filesSha256);
|
this.filesSha256 = new Set(parsedMetadata.output.filesSha256);
|
||||||
} else {
|
} else {
|
||||||
throw new Error("Invalid metadata");
|
throw new Error('Invalid metadata');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.updateMetadata();
|
this.updateMetadata();
|
||||||
@ -81,7 +76,7 @@ export class Bucket {
|
|||||||
|
|
||||||
updateMetadata() {
|
updateMetadata() {
|
||||||
const metadata: Metadata = {
|
const metadata: Metadata = {
|
||||||
filesSha256: Array.from(this.filesSha256),
|
filesSha256: Array.from(this.filesSha256)
|
||||||
};
|
};
|
||||||
Deno.writeTextFileSync(this.metadataPath, JSON.stringify(metadata));
|
Deno.writeTextFileSync(this.metadataPath, JSON.stringify(metadata));
|
||||||
}
|
}
|
||||||
@ -89,12 +84,9 @@ export class Bucket {
|
|||||||
async getVectorStore() {
|
async getVectorStore() {
|
||||||
if (
|
if (
|
||||||
existsSync(this.faissStorePath) &&
|
existsSync(this.faissStorePath) &&
|
||||||
existsSync(path.join(this.faissStorePath, "docstore.json"))
|
existsSync(path.join(this.faissStorePath, 'docstore.json'))
|
||||||
) {
|
) {
|
||||||
const vectorStore = await FaissStore.load(
|
const vectorStore = await FaissStore.load(this.faissStorePath, embeddings);
|
||||||
this.faissStorePath,
|
|
||||||
embeddings
|
|
||||||
);
|
|
||||||
return vectorStore;
|
return vectorStore;
|
||||||
}
|
}
|
||||||
// const vectorStore = await FaissStore.fromDocuments(docs, embeddings);
|
// const vectorStore = await FaissStore.fromDocuments(docs, embeddings);
|
||||||
@ -105,14 +97,14 @@ export class Bucket {
|
|||||||
|
|
||||||
get vectorStore() {
|
get vectorStore() {
|
||||||
if (this._vectorStore === null) {
|
if (this._vectorStore === null) {
|
||||||
throw new Error("Vector store not initialized");
|
throw new Error('Vector store not initialized');
|
||||||
}
|
}
|
||||||
return this._vectorStore;
|
return this._vectorStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async addDocuments(documents: Document[]) {
|
private async addDocuments(documents: Document[]) {
|
||||||
await this.vectorStore.addDocuments(documents);
|
await this.vectorStore.addDocuments(documents);
|
||||||
await this.vectorStore.save(this.faissStorePath);
|
// await this.vectorStore.save(this.faissStorePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
private fillSha256(documents: Document[]) {
|
private fillSha256(documents: Document[]) {
|
||||||
@ -123,9 +115,7 @@ export class Bucket {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private getFilteredDocs(documents: Document[]) {
|
private getFilteredDocs(documents: Document[]) {
|
||||||
return documents.filter(
|
return documents.filter((doc) => !this.filesSha256.has(doc.metadata.sha256));
|
||||||
(doc) => !this.filesSha256.has(doc.metadata.sha256)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private updateSha256(docs: Document[]) {
|
private updateSha256(docs: Document[]) {
|
||||||
@ -137,12 +127,12 @@ export class Bucket {
|
|||||||
|
|
||||||
async addDirectory(directoryPath: string) {
|
async addDirectory(directoryPath: string) {
|
||||||
if (!existsSync(directoryPath)) {
|
if (!existsSync(directoryPath)) {
|
||||||
throw new Error("Directory does not exist");
|
throw new Error('Directory does not exist');
|
||||||
}
|
}
|
||||||
// check if path is a directory or file
|
// check if path is a directory or file
|
||||||
const stats = Deno.statSync(directoryPath);
|
const stats = Deno.statSync(directoryPath);
|
||||||
if (stats.isFile) {
|
if (stats.isFile) {
|
||||||
throw new Error("Path is a file");
|
throw new Error('Path is a file');
|
||||||
}
|
}
|
||||||
|
|
||||||
const docs = await getDocsFromDirectory(directoryPath);
|
const docs = await getDocsFromDirectory(directoryPath);
|
||||||
@ -154,7 +144,7 @@ export class Bucket {
|
|||||||
this.updateMetadata();
|
this.updateMetadata();
|
||||||
const splitter = new RecursiveCharacterTextSplitter({
|
const splitter = new RecursiveCharacterTextSplitter({
|
||||||
chunkSize: 1000,
|
chunkSize: 1000,
|
||||||
chunkOverlap: 200,
|
chunkOverlap: 200
|
||||||
});
|
});
|
||||||
const allSplits = await splitter.splitDocuments(fileteredDocs);
|
const allSplits = await splitter.splitDocuments(fileteredDocs);
|
||||||
await this.addDocuments(allSplits);
|
await this.addDocuments(allSplits);
|
||||||
@ -163,10 +153,27 @@ export class Bucket {
|
|||||||
async addTextFile(filePath: string) {
|
async addTextFile(filePath: string) {
|
||||||
const loader = new TextLoader(filePath);
|
const loader = new TextLoader(filePath);
|
||||||
const docs = await loader.load();
|
const docs = await loader.load();
|
||||||
|
console.error('Loaded docs', docs.length);
|
||||||
this.fillSha256(docs);
|
this.fillSha256(docs);
|
||||||
const fileteredDocs = this.getFilteredDocs(docs);
|
const fileteredDocs = this.getFilteredDocs(docs);
|
||||||
|
console.error('Filtered docs', fileteredDocs.length);
|
||||||
this.updateSha256(docs);
|
this.updateSha256(docs);
|
||||||
await this.addDocuments(fileteredDocs);
|
console.error('Updated sha256', this.filesSha256.size);
|
||||||
|
// await this.addDocuments(fileteredDocs);
|
||||||
|
return this.vectorStore.addDocuments(fileteredDocs).catch((err) => {
|
||||||
|
console.error('Error adding documents', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async save() {
|
||||||
|
console.error('Save Bucket', this.vectorStore.docstore._docs.size);
|
||||||
|
if (existsSync(this.faissStorePath) && readdirSync(this.faissStorePath).length === 0) {
|
||||||
|
Deno.removeSync(this.bucketPath, { recursive: true });
|
||||||
|
}
|
||||||
|
if (this.vectorStore.docstore._docs.size === 0) {
|
||||||
|
throw new Error('No documents to save');
|
||||||
|
}
|
||||||
|
await this.vectorStore.save(this.faissStorePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
async addPDF(filePath: string) {
|
async addPDF(filePath: string) {
|
||||||
|
File diff suppressed because one or more lines are too long
Binary file not shown.
@ -1 +0,0 @@
|
|||||||
{"filesSha256":["bcecd6e1f4c1296ebaed37b2686a55485855bb3d04d71a16609297899ef89ba9","f67eb8d401f74305c8a4f21b2ac8986c1db3850aabe2753773ba00255fd4288c","61614a8362e20d8dc96bc8bcd52cd40b9bbf983c862253b9472f1044744bea02","f34a6fbd2e0f3fc1a1a8f3c019935e53e809d38b18fdab67ec742e02e3318a6f","e6929768ff182e0c6714a60157e94420c7942fa2cdad1f201a8302be9e86d064","b19540b04d609f5798e91d6de8ecd0fbbee9f7c3dfb008e2a198247241401c57","324d4f9a84e4849c002ae4091fe7614d2d00cb0bf4989574b307081669f9eb7a","0618796260fd0b8b4cc6daedbac59c06492a74eeebefbddf69fb13b2d2456da8","1a5d9b3ea54168ec8f89e1a93950b1d5c527c831b951070d3ffcce853c6e797f","0cc8092a90aa3f614a87258d372348706bc2f0d33e9235630849cb0aeec4342e","90ef40d14982b96bb2c4c657e17dbf74d8b15f1a36712d2759765c9e5de94995","ec62822e3eeffec14393b16a12d0dcb0920c11ad4fa1a3a6c59252ef39321c57","53f3b51bad507e0799121d30f1a96cb458d6caacda20ee6bd6447aa7a2041772","bc7551da68e709d023b76979f554252467465066285bb4c7aad07e05be82833c","a2fc83c128ebafcfc444c3740a3afcedc2e92b8c10f51724d41da57b8f670546","04c793bf24afcd608ecbbb3606ba1280602f02c678bb885b2e75f423ce394e22","3fe4c596a0bedd833fbe214081467647786a12561e3af0e4cd88a43c2787f35f","95f0df47bfd2b698f43aac7dc22ac13948fe093f838a25ab63c5f6f9b789d942","a9b8caefaa7a959928c587f74399a86549b7680894c1c882a631725a5252881e","ab68d62cb3631acf57b0704996303f2704730dd27fb30fd867092c8af90f4fb5","926688c2cc2bf5214f8ad2f82bdd68a630f0226118f50c54f8d618c2417f4c9f","4e52cee59ad6b3220e31dee7e2c9da499ce2f6c6a6a16305fefc4d05c0d63e08","c28379be7109e76bfccffcf7268b1e767c54e5f279179c773ddb33ba1a83717f","451ecba7690493bf70d26bf127bff2de09d7d5f0b51c16c363f3f372184a2988","f0b6f639135d28f12914f4fd9d8e7f11aa3c67e736cbbfbda36ec97f382b4577","a7516e047d1287385581727d5b019512288207bd2172bd241a0c6255d2b570c0","d67a47978e51dec70271dde449d9d38695a73c9787432379ab13a3b4ef78d7eb","ffbcb29742daf9184167d0e12e191e672b2d0662af402e8dfdea3f33117eb388","c19ae6410ab85759fbba1eb7c0ed49cb4f7a62759288ade5a287714e57f17823","a508c2976b25ed28733d26d2cd7601ee67aff3ad8cc0f1faf221e2ba23f325d4"]}
|
|
1
deno-src/constants.ts
Normal file
1
deno-src/constants.ts
Normal file
@ -0,0 +1 @@
|
|||||||
|
export const txtExts = ['.txt', '.md', '.mdx'];
|
@ -1,20 +1,77 @@
|
|||||||
import { FaissStore } from "@langchain/community/vectorstores/faiss";
|
// import { FaissStore } from '@langchain/community/vectorstores/faiss';
|
||||||
import { Bucket, embeddings, getDocsFromDirectory } from "./bucket.ts";
|
// import { Bucket, embeddings, getDocsFromDirectory } from './bucket.ts';
|
||||||
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
|
// import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
||||||
import { DirectoryLoader } from "langchain/document_loaders/fs/directory";
|
// import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
|
||||||
import {
|
// import { JSONLoader, JSONLinesLoader } from 'langchain/document_loaders/fs/json';
|
||||||
JSONLoader,
|
// import { TextLoader } from 'langchain/document_loaders/fs/text';
|
||||||
JSONLinesLoader,
|
// import { OpenAIEmbeddings } from '@langchain/openai';
|
||||||
} from "langchain/document_loaders/fs/json";
|
import { existsSync } from 'node:fs';
|
||||||
import { TextLoader } from "langchain/document_loaders/fs/text";
|
// import path from 'path';
|
||||||
import { OpenAIEmbeddings } from "@langchain/openai";
|
import { txtExts } from './constants.ts';
|
||||||
|
import { Bucket } from './bucket.ts';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
const bucket = new Bucket(
|
async function indexFiles(bucketName: string, files: string[]): Promise<void> {
|
||||||
"/Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/deno-src/buckets",
|
const bucket = new Bucket('./store', bucketName);
|
||||||
"dev"
|
// const bucket = new Bucket(extensionSupportPath, bucketName);
|
||||||
);
|
console.error('bucket path', bucket.bucketPath);
|
||||||
|
console.error('files', files);
|
||||||
await bucket.init();
|
await bucket.init();
|
||||||
// await bucket.addDirectory(
|
for (const file of files) {
|
||||||
// "/Users/hk/Dev/kunkun-docs/src/content/docs/guides/Extensions/Publish"
|
if (!existsSync(file)) {
|
||||||
|
throw new Error(`File ${file} does not exist`);
|
||||||
|
}
|
||||||
|
console.error('file', file);
|
||||||
|
// check if file is directory
|
||||||
|
const stats = Deno.statSync(file);
|
||||||
|
if (stats.isFile) {
|
||||||
|
const ext = path.extname(file);
|
||||||
|
if (txtExts.includes(ext)) {
|
||||||
|
console.error('Adding text file', file);
|
||||||
|
await bucket.addTextFile(file);
|
||||||
|
console.error('Finished adding text file', file);
|
||||||
|
} else if (ext === '.pdf') {
|
||||||
|
console.error('Adding pdf file', file);
|
||||||
|
await bucket.addPDF(file);
|
||||||
|
} else if (stats.isDirectory) {
|
||||||
|
console.error('Adding directory', file);
|
||||||
|
await bucket.addDirectory(file);
|
||||||
|
} else {
|
||||||
|
throw new Error(`Unsupported file type: ${ext}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await bucket.save();
|
||||||
|
}
|
||||||
|
indexFiles('Kunkun Docs', ['/Users/hk/Dev/kunkun-docs/src/content/docs/developer/DX.mdx']);
|
||||||
|
|
||||||
|
// const bucket = new Bucket(
|
||||||
|
// '/Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/extensions_support',
|
||||||
|
// 'Kunkun Docs'
|
||||||
// );
|
// );
|
||||||
// await bucket.addPDF("/Users/hk/Downloads/WACV_2025_Caroline_Huakun__Copy_.pdf");
|
// await bucket.init();
|
||||||
|
// const files = ['/Users/hk/Dev/kunkun-docs/src/content/docs/developer/manifest.mdx'];
|
||||||
|
// for (const file of files) {
|
||||||
|
// if (!existsSync(file)) {
|
||||||
|
// throw new Error(`File ${file} does not exist`);
|
||||||
|
// }
|
||||||
|
// console.error('file', file);
|
||||||
|
// // check if file is directory
|
||||||
|
// const stats = Deno.statSync(file);
|
||||||
|
// if (stats.isFile) {
|
||||||
|
// const ext = path.extname(file);
|
||||||
|
// if (txtExts.includes(ext)) {
|
||||||
|
// console.error('Adding text file', file);
|
||||||
|
// await bucket.addTextFile(file);
|
||||||
|
// } else if (ext === '.pdf') {
|
||||||
|
// console.error('Adding pdf file', file);
|
||||||
|
// await bucket.addPDF(file);
|
||||||
|
// } else if (stats.isDirectory) {
|
||||||
|
// console.error('Adding directory', file);
|
||||||
|
// await bucket.addDirectory(file);
|
||||||
|
// } else {
|
||||||
|
// throw new Error(`Unsupported file type: ${ext}`);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// await bucket.save();
|
||||||
|
@ -1,6 +1,48 @@
|
|||||||
import { expose } from "@kunkun/api/runtime/deno";
|
import { expose } from '@kunkun/api/runtime/deno';
|
||||||
|
import type { DenoAPI } from '../src/api.types.ts';
|
||||||
|
import { Bucket } from './bucket.ts';
|
||||||
|
import { existsSync } from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { txtExts } from './constants.ts';
|
||||||
|
|
||||||
|
export const extensionSupportPath = Deno.env.get('EXTENSION_SUPPORT');
|
||||||
|
if (!extensionSupportPath) {
|
||||||
|
throw new Error('EXTENSION_SUPPORT is not set');
|
||||||
|
}
|
||||||
|
|
||||||
expose({
|
expose({
|
||||||
|
async indexFiles(bucketName: string, files: string[]): Promise<void> {
|
||||||
})
|
const cwd = Deno.cwd();
|
||||||
|
console.error('cwd', cwd);
|
||||||
|
const bucket = new Bucket(extensionSupportPath, bucketName);
|
||||||
|
// const bucket = new Bucket(extensionSupportPath, bucketName);
|
||||||
|
console.error('bucket path', bucket.bucketPath);
|
||||||
|
console.error('files', files);
|
||||||
|
await bucket.init();
|
||||||
|
for (const file of files) {
|
||||||
|
if (!existsSync(file)) {
|
||||||
|
throw new Error(`File ${file} does not exist`);
|
||||||
|
}
|
||||||
|
console.error('file', file);
|
||||||
|
// check if file is directory
|
||||||
|
const stats = Deno.statSync(file);
|
||||||
|
if (stats.isFile) {
|
||||||
|
const ext = path.extname(file);
|
||||||
|
if (txtExts.includes(ext)) {
|
||||||
|
console.error('Adding text file', file);
|
||||||
|
await bucket.addTextFile(file);
|
||||||
|
console.error('Finished adding text file', file);
|
||||||
|
} else if (ext === '.pdf') {
|
||||||
|
console.error('Adding pdf file', file);
|
||||||
|
await bucket.addPDF(file);
|
||||||
|
} else if (stats.isDirectory) {
|
||||||
|
console.error('Adding directory', file);
|
||||||
|
await bucket.addDirectory(file);
|
||||||
|
} else {
|
||||||
|
throw new Error(`Unsupported file type: ${ext}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await bucket.save();
|
||||||
|
}
|
||||||
|
} satisfies DenoAPI);
|
||||||
|
45
package.json
45
package.json
@ -17,7 +17,50 @@
|
|||||||
"demoImages": [],
|
"demoImages": [],
|
||||||
"permissions": [
|
"permissions": [
|
||||||
"clipboard:read-text",
|
"clipboard:read-text",
|
||||||
"notification:all"
|
"notification:all",
|
||||||
|
"dialog:all",
|
||||||
|
{
|
||||||
|
"permission": "fs:exists",
|
||||||
|
"allow": [
|
||||||
|
{
|
||||||
|
"path": "$EXTENSION/deno-src/node_modules"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"permission": "shell:execute",
|
||||||
|
"allow": [
|
||||||
|
{
|
||||||
|
"cmd": {
|
||||||
|
"program": "deno",
|
||||||
|
"args": [
|
||||||
|
"install",
|
||||||
|
"--allow-scripts"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"permission": "shell:deno:spawn",
|
||||||
|
"allow": [
|
||||||
|
{
|
||||||
|
"path": "$EXTENSION/deno-src/index.ts",
|
||||||
|
"read": "*",
|
||||||
|
"write": "*",
|
||||||
|
"ffi": "*",
|
||||||
|
"sys": [
|
||||||
|
"uid"
|
||||||
|
],
|
||||||
|
"env": "*",
|
||||||
|
"net": [
|
||||||
|
"api.openai.com"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"shell:stdin-write",
|
||||||
|
"shell:kill"
|
||||||
],
|
],
|
||||||
"customUiCmds": [
|
"customUiCmds": [
|
||||||
{
|
{
|
||||||
|
13
pnpm-lock.yaml
generated
Normal file
13
pnpm-lock.yaml
generated
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
lockfileVersion: '9.0'
|
||||||
|
|
||||||
|
settings:
|
||||||
|
autoInstallPeers: true
|
||||||
|
excludeLinksFromLockfile: false
|
||||||
|
|
||||||
|
importers:
|
||||||
|
|
||||||
|
.:
|
||||||
|
dependencies:
|
||||||
|
'@kksh/api':
|
||||||
|
specifier: ^0.0.54
|
||||||
|
version: link:../../kunkun/packages/api
|
3
src/api.types.ts
Normal file
3
src/api.types.ts
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
export interface DenoAPI {
|
||||||
|
indexFiles(bucketName: string, files: string[]): Promise<void>;
|
||||||
|
}
|
@ -24,6 +24,7 @@
|
|||||||
>
|
>
|
||||||
Delete
|
Delete
|
||||||
</Button>
|
</Button>
|
||||||
|
<a href={`/database/${dbInfo.id}`}><Button>Manage Database</Button></a>
|
||||||
</Card.Content>
|
</Card.Content>
|
||||||
</Card.Root>
|
</Card.Root>
|
||||||
{/snippet}
|
{/snippet}
|
||||||
|
56
src/lib/deno.ts
Normal file
56
src/lib/deno.ts
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
import { log, fs, shell, path, toast } from '@kksh/api/ui/iframe';
|
||||||
|
import type { DenoAPI } from '../api.types';
|
||||||
|
|
||||||
|
export async function getRpcAPI(env: { OPENAI_API_KEY: string; EXTENSION_SUPPORT: string }) {
|
||||||
|
await installDenoDeps().catch((err) => {
|
||||||
|
return toast.error(`Failed to install deno dependencies; ${err.message}`);
|
||||||
|
});
|
||||||
|
const cwd = await path.join(await path.extensionDir(), 'deno-src');
|
||||||
|
console.log('cwd', cwd);
|
||||||
|
const { rpcChannel, process, command } = await shell.createDenoRpcChannel<object, DenoAPI>(
|
||||||
|
'$EXTENSION/deno-src/index.ts',
|
||||||
|
[],
|
||||||
|
{
|
||||||
|
cwd,
|
||||||
|
// allowAllEnv: true,
|
||||||
|
allowEnv: ['OPENAI_API_KEY', 'EXTENSION_SUPPORT', 'CWD'],
|
||||||
|
allowAllRead: true,
|
||||||
|
// allowRead: ['$EXTENSION_SUPPORT', '$EXTENSION/deno-src'],
|
||||||
|
// allowWrite: ['$EXTENSION_SUPPORT'],
|
||||||
|
allowAllWrite: true,
|
||||||
|
allowAllFfi: true, // /Users/hk/Dev/kunkun-extension-repos/kunkun-ext-rag/deno-src/node_modules/.deno/faiss-node@0.5.1/node_modules/faiss-node/build/Release/faiss-node.node
|
||||||
|
// allowAllSys: true, // uid
|
||||||
|
allowSys: ['uid'],
|
||||||
|
allowNet: ['api.openai.com'],
|
||||||
|
env
|
||||||
|
},
|
||||||
|
{}
|
||||||
|
);
|
||||||
|
const api = rpcChannel.getAPI();
|
||||||
|
return {
|
||||||
|
api,
|
||||||
|
rpcChannel,
|
||||||
|
process,
|
||||||
|
command
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function installDenoDeps() {
|
||||||
|
const cwd = await path.join(await path.extensionDir(), 'deno-src');
|
||||||
|
const nodeModulesPath = await path.join(cwd, 'node_modules');
|
||||||
|
console.log('nodeModulesPath', nodeModulesPath);
|
||||||
|
console.log('await fs.exists(nodeModulesPath)', await fs.exists(nodeModulesPath));
|
||||||
|
|
||||||
|
if (await fs.exists(nodeModulesPath)) {
|
||||||
|
console.log('Node modules already installed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const command = shell.createCommand('deno', ['install', '--allow-scripts'], { cwd });
|
||||||
|
const ret = await command.execute();
|
||||||
|
if (ret.code !== 0) {
|
||||||
|
// log.error(`Failed to install deno dependencies; ${ret.stderr}`);
|
||||||
|
console.error(`Failed to install deno dependencies; ${ret.stderr}`);
|
||||||
|
throw new Error('Failed to install deno dependencies', { cause: ret.stderr });
|
||||||
|
}
|
||||||
|
}
|
71
src/routes/database/[id]/+page.svelte
Normal file
71
src/routes/database/[id]/+page.svelte
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { dialog, path } from '@kksh/api/ui/iframe';
|
||||||
|
import { dbStore } from '@/stores/db';
|
||||||
|
import { Button } from '@kksh/svelte5';
|
||||||
|
import { getRpcAPI } from '@/deno';
|
||||||
|
import { goto } from '$app/navigation';
|
||||||
|
import { toast } from '@kksh/api/headless';
|
||||||
|
let { data } = $props();
|
||||||
|
|
||||||
|
const selectedDb = $dbStore.find((db) => db.id === data.id);
|
||||||
|
if (!selectedDb) {
|
||||||
|
toast.error('Database not found', { description: 'Name: ' + data.id });
|
||||||
|
goto('/');
|
||||||
|
}
|
||||||
|
let rpc: Awaited<ReturnType<typeof getRpcAPI>> | undefined;
|
||||||
|
|
||||||
|
async function indexFiles(files: string[]) {
|
||||||
|
if (!selectedDb) {
|
||||||
|
toast.error('Database not found', { description: 'Name: ' + data.id });
|
||||||
|
return goto('/');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const extSupportDir = await path.extensionSupportDir();
|
||||||
|
rpc = await getRpcAPI({
|
||||||
|
OPENAI_API_KEY: selectedDb.apiKey,
|
||||||
|
EXTENSION_SUPPORT: extSupportDir
|
||||||
|
});
|
||||||
|
rpc.command.stderr.on('data', (data) => {
|
||||||
|
console.warn(data);
|
||||||
|
});
|
||||||
|
console.log('Start indexing files');
|
||||||
|
await rpc.api.indexFiles(selectedDb!.name, files);
|
||||||
|
console.log('Finished indexing files');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error indexing files', error);
|
||||||
|
toast.error('Failed to index files');
|
||||||
|
} finally {
|
||||||
|
setTimeout(async () => {
|
||||||
|
await rpc?.process.kill();
|
||||||
|
}, 2_000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function addFiles() {
|
||||||
|
dialog
|
||||||
|
.open({
|
||||||
|
multiple: true,
|
||||||
|
directory: false
|
||||||
|
})
|
||||||
|
.then(async (res: string[]) => {
|
||||||
|
await indexFiles(res);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function addDirectory() {
|
||||||
|
dialog
|
||||||
|
.open({
|
||||||
|
multiple: true,
|
||||||
|
directory: true
|
||||||
|
})
|
||||||
|
.then(async (res: string[]) => {
|
||||||
|
await indexFiles(res);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class="container">
|
||||||
|
<h1 class="text-2xl font-bold">Manage Database</h1>
|
||||||
|
<Button onclick={addFiles}>Add Files</Button>
|
||||||
|
<Button onclick={addDirectory}>Add Directory</Button>
|
||||||
|
</div>
|
5
src/routes/database/[id]/+page.ts
Normal file
5
src/routes/database/[id]/+page.ts
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
import type { PageLoad } from './$types';
|
||||||
|
|
||||||
|
export const load: PageLoad = ({ params: { id } }) => {
|
||||||
|
return { id: parseInt(id) };
|
||||||
|
};
|
Loading…
x
Reference in New Issue
Block a user