diff --git a/.forum/src/agents/historian.ts b/.forum/src/agents/historian.ts index a417cd4..c1e796e 100644 --- a/.forum/src/agents/historian.ts +++ b/.forum/src/agents/historian.ts @@ -8,6 +8,7 @@ import { parseArgs } from "https://deno.land/std@0.224.0/cli/parse_args.ts"; import { Database } from "jsr:@db/sqlite"; +import * as sqliteVec from "npm:sqlite-vec"; async function runCommand( cmd: string, @@ -64,28 +65,8 @@ async function gatherHistoricalContext() { const checkDocs = await runCommand("git", ["ls-tree", "meta-state", "docs_graph.sqlite"]); const checkTelemetry = await runCommand("git", ["ls-tree", "meta-state", "telemetry_graph.sqlite"]); - // UDF implementation for cosine similarity to bypass sqlite-vec limitations - const cosineSimilarity = (vecAStr: string, vecBStr: string) => { - try { - const vecA = JSON.parse(vecAStr); - const vecB = JSON.parse(vecBStr); - let dotProduct = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < vecA.length; i++) { - dotProduct += vecA[i] * vecB[i]; - normA += vecA[i] * vecA[i]; - normB += vecB[i] * vecB[i]; - } - if (normA === 0 || normB === 0) return 0; - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); - } catch { - return 0; - } - }; - // Dummy vector string based on search string for PoC query, normally we would generate an embedding for searchVectorString - const dummyVectorStr = JSON.stringify([0,0,0]); + const dummyVector = new Uint8Array(new Float32Array([0,0,0]).buffer); if (checkDocs.code === 0 && checkDocs.stdout.trim() !== "") { const tempDbPath = await Deno.makeTempFile({ suffix: ".sqlite" }); @@ -99,10 +80,12 @@ async function gatherHistoricalContext() { dbFile.close(); const db = new Database(tempDbPath); - db.function("cosine_similarity", cosineSimilarity); + db.enableLoadExtension = true; + sqliteVec.load(db); + db.enableLoadExtension = false; try { - const results = db.prepare("SELECT text, cosine_similarity(embedding, ?) as similarity FROM docs ORDER BY similarity DESC LIMIT 3").all(dummyVectorStr); + const results = db.prepare("SELECT text, vec_distance_cosine(embedding, ?) as similarity FROM docs ORDER BY similarity DESC LIMIT 3").all(dummyVector); for (const row of results) { extractedVectors.push(`Doc2Vec: ${row.text}`); } @@ -125,11 +108,13 @@ async function gatherHistoricalContext() { dbFile.close(); const db = new Database(tempDbPath); - db.function("cosine_similarity", cosineSimilarity); + db.enableLoadExtension = true; + sqliteVec.load(db); + db.enableLoadExtension = false; try { // Assume telemetry table for PoC purposes - const results = db.prepare("SELECT text, cosine_similarity(embedding, ?) as similarity FROM telemetry ORDER BY similarity DESC LIMIT 3").all(dummyVectorStr); + const results = db.prepare("SELECT text, vec_distance_cosine(embedding, ?) as similarity FROM telemetry ORDER BY similarity DESC LIMIT 3").all(dummyVector); for (const row of results) { extractedVectors.push(`Telemetry2Vec: ${row.text}`); } diff --git a/.forum/src/agents/translator.ts b/.forum/src/agents/translator.ts index e4994b0..a5930ad 100644 --- a/.forum/src/agents/translator.ts +++ b/.forum/src/agents/translator.ts @@ -10,6 +10,8 @@ import { parseArgs } from "https://deno.land/std@0.224.0/cli/parse_args.ts"; import { join } from "https://deno.land/std@0.224.0/path/mod.ts"; import { existsSync } from "https://deno.land/std@0.224.0/fs/exists.ts"; +import * as scip from "npm:@sourcegraph/scip-typescript@0.3.3/dist/src/scip.js"; + const CWD = Deno.cwd(); async function runCommand( @@ -74,23 +76,31 @@ async function updateDocumentation() { const proposedUpdates: { file: string, action: string, sourceAst: string }[] = []; - for (const file of changedFiles) { - const astPath = join(CWD, ".forum", "ast", `${file}.ast`); - if (existsSync(astPath)) { - try { - const astContent = Deno.readTextFileSync(astPath); + let scipIndex: any = null; + const scipPath = join(CWD, "index.scip"); + if (!existsSync(scipPath)) { + console.warn(" ⚠️ index.scip not found. Failing gracefully."); + } else { + try { + const buffer = Deno.readFileSync(scipPath); + scipIndex = scip.scip.Index.deserializeBinary(buffer); + console.log(` Successfully loaded SCIP index with ${scipIndex.documents.length} documents.`); + } catch (e) { + console.warn(" ⚠️ Failed to deserialize index.scip:", e); + } + } - // Very basic structural drift detection by parsing AST nodes - // In a real system, you'd parse tree-sitter S-expressions or JSON. - if (astContent.includes("function_declaration") || astContent.includes("class_declaration")) { + if (scipIndex) { + for (const file of changedFiles) { + const doc = scipIndex.documents.find((d: any) => d.relative_path === file); + if (doc) { + if (doc.occurrences && doc.occurrences.length > 0) { proposedUpdates.push({ file: `docs/drift/${file}.md`, - action: `Document structural changes detected in AST for ${file}`, - sourceAst: astPath + action: `Document structural changes detected in SCIP index for ${file}`, + sourceAst: scipPath }); } - } catch (e) { - console.warn(` ⚠️ Failed to read AST for ${file}:`, e); } } } diff --git a/deno.lock b/deno.lock index 56de2a7..973bd22 100644 --- a/deno.lock +++ b/deno.lock @@ -55,9 +55,11 @@ "npm:@peculiar/asn1-x509@^2.6.1": "2.9.4", "npm:@peculiar/x509@*": "1.14.3", "npm:@peculiar/x509@^1.14.3": "1.14.3", + "npm:@sourcegraph/scip-typescript@0.3.3": "0.3.3", "npm:ioredis@*": "6.0.0", "npm:postgres@3": "3.4.4", "npm:postgres@3.4.4": "3.4.4", + "npm:sqlite-vec@*": "0.1.9", "npm:yaml@*": "2.9.0" }, "jsr": { @@ -418,6 +420,17 @@ "tsyringe" ] }, + "@sourcegraph/scip-typescript@0.3.3": { + "integrity": "sha512-G7pPx0DttN8qPc2+afEyhkSZeJP0L/zV9+HTEFKX53wNpzzWImED6W8MHEsCxQA7bnLAhtXJB/yQVH/dnKXedw==", + "dependencies": [ + "commander", + "google-protobuf", + "pretty-ms", + "progress", + "typescript" + ], + "bin": true + }, "asn1js@3.0.10": { "integrity": "sha512-S2s3aOytiKdFRdulw2qPE51MzjzVOisppcVv7jVFR+Kw0kxwvFrDcYA0h7Ndqbmj0HkMIXYWaoj7fli8kgx1eg==", "dependencies": [ @@ -429,6 +442,9 @@ "cluster-key-slot@1.1.1": { "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==" }, + "commander@9.5.0": { + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" + }, "debug@4.4.3": { "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": [ @@ -438,6 +454,9 @@ "denque@2.1.0": { "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, + "google-protobuf@3.21.4": { + "integrity": "sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ==" + }, "ioredis@6.0.0": { "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==", "dependencies": [ @@ -452,9 +471,21 @@ "ms@2.1.3": { "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "parse-ms@2.1.0": { + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + }, "postgres@3.4.4": { "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==" }, + "pretty-ms@7.0.1": { + "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "dependencies": [ + "parse-ms" + ] + }, + "progress@2.0.3": { + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + }, "pvtsutils@1.3.6": { "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", "dependencies": [ @@ -470,6 +501,41 @@ "reflect-metadata@0.2.2": { "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==" }, + "sqlite-vec-darwin-arm64@0.1.9": { + "integrity": "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "sqlite-vec-darwin-x64@0.1.9": { + "integrity": "sha512-KDlVyqQT7pnOhU1ymB9gs7dMbSoVmKHitT+k1/xkjarcX8bBqPxWrGlK/R+C5WmWkfvWwyq5FfXfiBYCBs6PlA==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "sqlite-vec-linux-arm64@0.1.9": { + "integrity": "sha512-5wXVJ9c9kR4CHm/wVqXb/R+XUHTdpZ4nWbPHlS+gc9qQFVHs92Km4bPnCKX4rtcPMzvNis+SIzMJR1SCEwpuUw==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "sqlite-vec-linux-x64@0.1.9": { + "integrity": "sha512-w3tCH8xK2finW8fQJ/m8uqKodXUZ9KAuAar2UIhz4BHILfpE0WM/MTGCRfa7RjYbrYim5Luk3guvMOGI7T7JQA==", + "os": ["linux"], + "cpu": ["x64"] + }, + "sqlite-vec-windows-x64@0.1.9": { + "integrity": "sha512-y3gEIyy/17bq2QFPQOWLE68TYWcRZkBQVA2XLrTPHNTOp55xJi/BBBmOm40tVMDMjtP+Elpk6UBUXdaq+46b0Q==", + "os": ["win32"], + "cpu": ["x64"] + }, + "sqlite-vec@0.1.9": { + "integrity": "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA==", + "optionalDependencies": [ + "sqlite-vec-darwin-arm64", + "sqlite-vec-darwin-x64", + "sqlite-vec-linux-arm64", + "sqlite-vec-linux-x64", + "sqlite-vec-windows-x64" + ] + }, "standard-as-callback@2.1.0": { "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, @@ -485,6 +551,10 @@ "tslib@1.14.1" ] }, + "typescript@4.9.5": { + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "bin": true + }, "undici@5.29.0": { "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", "dependencies": [