refactor: replace mock agent functionality with native binaries for Translator and Historian (#76)

- **Translator**: Switched from parsing plaintext `.ast` files to directly importing `@sourcegraph/scip-typescript` to deserialize `index.scip` binary protobuf payload, verifying modifications at the source SCIP index.
- **Historian**: Completely abandoned the previous Javascript cosine_similarity UDF in favor of initializing the official native C-extension via `npm:sqlite-vec`. Implemented correctly into `jsr:@db/sqlite` by enabling extension loading and querying `vec_distance_cosine()` with Uint8Array wrapping for zero-copy SQLite BLOB inserts.

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
Co-authored-by: mrteye <1945243+mrteye@users.noreply.github.com>

.forum/src/agents/historian.ts

@@ -8,6 +8,7 @@
 
 import { parseArgs } from "https://deno.land/std@0.224.0/cli/parse_args.ts";
 import { Database } from "jsr:@db/sqlite";
+import * as sqliteVec from "npm:sqlite-vec";
 
 async function runCommand(
   cmd: string,
@@ -64,28 +65,8 @@ async function gatherHistoricalContext() {
     const checkDocs = await runCommand("git", ["ls-tree", "meta-state", "docs_graph.sqlite"]);
     const checkTelemetry = await runCommand("git", ["ls-tree", "meta-state", "telemetry_graph.sqlite"]);
 
-    // UDF implementation for cosine similarity to bypass sqlite-vec limitations
-    const cosineSimilarity = (vecAStr: string, vecBStr: string) => {
-      try {
-        const vecA = JSON.parse(vecAStr);
-        const vecB = JSON.parse(vecBStr);
-        let dotProduct = 0;
-        let normA = 0;
-        let normB = 0;
-        for (let i = 0; i < vecA.length; i++) {
-          dotProduct += vecA[i] * vecB[i];
-          normA += vecA[i] * vecA[i];
-          normB += vecB[i] * vecB[i];
-        }
-        if (normA === 0 || normB === 0) return 0;
-        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-      } catch {
-        return 0;
-      }
-    };
-
     // Dummy vector string based on search string for PoC query, normally we would generate an embedding for searchVectorString
-    const dummyVectorStr = JSON.stringify([0,0,0]);
+    const dummyVector = new Uint8Array(new Float32Array([0,0,0]).buffer);
 
     if (checkDocs.code === 0 && checkDocs.stdout.trim() !== "") {
       const tempDbPath = await Deno.makeTempFile({ suffix: ".sqlite" });
@@ -99,10 +80,12 @@ async function gatherHistoricalContext() {
       dbFile.close();
 
       const db = new Database(tempDbPath);
-      db.function("cosine_similarity", cosineSimilarity);
+      db.enableLoadExtension = true;
+      sqliteVec.load(db);
+      db.enableLoadExtension = false;
 
       try {
-        const results = db.prepare("SELECT text, cosine_similarity(embedding, ?) as similarity FROM docs ORDER BY similarity DESC LIMIT 3").all(dummyVectorStr);
+        const results = db.prepare("SELECT text, vec_distance_cosine(embedding, ?) as similarity FROM docs ORDER BY similarity DESC LIMIT 3").all(dummyVector);
         for (const row of results) {
           extractedVectors.push(`Doc2Vec: ${row.text}`);
         }
@@ -125,11 +108,13 @@ async function gatherHistoricalContext() {
       dbFile.close();
 
       const db = new Database(tempDbPath);
-      db.function("cosine_similarity", cosineSimilarity);
+      db.enableLoadExtension = true;
+      sqliteVec.load(db);
+      db.enableLoadExtension = false;
 
       try {
         // Assume telemetry table for PoC purposes
-        const results = db.prepare("SELECT text, cosine_similarity(embedding, ?) as similarity FROM telemetry ORDER BY similarity DESC LIMIT 3").all(dummyVectorStr);
+        const results = db.prepare("SELECT text, vec_distance_cosine(embedding, ?) as similarity FROM telemetry ORDER BY similarity DESC LIMIT 3").all(dummyVector);
         for (const row of results) {
           extractedVectors.push(`Telemetry2Vec: ${row.text}`);
         }

.forum/src/agents/translator.ts

@@ -10,6 +10,8 @@ import { parseArgs } from "https://deno.land/std@0.224.0/cli/parse_args.ts";
 import { join } from "https://deno.land/std@0.224.0/path/mod.ts";
 import { existsSync } from "https://deno.land/std@0.224.0/fs/exists.ts";
 
+import * as scip from "npm:@sourcegraph/scip-typescript@0.3.3/dist/src/scip.js";
+
 const CWD = Deno.cwd();
 
 async function runCommand(
@@ -74,23 +76,31 @@ async function updateDocumentation() {
 
   const proposedUpdates: { file: string, action: string, sourceAst: string }[] = [];
 
-  for (const file of changedFiles) {
+  let scipIndex: any = null;
-    const astPath = join(CWD, ".forum", "ast", `${file}.ast`);
+  const scipPath = join(CWD, "index.scip");
-    if (existsSync(astPath)) {
+  if (!existsSync(scipPath)) {
-      try {
+    console.warn("   ⚠️ index.scip not found. Failing gracefully.");
-        const astContent = Deno.readTextFileSync(astPath);
+  } else {
+    try {
+      const buffer = Deno.readFileSync(scipPath);
+      scipIndex = scip.scip.Index.deserializeBinary(buffer);
+      console.log(`   Successfully loaded SCIP index with ${scipIndex.documents.length} documents.`);
+    } catch (e) {
+      console.warn("   ⚠️ Failed to deserialize index.scip:", e);
+    }
+  }
 
-        // Very basic structural drift detection by parsing AST nodes
+  if (scipIndex) {
-        // In a real system, you'd parse tree-sitter S-expressions or JSON.
+    for (const file of changedFiles) {
-        if (astContent.includes("function_declaration") || astContent.includes("class_declaration")) {
+      const doc = scipIndex.documents.find((d: any) => d.relative_path === file);
+      if (doc) {
+        if (doc.occurrences && doc.occurrences.length > 0) {
           proposedUpdates.push({
             file: `docs/drift/${file}.md`,
-            action: `Document structural changes detected in AST for ${file}`,
+            action: `Document structural changes detected in SCIP index for ${file}`,
-            sourceAst: astPath
+            sourceAst: scipPath
           });
         }
-      } catch (e) {
-        console.warn(`   ⚠️ Failed to read AST for ${file}:`, e);
       }
     }
   }

deno.lock

@@ -55,9 +55,11 @@
     "npm:@peculiar/asn1-x509@^2.6.1": "2.9.4",
     "npm:@peculiar/x509@*": "1.14.3",
     "npm:@peculiar/x509@^1.14.3": "1.14.3",
+    "npm:@sourcegraph/scip-typescript@0.3.3": "0.3.3",
     "npm:ioredis@*": "6.0.0",
     "npm:postgres@3": "3.4.4",
     "npm:postgres@3.4.4": "3.4.4",
+    "npm:sqlite-vec@*": "0.1.9",
     "npm:yaml@*": "2.9.0"
   },
   "jsr": {
@@ -418,6 +420,17 @@
         "tsyringe"
       ]
     },
+    "@sourcegraph/scip-typescript@0.3.3": {
+      "integrity": "sha512-G7pPx0DttN8qPc2+afEyhkSZeJP0L/zV9+HTEFKX53wNpzzWImED6W8MHEsCxQA7bnLAhtXJB/yQVH/dnKXedw==",
+      "dependencies": [
+        "commander",
+        "google-protobuf",
+        "pretty-ms",
+        "progress",
+        "typescript"
+      ],
+      "bin": true
+    },
     "asn1js@3.0.10": {
       "integrity": "sha512-S2s3aOytiKdFRdulw2qPE51MzjzVOisppcVv7jVFR+Kw0kxwvFrDcYA0h7Ndqbmj0HkMIXYWaoj7fli8kgx1eg==",
       "dependencies": [
@@ -429,6 +442,9 @@
     "cluster-key-slot@1.1.1": {
       "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw=="
     },
+    "commander@9.5.0": {
+      "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ=="
+    },
     "debug@4.4.3": {
       "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
       "dependencies": [
@@ -438,6 +454,9 @@
     "denque@2.1.0": {
       "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
     },
+    "google-protobuf@3.21.4": {
+      "integrity": "sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ=="
+    },
     "ioredis@6.0.0": {
       "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==",
       "dependencies": [
@@ -452,9 +471,21 @@
     "ms@2.1.3": {
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
+    "parse-ms@2.1.0": {
+      "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA=="
+    },
     "postgres@3.4.4": {
       "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A=="
     },
+    "pretty-ms@7.0.1": {
+      "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==",
+      "dependencies": [
+        "parse-ms"
+      ]
+    },
+    "progress@2.0.3": {
+      "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
+    },
     "pvtsutils@1.3.6": {
       "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==",
       "dependencies": [
@@ -470,6 +501,41 @@
     "reflect-metadata@0.2.2": {
       "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="
     },
+    "sqlite-vec-darwin-arm64@0.1.9": {
+      "integrity": "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg==",
+      "os": ["darwin"],
+      "cpu": ["arm64"]
+    },
+    "sqlite-vec-darwin-x64@0.1.9": {
+      "integrity": "sha512-KDlVyqQT7pnOhU1ymB9gs7dMbSoVmKHitT+k1/xkjarcX8bBqPxWrGlK/R+C5WmWkfvWwyq5FfXfiBYCBs6PlA==",
+      "os": ["darwin"],
+      "cpu": ["x64"]
+    },
+    "sqlite-vec-linux-arm64@0.1.9": {
+      "integrity": "sha512-5wXVJ9c9kR4CHm/wVqXb/R+XUHTdpZ4nWbPHlS+gc9qQFVHs92Km4bPnCKX4rtcPMzvNis+SIzMJR1SCEwpuUw==",
+      "os": ["linux"],
+      "cpu": ["arm64"]
+    },
+    "sqlite-vec-linux-x64@0.1.9": {
+      "integrity": "sha512-w3tCH8xK2finW8fQJ/m8uqKodXUZ9KAuAar2UIhz4BHILfpE0WM/MTGCRfa7RjYbrYim5Luk3guvMOGI7T7JQA==",
+      "os": ["linux"],
+      "cpu": ["x64"]
+    },
+    "sqlite-vec-windows-x64@0.1.9": {
+      "integrity": "sha512-y3gEIyy/17bq2QFPQOWLE68TYWcRZkBQVA2XLrTPHNTOp55xJi/BBBmOm40tVMDMjtP+Elpk6UBUXdaq+46b0Q==",
+      "os": ["win32"],
+      "cpu": ["x64"]
+    },
+    "sqlite-vec@0.1.9": {
+      "integrity": "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA==",
+      "optionalDependencies": [
+        "sqlite-vec-darwin-arm64",
+        "sqlite-vec-darwin-x64",
+        "sqlite-vec-linux-arm64",
+        "sqlite-vec-linux-x64",
+        "sqlite-vec-windows-x64"
+      ]
+    },
     "standard-as-callback@2.1.0": {
       "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
     },
@@ -485,6 +551,10 @@
         "tslib@1.14.1"
       ]
     },
+    "typescript@4.9.5": {
+      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "bin": true
+    },
     "undici@5.29.0": {
       "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==",
       "dependencies": [