Tyler Gillispie
4df94c2a19
- Updates `forum/DATA_STRUCTURES.md` with missing concepts: Protocol Buffers, TurboQuant, Git Merkle DAG Diffing, Dependency Graphing, and Declarative Frontmatter (UUIDv7). - Expands `forum/experiments/lab.ts` with 5 new proofs-of-concept for the new data structures. - Adds `protobuf_poc.ts`, `merkle_diff_poc.ts`, `vector_db_poc.ts`, `dependency_graph_poc.ts`, and `telemetry_poc.ts`. Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> Co-authored-by: mrteye <1945243+mrteye@users.noreply.github.com>
77 lines
2.0 KiB
TypeScript
77 lines
2.0 KiB
TypeScript
import {
|
|
assert,
|
|
assertEquals,
|
|
} from "https://deno.land/std@0.224.0/testing/asserts.ts";
|
|
|
|
/**
|
|
* Proof of Concept: Embedded Vector Database (Mocking sqlite-vec & TurboQuant)
|
|
*
|
|
* This module demonstrates the concept of hashing semantic concepts into vectors
|
|
* and performing cosine similarity to achieve fuzzy retrieval of associative memory
|
|
* without needing an external vector database.
|
|
*/
|
|
|
|
// Simple mock for cosine similarity of 1D arrays
|
|
function cosineSimilarity(vecA: number[], vecB: number[]): number {
|
|
let dotProduct = 0;
|
|
let normA = 0;
|
|
let normB = 0;
|
|
for (let i = 0; i < vecA.length; i++) {
|
|
dotProduct += vecA[i] * vecB[i];
|
|
normA += vecA[i] ** 2;
|
|
normB += vecB[i] ** 2;
|
|
}
|
|
if (normA === 0 || normB === 0) return 0;
|
|
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
}
|
|
|
|
// Mock of quantized vectors (e.g. TurboQuant 2-bit/4-bit compression)
|
|
const MOCK_VECTOR_DB = [
|
|
{ id: "docs-1", text: "How to run the server", vector: [0.8, 0.1, 0.1, 0.0] },
|
|
{
|
|
id: "docs-2",
|
|
text: "Database connection logic",
|
|
vector: [0.1, 0.9, 0.2, 0.1],
|
|
},
|
|
{
|
|
id: "telemetry-1",
|
|
text: "Server latency spikes",
|
|
vector: [0.2, 0.1, 0.9, 0.3],
|
|
},
|
|
];
|
|
|
|
if (import.meta.main) {
|
|
console.log("Running Embedded Vector Database PoC tests...");
|
|
|
|
try {
|
|
// Query representing "I have a slow server issue"
|
|
const queryVector = [0.3, 0.0, 0.9, 0.2];
|
|
|
|
console.log("Querying Vector DB...");
|
|
|
|
const results = MOCK_VECTOR_DB.map((doc) => ({
|
|
...doc,
|
|
score: cosineSimilarity(queryVector, doc.vector),
|
|
})).sort((a, b) => b.score - a.score);
|
|
|
|
console.log(
|
|
"Top result:",
|
|
results[0].text,
|
|
`(Score: ${results[0].score.toFixed(2)})`,
|
|
);
|
|
|
|
assert(
|
|
results[0].score > 0.8,
|
|
"The telemetry doc should be the highest match",
|
|
);
|
|
assertEquals(results[0].id, "telemetry-1");
|
|
|
|
console.log(
|
|
"✅ Embedded Vector DB PoC successful: Fuzzy semantic match found.",
|
|
);
|
|
} catch (err) {
|
|
console.error("❌ Embedded Vector DB PoC failed:", err);
|
|
Deno.exit(1);
|
|
}
|
|
}
|