npm install @reaatech/llm-cache
# or
pnpm add @reaatech/llm-cache

import { CacheEngine, InMemoryAdapter, OpenAIEmbedder } from "@reaatech/llm-cache";
 
const cache = new CacheEngine({
  storage: new InMemoryAdapter(),
  vectorStorage: new InMemoryAdapter(),
  embedder: new OpenAIEmbedder({
    provider: "openai",
    model: "text-embedding-3-small",
    dimensions: 1536,
    apiKey: process.env.OPENAI_API_KEY,
  }),
  config: {
    storage: { adapter: "memory" },
    vectorStorage: { adapter: "memory" },
    embedding: {
      provider: "openai",
      model: "text-embedding-3-small",
      dimensions: 1536,
      batchSize: 100,
      maxRetries: 3,
    },
    similarity: { threshold: 0.8, metric: "cosine", maxResults: 10 },
    ttl: {
      default: 3600,
      factual: 1800,
      creative: 7200,
      analytical: 3600,
      sensitive: 600,
      byUseCase: {},
    },
    segmentation: { enabled: true, defaultUseCase: "general" },
    cost: { enabled: true, currency: "USD" },
    observability: { metrics: true, tracing: false, logging: "info" },
  },
});
 
// Store a response
await cache.set(
  "What is TypeScript?",
  { answer: "A typed superset of JavaScript" },
  { model: "gpt-4", modelVersion: "gpt-4-0613" },
);
 
// Exact match — < 1ms
const exact = await cache.get("What is TypeScript?", {
  model: "gpt-4",
  modelVersion: "gpt-4-0613",
});
// → { hit: true, type: "exact", entry: {...} }
 
// Semantic match — uses embedding similarity
const semantic = await cache.get("Tell me about TypeScript", {
  model: "gpt-4",
  modelVersion: "gpt-4-0613",
});
// → { hit: true, type: "semantic", confidence: 0.92, entry: {...} }

import { CacheEngine } from "@reaatech/llm-cache";
 
const engine = new CacheEngine({ storage, vectorStorage, embedder, config });

type CacheResult =
  | { hit: true; type: "exact" | "semantic"; entry: CacheEntry; confidence?: number; similarity?: number; cachedAt: Date; age: number }
  | { hit: false; reason: "not_found" | "below_threshold" | "expired" | "dimension_mismatch" };

// Each use case has an isolated cache namespace
await cache.set("classify: spam", { label: "spam" }, {
  model: "gpt-4",
  modelVersion: "gpt-4-0613",
  useCase: "classification",
});
 
// Same prompt in a different use case will miss
const result = await cache.get("classify: spam", {
  model: "gpt-4",
  modelVersion: "gpt-4-0613",
  useCase: "summarization",
});
// → { hit: false, reason: "not_found" }

const entry = await cache.set(
  "Patient: John Doe, SSN: 123-45-6789",
  response,
  { model: "gpt-4", modelVersion: "gpt-4-0613" },
  { sensitive: true, ttl: 600 },
);
// Entry gets the config's `sensitive` TTL (600s default)
// Encryption is applied if encryptionService is configured

// After upgrading from gpt-4 to gpt-4-turbo, invalidate old entries
const removed = await cache.invalidate({ modelVersion: "gpt-4-0613" });
console.log(`Cleared ${removed.total} old model entries`);
 
// New requests with gpt-4-turbo will generate fresh cache entries
const result = await cache.get(prompt, {
  model: "gpt-4",
  modelVersion: "gpt-4-turbo",
});