import { createDeduplicator } from 'memory-dedup';

// Provide any embedding function
const dedup = createDeduplicator({
  embedder: async (text) => {
    const response = await openai.embeddings.create({
      model: 'text-embedding-3-small',
      input: text,
    });
    return response.data[0].embedding;
  },
  threshold: 0.90,
  mergePolicy: 'keep-newest',
});

// Add entries with automatic deduplication
const result1 = await dedup.add({
  id: 'entry-1',
  content: 'User lives in New York City',
  metadata: { timestamp: Date.now(), source: 'conversation-10' },
});
// result1.action === 'added', result1.classification === 'unique'

const result2 = await dedup.add({
  id: 'entry-2',
  content: 'User resides in NYC',
  metadata: { timestamp: Date.now(), source: 'conversation-42' },
});
// result2.classification === 'semantic_duplicate'
// result2.action === 'merged', result2.survivorId === 'entry-2'

// Check without modifying the index
const checkResult = await dedup.check({
  id: 'entry-3',
  content: 'User is based in New York',
});
// checkResult.classification === 'semantic_duplicate'

// Get deduplicated entries
const entries = dedup.getEntries();

// View statistics
const stats = dedup.stats();
console.log(`Dedup rate: ${stats.exactDuplicates + stats.semanticDuplicates} / ${stats.totalChecks}`);

const unsub = dedup.on('duplicate-found', (payload) => {
  console.log('Duplicate detected:', payload);
});

dedup.on('merged', (payload) => {
  console.log('Entries merged:', payload);
});

dedup.on('evicted', (payload) => {
  console.log('Entry evicted:', payload);
});

dedup.on('added', (payload) => {
  console.log('Entry added:', payload);
});

// Unsubscribe when done
unsub();

const dedup = createDeduplicator({
  embedder: myEmbedder,
  store: myCustomStore, // implements StoreBackend interface
});

import { createDeduplicator } from 'memory-dedup';

const dedup = createDeduplicator({
  embedder: async (text) => { /* return number[] */ },
  threshold: 0.90,
  mergePolicy: 'keep-newest',
});

const result = await dedup.check({ id: 'e1', content: 'User likes coffee' });

const result = await dedup.add({
  id: 'e2',
  content: 'User enjoys coffee',
  metadata: { confidence: 0.9 },
});

const batch = await dedup.addBatch([
  { id: 'e1', content: 'User likes tea' },
  { id: 'e2', content: 'User enjoys tea' },
  { id: 'e3', content: 'Server runs on port 3000' },
]);

const result = await dedup.sweep();

const result = await dedup.compact();

const entries = dedup.getEntries();

dedup.remove('entry-1');

dedup.clear();

const s = dedup.stats();

const count = dedup.size();

const unsub = dedup.on('duplicate-found', (payload) => {
  console.log(payload);
});
unsub(); // remove the handler

dedup.off('duplicate-found', myHandler);

interface MemoryEntry {
  /** Unique identifier for this entry. */
  id: string;
  /** The text content of the memory entry. Embedded and compared for similarity. */
  content: string;
  /** Optional metadata for merge decisions, provenance, and similarity boosting. */
  metadata?: Record<string, unknown>;
}

interface EntryMetadata {
  /** Unix timestamp (ms) when the entry was created. */
  timestamp?: number;
  /** Unix timestamp (ms), alias for timestamp. */
  createdAt?: number;
  /** Confidence score from extraction (0.0 to 1.0). */
  confidence?: number;
  /** Additional caller-defined fields. */
  [key: string]: unknown;
}

interface DedupOptions {
  embedder: (text: string) => Promise<number[]>;
  threshold?: number;            // default: 0.90
  exactThreshold?: number;       // default: 0.98
  relatedThreshold?: number;     // default: 0.75
  mergePolicy?:
    | 'keep-newest'
    | 'keep-oldest'
    | 'keep-longest'
    | 'keep-highest-confidence'
    | 'merge'
    | ((a: MemoryEntry, b: MemoryEntry, sim: number) => MemoryEntry);
  store?: StoreBackend;          // default: InMemoryStore
}

interface DedupResult {
  classification: 'exact_duplicate' | 'semantic_duplicate' | 'related' | 'unique';
  matchId?: string;
  similarity?: number;
  hashMatch?: boolean;
  durationMs: number;
}

interface AddResult extends DedupResult {
  action: 'added' | 'merged' | 'skipped';
  survivorId?: string;
  evictedId?: string;
}

interface BatchResult {
  results: AddResult[];
  totalProcessed: number;
  uniqueAdded: number;
  duplicatesFound: number;
  durationMs: number;
}

interface SweepResult {
  duplicatePairs: Array<[string, string]>;
  duplicateCount: number;
  evictedCount: number;
  evictedIds: string[];
  totalScanned: number;
  durationMs: number;
}

interface CompactResult extends SweepResult {
  clustersFound: number;
  mergedCount: number;
}

interface DedupStats {
  totalEntries: number;
  totalChecks: number;
  exactDuplicates: number;
  semanticDuplicates: number;
  uniqueEntries: number;
  durationMs?: number;
}

interface StoreBackend {
  add(entry: MemoryEntry, embedding: number[], hash: string): void;
  get(id: string): MemoryEntry | null;
  remove(id: string): void;
  all(): MemoryEntry[];
  getEmbedding(id: string): number[] | null;
  getHash(hash: string): string | null;
  size(): number;
  clear(): void;
}

const dedup = createDeduplicator({
  embedder: myEmbedder,
  threshold: 0.92,         // semantic duplicate threshold
  exactThreshold: 0.99,    // exact duplicate threshold
  relatedThreshold: 0.80,  // related-but-different threshold
});

try {
  await dedup.add({ id: 'e1', content: 'Some fact' });
} catch (err) {
  // Handle embedder or store errors
  console.error('Dedup failed:', err.message);
}

const dedup = createDeduplicator({
  embedder: myEmbedder,
  mergePolicy: (candidate, match, similarity) => {
    // Keep the entry with more metadata fields
    const candidateKeys = Object.keys(candidate.metadata ?? {}).length;
    const matchKeys = Object.keys(match.metadata ?? {}).length;
    return candidateKeys >= matchKeys ? candidate : match;
  },
});

import { createDeduplicator } from 'memory-dedup';
import { createEmbedCache } from 'embed-cache';

const cache = createEmbedCache({ maxSize: 10_000 });

const dedup = createDeduplicator({
  embedder: async (text) => {
    const cached = cache.get(text);
    if (cached) return cached;
    const embedding = await rawEmbedder(text);
    cache.set(text, embedding);
    return embedding;
  },
});

import OpenAI from 'openai';
import { createDeduplicator } from 'memory-dedup';

const openai = new OpenAI();

const dedup = createDeduplicator({
  embedder: async (text) => {
    const response = await openai.embeddings.create({
      model: 'text-embedding-3-small',
      input: text,
    });
    return response.data[0].embedding;
  },
});

import { CohereClientV2 } from 'cohere-ai';
import { createDeduplicator } from 'memory-dedup';

const cohere = new CohereClientV2();

const dedup = createDeduplicator({
  embedder: async (text) => {
    const response = await cohere.embed({
      texts: [text],
      model: 'embed-english-v3.0',
      inputType: 'search_document',
      embeddingTypes: ['float'],
    });
    return response.embeddings.float[0];
  },
});

import { pipeline } from '@xenova/transformers';
import { createDeduplicator } from 'memory-dedup';

const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');

const dedup = createDeduplicator({
  embedder: async (text) => {
    const output = await extractor(text, { pooling: 'mean', normalize: true });
    return Array.from(output.data);
  },
});

import { createDeduplicator, StoreBackend, MemoryEntry } from 'memory-dedup';

class MyStore implements StoreBackend {
  private entries = new Map<string, MemoryEntry>();
  private embeddings = new Map<string, number[]>();
  private hashes = new Map<string, string>();

  add(entry: MemoryEntry, embedding: number[], hash: string): void {
    this.entries.set(entry.id, entry);
    this.embeddings.set(entry.id, embedding);
    this.hashes.set(hash, entry.id);
  }

  get(id: string): MemoryEntry | null {
    return this.entries.get(id) ?? null;
  }

  remove(id: string): void {
    this.entries.delete(id);
    this.embeddings.delete(id);
  }

  all(): MemoryEntry[] {
    return Array.from(this.entries.values());
  }

  getEmbedding(id: string): number[] | null {
    return this.embeddings.get(id) ?? null;
  }

  getHash(hash: string): string | null {
    return this.hashes.get(hash) ?? null;
  }

  size(): number {
    return this.entries.size;
  }

  clear(): void {
    this.entries.clear();
    this.embeddings.clear();
    this.hashes.clear();
  }
}

const dedup = createDeduplicator({
  embedder: myEmbedder,
  store: new MyStore(),
});

setInterval(async () => {
  const result = await dedup.sweep();
  console.log(
    `Sweep: scanned=${result.totalScanned}, ` +
    `duplicates=${result.duplicateCount}, ` +
    `evicted=${result.evictedCount}`
  );
}, 5 * 60 * 1000);

import {
  createDeduplicator,
  // Types
  type MemoryEntry,
  type EntryMetadata,
  type DedupOptions,
  type DedupResult,
  type AddResult,
  type BatchResult,
  type SweepResult,
  type CompactResult,
  type DedupStats,
  type MemoryDedup,
  type StoreBackend,
} from 'memory-dedup';