Build your own Glean with HydraDB

// Example: Setting up Composio connector
const composioConfig = {
  connectors: [
    {
      name: 'slack',
      config: {
        token: process.env.SLACK_BOT_TOKEN,
        channels: ['general', 'random', 'project-*']
      }
    },
    {
      name: 'gmail',
      config: {
        credentials: process.env.GMAIL_CREDENTIALS,
        labels: ['INBOX', 'SENT', 'IMPORTANT']
      }
    },
    {
      name: 'notion',
      config: {
        token: process.env.NOTION_TOKEN,
        databases: ['projects', 'docs', 'meetings']
      }
    }
  ]
};

// Example: Slack API integration
class SlackConnector {
  constructor(token) {
    this.token = token;
    this.client = new WebClient(token);
  }

  async fetchMessages(channelId, limit = 100) {
    const result = await this.client.conversations.history({
      channel: channelId,
      limit: limit
    });
    
    return result.messages.map(msg => ({
      id: msg.ts,
      text: msg.text,
      user: msg.user,
      timestamp: msg.ts,
      channel: channelId,
      type: 'slack_message'
    }));
  }

  async fetchChannels() {
    const result = await this.client.conversations.list();
    return result.channels;
  }
}

// Example: Gmail API integration (npm install googleapis)
// Reads plain-text message bodies; add HTML handling for HTML-only emails.
import { google } from 'googleapis';

class GmailConnector {
  constructor(credentials) {
    this.gmail = google.gmail({ version: 'v1', auth: credentials });
  }

  async fetchEmails(query = 'in:inbox', maxResults = 100) {
    const response = await this.gmail.users.messages.list({
      userId: 'me',
      q: query,
      maxResults: maxResults
    });

    const emails = [];
    for (const message of response.data.messages || []) {
      const email = await this.gmail.users.messages.get({
        userId: 'me',
        id: message.id
      });
      
      emails.push({
        id: email.data.id,
        subject: this.getHeader(email.data.payload.headers, 'Subject'),
        from: this.getHeader(email.data.payload.headers, 'From'),
        body: this.getBody(email.data.payload),
        timestamp: email.data.internalDate,
        type: 'gmail'
      });
    }
    return emails;
  }

  getHeader(headers = [], name) {
    return headers.find(h => h.name.toLowerCase() === name.toLowerCase())?.value || '';
  }

  getBody(payload) {
    if (payload.mimeType === 'text/plain' && payload.body?.data) {
      return Buffer.from(payload.body.data, 'base64url').toString('utf8');
    }
    return (payload.parts || []).map(part => this.getBody(part)).filter(Boolean).join('\n');
  }
}

// Unified data structure for HydraDB app upload
const normalizedData = {
  id: 'unique_id',
  database: 'your_database',
  collection: 'your_collection',
  title: 'Document/Message Title',
  timestamp: '2024-01-01T00:00:00Z', // ISO timestamp
  content: {
    text: 'Main content text',
    html_base64: 'base64_encoded_html',
    markdown: 'markdown_content'
  },
  url: 'https://app.com/item/123', // Optional: source URL
  description: 'Optional description of the source', // Optional
  metadata: {}, // Optional database-level metadata
  additional_metadata: {
    source_category: 'slack_message',
    source_label: 'Document/Message Title',
    author: 'user@company.com',
    id: 'original_id',
    tags: ['project-a', 'urgent', 'meeting-notes']
  },
  acl: ['user_email:user1@company.com', 'user_email:user2@company.com']
};

class PreferenceLearner {
  constructor() {
    this.preferencePatterns = new Map();
  }

  async learnFromInteraction(userId, interaction) {
    const {
      query,
      selectedResults,
      responseFormat,
      followUpQueries
    } = interaction;

    const patterns = this.preferencePatterns.get(userId) || {
      queryPatterns: [],
      formatPreferences: {},
      sourcePreferences: {}
    };

    // Learn query patterns
    this.learnQueryPatterns(patterns, query, followUpQueries);
    
    // Learn format preferences
    this.learnFormatPreferences(patterns, responseFormat, selectedResults);
    
    // Learn source preferences
    this.learnSourcePreferences(patterns, selectedResults);
    
    this.preferencePatterns.set(userId, patterns);
  }

  learnQueryPatterns(patterns, query, followUpQueries) {
    // Analyze query complexity, length, and type
    const queryAnalysis = {
      length: query.length,
      complexity: this.analyzeComplexity(query),
      type: this.classifyQueryType(query),
      hasFilters: query.includes('in:') || query.includes('from:'),
      timestamp: new Date().toISOString()
    };

    patterns.queryPatterns.push(queryAnalysis);
    
    // Keep only recent patterns
    if (patterns.queryPatterns.length > 100) {
      patterns.queryPatterns = patterns.queryPatterns.slice(-100);
    }
  }

  learnFormatPreferences(patterns, format, selectedResults) {
    if (!patterns.formatPreferences[format]) {
      patterns.formatPreferences[format] = 0;
    }
    patterns.formatPreferences[format]++;
  }

  learnSourcePreferences(patterns, selectedResults) {
    selectedResults.forEach(result => {
      const sourceType = result.additionalMetadata?.source_category;
      if (!sourceType) return;
      if (!patterns.sourcePreferences[sourceType]) {
        patterns.sourcePreferences[sourceType] = 0;
      }
      patterns.sourcePreferences[sourceType]++;
    });
  }

  analyzeComplexity(query) {
    const words = query.split(' ').length;
    const hasQuotes = query.includes('"');
    const hasOperators = /AND|OR|NOT|in:|from:|to:/.test(query);
    
    let complexity = 'simple';
    if (words > 5 || hasQuotes || hasOperators) complexity = 'complex';
    if (words > 10 || (hasQuotes && hasOperators)) complexity = 'advanced';
    
    return complexity;
  }

  classifyQueryType(query) {
    if (query.includes('how to') || query.includes('steps')) return 'how-to';
    if (query.includes('what is') || query.includes('define')) return 'definition';
    if (query.includes('when') || query.includes('schedule')) return 'temporal';
    if (query.includes('who') || query.includes('contact')) return 'person';
    if (query.includes('where') || query.includes('location')) return 'location';
    return 'general';
  }
}

const MemoryEnhancedSearch = ({ userId, searchApi }) => {
  const [query, setQuery] = useState('');
  const [results, setResults] = useState(null);
  const [userPreferences, setUserPreferences] = useState(null);
  const [suggestions, setSuggestions] = useState([]);

  // Profile helpers come from 2.3. searchApi calls your authenticated backend.
  const preferenceLearner = new PreferenceLearner();

  useEffect(() => {
    // Load user preferences on component mount
    loadUserPreferences();
  }, [userId]);

  const loadUserPreferences = async () => {
    const profile = getUserProfile(userId);
    setUserPreferences(profile);
    
    // Generate search suggestions based on user's history
    const suggestions = generateSuggestions(profile);
    setSuggestions(suggestions);
  };

  const generateSuggestions = (profile) => {
    const suggestions = [];
    
    // Suggest based on frequent queries
    if (profile.frequentQueries.length > 0) {
      suggestions.push({
        type: 'frequent',
        text: profile.frequentQueries[0],
        label: 'Frequently searched'
      });
    }
    
    // Suggest based on recent searches
    if (profile.searchHistory.length > 0) {
      const recentSearches = profile.searchHistory
        .slice(-3)
        .map(h => h.query);
      
      recentSearches.forEach(query => {
        suggestions.push({
          type: 'recent',
          text: query,
          label: 'Recent search'
        });
      });
    }
    
    return suggestions;
  };

  const handleSearch = async () => {
    const startTime = Date.now();
    
    try {
      const searchResults = await searchApi(query);
      setResults(searchResults);
      
      // Learn from this interaction
      const interaction = {
        query,
        selectedResults: [], // Will be populated when user clicks results
        responseFormat: 'default',
        searchFilters: {},
        timeSpent: Date.now() - startTime,
        followUpQueries: []
      };
      
      await preferenceLearner.learnFromInteraction(userId, interaction);
      
    } catch (error) {
      console.error('Search failed:', error);
    }
  };

  const handleResultClick = async (result) => {
    // Update user preferences when they click on results
    const profile = getUserProfile(userId);
    
    // Mark this source type as preferred
    const sourceCategory = result.additionalMetadata?.source_category;
    if (sourceCategory && !profile.preferredSourceTypes.includes(sourceCategory)) {
      profile.preferredSourceTypes.push(sourceCategory);
      await saveUserProfile(userId, profile);
    }
  };

  return (
    <div className="memory-enhanced-search">
      <div className="search-header">
        <input
          type="text"
          value={query}
          onChange={(e) => setQuery(e.target.value)}
          placeholder="Search with your preferences in mind..."
          className="search-input"
        />
        <button onClick={handleSearch}>Search</button>
      </div>

      {/* Show personalized suggestions */}
      {suggestions.length > 0 && (
        <div className="search-suggestions">
          <h4>Based on your preferences:</h4>
          {suggestions.map((suggestion, index) => (
            <button
              key={index}
              className="suggestion-chip"
              onClick={() => setQuery(suggestion.text)}
            >
              {suggestion.text}
              <span className="suggestion-label">{suggestion.label}</span>
            </button>
          ))}
        </div>
      )}

      {/* Show user preferences summary */}
      {userPreferences && (
        <div className="user-preferences">
          <details>
            <summary>Your Search Preferences</summary>
            <div className="preferences-content">
              <p><strong>Preferred sources:</strong> {userPreferences.preferredSourceTypes.join(', ') || 'None set'}</p>
              <p><strong>Response style:</strong> {userPreferences.responseStyle}</p>
              <p><strong>Frequent searches:</strong> {userPreferences.frequentQueries.slice(0, 3).join(', ')}</p>
            </div>
          </details>
        </div>
      )}

      {results && (
        <SearchResults 
          results={results} 
          onResultClick={handleResultClick}
          userPreferences={userPreferences}
        />
      )}
    </div>
  );
};

// React component example
import React, { useState, useEffect } from 'react';

const GleanSearchInterface = ({ searchApi }) => {
  const [query, setQuery] = useState('');
  const [results, setResults] = useState(null);
  const [loading, setLoading] = useState(false);
  const [filters, setFilters] = useState({
    sourceTypes: [],
    dateRange: null,
    authors: []
  });

  const handleSearch = async () => {
    setLoading(true);
    try {
      const searchResults = await searchApi(query, { sourceTypes: filters.sourceTypes });
      setResults(searchResults);
    } catch (error) {
      console.error('Search failed:', error);
    } finally {
      setLoading(false);
    }
  };

  return (
    <div className="glean-search">
      <div className="search-header">
        <input
          type="text"
          value={query}
          onChange={(e) => setQuery(e.target.value)}
          placeholder="Search across all your work..."
          className="search-input"
        />
        <button onClick={handleSearch} disabled={loading}>
          {loading ? 'Searching...' : 'Search'}
        </button>
      </div>

      <div className="search-filters">
        <SourceTypeFilter
          value={filters.sourceTypes}
          onChange={(types) => setFilters({...filters, sourceTypes: types})}
        />
        <DateRangeFilter
          value={filters.dateRange}
          onChange={(range) => setFilters({...filters, dateRange: range})}
        />
      </div>

      {results && (
        <SearchResults results={results} />
      )}
    </div>
  );
};

const SearchResults = ({ results }) => {
  const { chunks, graphContext } = results.data ?? {};

  return (
    <div className="search-results">
      {chunks && chunks.length > 0 && (
        <div className="source-results">
          <h3>Results ({chunks.length})</h3>
          {chunks.map((chunk, index) => (
            <ChunkCard key={index} chunk={chunk} />
          ))}
        </div>
      )}

      {graphContext && graphContext.queryPaths && (
        <div className="graph-context">
          <h3>Related Knowledge Graph Paths</h3>
          <pre>{JSON.stringify(graphContext.queryPaths, null, 2)}</pre>
        </div>
      )}
    </div>
  );
};

const ChunkCard = ({ chunk }) => {
  return (
    <div className="source-card">
      <div className="source-header">
        <span className="source-type">{chunk.sourceType}</span>
        <span className="source-title">{chunk.sourceTitle}</span>
        <span className="source-date">{formatDate(chunk.sourceUploadTime)}</span>
      </div>

      <div className="source-chunk">
        <p>{chunk.chunkContent}</p>
      </div>
    </div>
  );
};

class DataSyncManager {
  constructor(connectors, database, collection) {
    this.connectors = connectors;
    this.database = database;
    this.collection = collection;
    this.syncIntervals = {
      slack: 5 * 60 * 1000, // 5 minutes
      gmail: 10 * 60 * 1000, // 10 minutes
      notion: 30 * 60 * 1000, // 30 minutes
      documents: 60 * 60 * 1000 // 1 hour
    };
  }

  async startSync() {
    // Start sync jobs for each connector
    Object.entries(this.syncIntervals).forEach(([connector, interval]) => {
      setInterval(() => {
        this.syncConnector(connector);
      }, interval);
    });
  }

  async syncConnector(connectorName) {
    try {
      const connector = this.connectors[connectorName];
      const newData = await connector.fetchNewData();
      
      if (newData.length > 0) {
        const normalizedData = newData.map(item => 
          this.normalizeData(item, connectorName)
        );
        
        // Use batch upload with verification (uploadWithVerification from 1.3)
        await uploadWithVerification(normalizedData, this.database, this.collection);
        console.log(`Synced ${newData.length} items from ${connectorName}`);
      }
    } catch (error) {
      console.error(`Sync failed for ${connectorName}:`, error);
      // Implement retry logic with exponential backoff
      await this.retrySync(connectorName, error);
    }
  }

  async retrySync(connectorName, error, attempt = 1) {
    const maxAttempts = 3;
    const delay = Math.pow(2, attempt) * 1000; // Exponential backoff
    
    if (attempt < maxAttempts) {
      console.log(`Retrying sync for ${connectorName} in ${delay}ms (attempt ${attempt + 1})`);
      setTimeout(() => {
        this.syncConnector(connectorName);
      }, delay);
    } else {
      console.error(`Max retry attempts reached for ${connectorName}:`, error);
    }
  }

  normalizeData(item, sourceType) {
    return {
      id: `${sourceType}_${item.id}`,
      title: item.title || item.subject || item.text?.substring(0, 100),
      type: sourceType,
      timestamp: item.timestamp || item.created_at || new Date().toISOString(),
      content: {
        text: item.text || item.body || item.content,
        html_base64: item.html ? btoa(item.html) : '',
        markdown: item.markdown || ''
      },
      url: item.url,
      description: item.description,
      metadata: {},
      additional_metadata: {
        id: item.id,
        author: item.author || item.user,
        tags: item.tags || [],
        created_at: item.created_at,
        updated_at: item.updated_at
      }
    };
  }
}

class TenantManager {
  constructor(hasPermission) {
    if (typeof hasPermission !== "function") {
      throw new TypeError("Pass your application permission-checking function");
    }
    this.hasPermission = hasPermission;
    this.tenants = new Map();
  }

  async createTenant(database, config) {
    const tenant = {
      id: database,
      subTenants: new Map(),
      permissions: config.permissions || {},
      dataSources: config.dataSources || []
    };
    
    this.tenants.set(database, tenant);
    return tenant;
  }

  async createSubTenant(database, collection, config) {
    const tenant = this.tenants.get(database);
    if (!tenant) throw new Error('Database not found');

    const subTenant = {
      id: collection,
      parentTenant: database,
      permissions: config.permissions || {},
      dataSources: config.dataSources || []
    };

    tenant.subTenants.set(collection, subTenant);
    return subTenant;
  }

  async searchWithTenantContext(query, database, callerAcl, userId, collection = null) {
    const tenant = this.tenants.get(database);
    if (!tenant) throw new Error('Database not found');

    // Check user permissions
    if (!userId || !(await this.hasPermission(tenant, collection, userId))) {
      throw new Error('Access denied');
    }

    // search() from 2.1; the collection scopes retrieval, so no metadata filter is needed
    return await search(query, database, callerAcl, { collection });
  }
}

class DataPrivacyManager {
  constructor() {
    this.retentionPolicies = new Map();
  }

  async applyRetentionPolicy(database, policy) {
    const { retentionDays, dataTypes, autoDelete } = policy;

    if (autoDelete) {
      const cutoff = new Date();
      cutoff.setDate(cutoff.getDate() - retentionDays);

      // DELETE /context deletes by explicit id (there is no date filter), so
      // list the memories first and delete only those older than the cutoff.
      const expiredIds = await this.findExpiredMemories(database, cutoff);
      if (expiredIds.length > 0) {
        await this.deleteOldData(database, expiredIds);
      }
    }
  }

  async findExpiredMemories(database, cutoff, collection = null) {
    const body = { type: 'memory', database: database };
    if (collection) {
      body.collection = collection;
    }

    const response = await fetch('https://api.hydradb.com/context/list', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${API_KEY}`,
        'API-Version': '2',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify(body)
    });

    const envelope = await response.json();
    // A type: 'memory' listing returns data.user_memories, not data.sources, and
    // each row's identifier is memory_id (knowledge listings use data.sources
    // with id). Reading data.sources here would always yield an empty array.
    const memories = envelope.data?.user_memories ?? [];

    return memories
      .filter((memory) => memory.timestamp && new Date(memory.timestamp) < cutoff)
      .map((memory) => memory.memory_id);
  }

  async deleteOldData(database, ids, collection = null) {
    // Use HydraDB's context deletion endpoint for memory deletion.
    // Raw fetch to the REST API: the JSON body uses snake_case wire keys.
    const body = { type: 'memory', database: database, ids: ids };
    if (collection) {
      body.collection = collection;
    }

    const response = await fetch('https://api.hydradb.com/context', {
      method: 'DELETE',
      headers: {
        'Authorization': `Bearer ${API_KEY}`,
        'API-Version': '2',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify(body)
    });

    return response.json();
  }

  async anonymizeData(data, anonymizationRules) {
    // Implement data anonymization based on rules
    let anonymizedData = { ...data };
    
    anonymizationRules.forEach(rule => {
      if (rule.field && anonymizedData[rule.field]) {
        anonymizedData[rule.field] = this.anonymizeValue(
          anonymizedData[rule.field], 
          rule.method
        );
      }
    });
    
    return anonymizedData;
  }

  anonymizeValue(value, method) {
    switch (method) {
      case 'hash':
        return crypto.createHash('sha256').update(value).digest('hex');
      case 'mask':
        return value.replace(/./g, '*');
      case 'redact':
        return '[REDACTED]';
      default:
        return value;
    }
  }
}

class SearchCache {
  constructor() {
    this.cache = new Map();
    this.ttl = 5 * 60 * 1000; // 5 minutes
  }

  async getCachedResults(query, filters) {
    const key = this.generateCacheKey(query, filters);
    const cached = this.cache.get(key);
    
    if (cached && Date.now() - cached.timestamp < this.ttl) {
      return cached.results;
    }
    
    return null;
  }

  async setCachedResults(query, filters, results) {
    const key = this.generateCacheKey(query, filters);
    this.cache.set(key, {
      results,
      timestamp: Date.now()
    });
  }

  generateCacheKey(query, filters) {
    return `${query}_${JSON.stringify(filters)}`;
  }

  clearExpired() {
    const now = Date.now();
    for (const [key, value] of this.cache.entries()) {
      if (now - value.timestamp > this.ttl) {
        this.cache.delete(key);
      }
    }
  }
}

class RateLimiter {
  constructor(limit, windowMs) {
    this.limit = limit;
    this.windowMs = windowMs;
    this.requests = new Map();
  }

  async checkLimit(userId) {
    const now = Date.now();
    const userRequests = this.requests.get(userId) || [];
    
    // Remove old requests outside the window
    const validRequests = userRequests.filter(
      timestamp => now - timestamp < this.windowMs
    );
    
    if (validRequests.length >= this.limit) {
      return false; // Rate limit exceeded
    }
    
    // Add current request
    validRequests.push(now);
    this.requests.set(userId, validRequests);
    
    return true; // Request allowed
  }
}