Vector Search

{
  "collection": "dataset_schemas",
  "vectors": {
    "size": 1536,
    "distance": "Cosine"
  },
  "payload": {
    "dataset_id": "uuid",
    "table_name": "string",
    "column_name": "string",
    "data_type": "string",
    "description": "string"
  }
}

table_text = f"""
Table: {table_name}
Description: {table_description}
Columns: {', '.join(column_names)}
Sample queries: {common_queries}
"""

column_text = f"""
Column: {column_name}
Table: {table_name}
Type: {data_type}
Description: {description}
Sample values: {sample_values}
Statistics: {min}, {max}, {avg}, {distinct_count}
"""

relationship_text = f"""
{table1}.{column1} relates to {table2}.{column2}
Type: {foreign_key|one_to_many|many_to_many}
Description: {relationship_description}
"""

Production:
  - OpenAI text-embedding-3-large
  - Cohere embed-english-v3.0
  
Open Source:
  - sentence-transformers/all-MiniLM-L6-v2
  - BAAI/bge-large-en-v1.5
  
Multilingual:
  - OpenAI text-embedding-3-small
  - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2

def expand_query(user_query):
    # Add synonyms
    expanded = add_synonyms(user_query)
    
    # Add related terms
    expanded += get_related_terms(user_query)
    
    # Add common variations
    expanded += generate_variations(user_query)
    
    return expanded

def calculate_score(query_vector, doc_vector, metadata):
    # Cosine similarity
    vector_score = cosine_similarity(query_vector, doc_vector)
    
    # Boost for exact matches
    exact_match_boost = 2.0 if is_exact_match(query, metadata) else 1.0
    
    # Recency boost
    recency_boost = calculate_recency_boost(metadata.last_accessed)
    
    # Usage frequency boost
    usage_boost = log(metadata.query_count + 1)
    
    return vector_score * exact_match_boost * recency_boost * usage_boost

Finance:
  revenue: [income, sales, turnover, receipts]
  customer: [client, account, counterparty]
  transaction: [trade, deal, order, payment]

Healthcare:
  patient: [member, beneficiary, client]
  provider: [doctor, physician, practitioner]
  claim: [bill, invoice, submission]

# Previous queries influence current search
context_embedding = combine_embeddings([
    current_query_embedding,
    weighted_previous_queries,
    active_dataset_context
])

hnsw_config:
  m: 16  # Number of connections
  ef_construct: 100  # Construction accuracy
  ef: 50  # Search accuracy
  max_m: 16
  seed: 42

@lru_cache(maxsize=10000)
def get_cached_embedding(text):
    return embedding_model.encode(text)

# Process in batches for efficiency
def embed_batch(texts, batch_size=100):
    embeddings = []
    for i in range(0, len(texts), batch_size):
        batch = texts[i:i + batch_size]
        batch_embeddings = model.encode(batch)
        embeddings.extend(batch_embeddings)
    return embeddings

# Collect training pairs
training_data = [
    ("show revenue", "sales_transactions.total_amount"),
    ("customer list", "customers.customer_name"),
    ("monthly sales", "orders.order_date, orders.amount")
]

# Fine-tune embeddings
fine_tuned_model = train_model(base_model, training_data)

def hybrid_search(query, alpha=0.7):
    vector_results = vector_search(query)
    keyword_results = keyword_search(query)
    
    # Weighted combination
    combined = alpha * vector_results + (1 - alpha) * keyword_results
    return deduplicate_and_sort(combined)

# Mean Reciprocal Rank
def calculate_mrr(queries, correct_results):
    reciprocal_ranks = []
    for query, correct in zip(queries, correct_results):
        results = search(query)
        rank = get_rank(correct, results)
        reciprocal_ranks.append(1/rank if rank else 0)
    return mean(reciprocal_ranks)

{
  "query": "customer revenue last month",
  "tokens": ["customer", "revenue", "last", "month"],
  "expanded": ["client", "income", "previous", "30 days"],
  "embedding": [0.123, -0.456, ...],
  "similar_queries": ["sales by customer", "monthly income"]
}