vchord_bm25

pig build pkg vchord_bm25         # 构建 RPM / DEB 包

pig repo add pgsql -u          # 添加仓库并更新缓存

shared_preload_libraries = 'vchord_bm25';

CREATE EXTENSION vchord_bm25;

-- 启用所需扩展
CREATE EXTENSION IF NOT EXISTS pg_tokenizer CASCADE;
CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE;

-- 创建分词器（如用于英文的 LLMLingua2）
SELECT create_tokenizer('tokenizer1', $$
model = "llmlingua2"
$$);

-- 创建包含文本内容的表
CREATE TABLE documents (
  id SERIAL PRIMARY KEY,
  passage TEXT,
  embedding bm25vector
);

-- 将文本段落分词为 bm25vector
UPDATE documents SET embedding = tokenize(passage, 'tokenizer1');

-- 创建 BM25 索引
CREATE INDEX documents_embedding_bm25 ON documents USING bm25 (embedding bm25_ops);

-- 使用 BM25 排序查询
SELECT id, passage, embedding <&> to_bm25query('documents_embedding_bm25', tokenize('search query', 'tokenizer1')) AS score
FROM documents
ORDER BY score
LIMIT 10;

-- 基本搜索查询
-- to_bm25query(索引名称, 分词后的查询)
SELECT id, passage, embedding <&> to_bm25query('documents_embedding_bm25', tokenize('database system', 'tokenizer1')) AS score
FROM documents
ORDER BY score
LIMIT 10;

-- 创建带结巴预分词器的文本分析器
SELECT create_text_analyzer('zh_text_analyzer', $$
[pre_tokenizer.jieba]
$$);

-- 创建在语料上训练的自定义模型分词器
SELECT create_custom_model_tokenizer_and_trigger(
    tokenizer_name => 'zh_tokenizer',
    model_name => 'zh_model',
    text_analyzer_name => 'zh_text_analyzer',
    table_name => 'documents',
    source_column => 'passage',
    target_column => 'embedding'
);