pg_tokenizer

pig build pkg pg_tokenizer         # 构建 RPM / DEB 包

pig repo add pgsql -u          # 添加仓库并更新缓存

shared_preload_libraries = 'pg_tokenizer';

CREATE EXTENSION pg_tokenizer;

CREATE EXTENSION pg_tokenizer;

-- 使用 LLMLingua2 模型创建分词器
SELECT create_tokenizer('tokenizer1', $$
model = "llmlingua2"
$$);

-- 分词文本
SELECT tokenize('PostgreSQL is a powerful, open-source object-relational database system. It has over 15 years of active development.', 'tokenizer1');

-- 英文分词器
SELECT create_tokenizer('en_tokenizer', $$
model = "llmlingua2"
$$);

-- 中文分词器
SELECT create_tokenizer('zh_tokenizer', $$
model = "jieba"
$$);

-- 日文分词器
SELECT create_tokenizer('ja_tokenizer', $$
model = "lindera/ipadic"
$$);

-- 分词英文文本
SELECT tokenize('full text search in PostgreSQL', 'en_tokenizer');

-- 分词中文文本
SELECT tokenize('PostgreSQL是一个强大的数据库系统', 'zh_tokenizer');

CREATE EXTENSION IF NOT EXISTS pg_tokenizer CASCADE;
CREATE EXTENSION IF NOT EXISTS vchord_bm25 CASCADE;

-- 创建分词器
SELECT create_tokenizer('my_tokenizer', $$
model = "llmlingua2"
$$);

-- 将文本分词为 bm25vector 用于索引和搜索
SELECT tokenize('your search query', 'my_tokenizer');