centos coreseek安装手记

2013-09-28 04:06 系统架构狂奔的蜗牛

一、准备工作
安装依赖库
1、autoconf

wget http://mirror.anl.gov/pub/gnu/autoconf/autoconf-2.69.tar.gz
    tar zxvf autoconf-2.69.tar.gz
    cd autoconf-2.69/
    ./configure --prefix=/usr/
    make
    make install
    autoconf --version
2、automake

wget http://mirrors.usc.edu/pub/gnu/automake/automake-1.11.tar.gz
    tar zxf automake-1.11.tar.gz 
    cd automake-1.11
    ./configure --prefix=/usr/
    make
    make install
    automake --version
3、libtool

wget http://mirrors.usc.edu/pub/gnu/libtool/libtool-2.4.tar.gz
    tar zxf libtool-2.4.tar.gz
    cd libtool-2.4
    ./configure --prefix=/usr/
    make
    make install
    libtool --version

剩下的看
http://www.coreseek.cn/product_install/install_on_bsd_linux/#deps

如果你的gcc>=4.7安装的过程中：会make编译不通过，提示： error: ‘ExprEval’ was not declared in this scope

解决办法：

搜索之后，仅找到一个有用的链接：http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=667378，是关于coreseek上游的sphinx的同样问题的，看了一下了解似乎是gcc 4.7、gcc 4.8的C++作用域的问题（？），上面的debian bugs里面提供了一个patch，看了下patch的内容，也对自己本地的coreseek的制作了同样的补丁，压缩包发布如下，使用时注意版本：

解压后，用sphinxexpr.cpp覆盖coreseek-4.1-beta/csft-4.1/src下的同名文件重新编译即可

由于在configure csft的时候要mysql库文件支持，但是服务器没有安装mysql，于是官方建议yum安装myql库文件，
yum install mysql-devel libxml2-devel expat-devel
中途遇到yum安装源问题，服务器当前的安装源是iso，以前运维搞的一个坑。无语。。
看了一下系统是Red Hat Enterprise Linux Server release 6.1 (Santiago)
于是搜索redhat 6 yum source 
找到了安装源

[base]
	name=CentOS-$releasever - Base
	baseurl=http://mirrors.163.com/centos/6.0/os/$basearch/
	gpgcheck=1
	gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-6

但是安装的时候提示
找不到：http://mirrors.163.com/centos/6.0/os/x86_64/repodata/repomd.xml
于是打开网址http://mirrors.163.com/centos/
打开6.0文件夹
里面有个readme 有一句说：you should use /6/ and not /6.0/ in your path. Please see this FAQ
那我就用6代替6.0

于是安装源修改为

[base]
	name=CentOS-$releasever - Base
	baseurl=http://mirrors.163.com/centos/6/os/$basearch/
	gpgcheck=1
	gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-6

然后yum install 正常了

sql_query_post_index
	insert into user (User,Password,Host) values ('coreseek',password('coreseek'),'10.0.1.12')

索引合并 把delta合并到main中

indexer --merge main delta --merge-dst-range deleted 0 0 --rotate

配置文件内容：
用到了增量索引
首先要在数据库里面新建一个counter表，记录最大id

CREATE TABLE `coreseek_counter` (
	  `id` int(11) NOT NULL AUTO_INCREMENT,
	  `maxid` int(11) DEFAULT NULL,
	  PRIMARY KEY (`id`)
	) ENGINE=InnoDB DEFAULT CHARSET=utf8

source article_src
	{
			type                    = mysql
			sql_host                = 10.0.1.11
			sql_user                = coreseek
			sql_pass                = coreseek
			sql_db                  = wwwdb
			sql_port                = 3306  # optional, default is 3306
		sql_query_pre = SET NAMES utf8
		sql_query_pre = REPLACE INTO coreseek_counter SELECT 1, MAX(id) FROM cms_bringin
	#    sql_query_range = SELECT MIN(id),MAX(id) FROM cms_bringin
	#    sql_range_step = 1000
	#    sql_query = SELECT id,title,content,UNIX_TIMESTAMP(updatedtime) as updatedtime FROM cms_bringin WHERE id>=$start AND id<=$end and id<=( SELECT maxid FROM coreseek_counter WHERE id=1 )
		sql_query = SELECT id,title,content,UNIX_TIMESTAMP(updatedtime) as updatedtime FROM cms_bringin WHERE id<=( SELECT maxid FROM coreseek_counter WHERE id=1 )
	#注意MySQL中的DATE和DATETIME列类型不能直接作为时间戳使用，必须使用UNIX_TIMESTAMP函数将这些列做显式转换
			sql_attr_timestamp      = updatedtime
		sql_query_info_pre = SET NAMES utf8
			sql_query_info          = SELECT * FROM cms_bringin WHERE id=$id
	}
	source added_src : article_src
	{
		sql_query_pre = SET NAMES utf8
		sql_query = SELECT id, title,content,updatedtime FROM cms_bringin \
			WHERE id>( SELECT maxid FROM coreseek_counter WHERE id=1 )
	}

index article
	{
			source                  = article_src
			path                    = /usr/local/coreseek/var/data/article
			docinfo                 = extern
		ngram_len=0
		charset_dictpath        = /usr/local/mmseg3/etc/
		charset_type            = zh_cn.utf-8
		stopwords           = /usr/local/coreseek/var/stopwords.txt
	}
	index added : article
	{
		source          = added_src
		path            = /usr/local/coreseek/var/data/article_added
	}

#index articlert
	#{
	#       type                    = rt
	#       rt_mem_limit            = 32M
	#
	#       path                    = /usr/local/coreseek/var/data/articlert
	#       charset_type            = utf-8
	#
	#       rt_field                = title
	#       rt_field                = content
	#    rt_attr_timestamp  = updatedtime
	#}

indexer
	{
			mem_limit               = 32M
	}
	searchd
	{
			listen                  = 9312
	#       listen                  = 9306:mysql41
			log                     = /usr/local/coreseek/var/log/searchd.log
			query_log               = /usr/local/coreseek/var/log/query.log
			read_timeout            = 5
			max_children            = 30
			pid_file                = /usr/local/coreseek/var/log/searchd.pid
			max_matches             = 1000
			seamless_rotate         = 1
			preopen_indexes         = 1
			unlink_old              = 1
			workers                 = threads # for RT to work
	}

下面是更新脚本：

coreseek_added.sh（增量索引，crontab每分钟执行一次）
	#!/bin/bash
	/usr/bin/printf "\n----- ----- %s ----- -----\n\n" $(date "+%Y-%m-%d_%H:%M:%S")
	/usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/mysql.conf added --rotate

#每天的01-23点每分钟执行一下增量索引，中间空出来两个小时是用来合并增量索引到主索引的这个比较耗时间，防止冲突就留出来2个小时候
	4 */1 01-23 * * * /Server/scripts/coreseek_added.sh >> /Server/scripts/logs/coreseek_added.log 2>&1

coreseek_merge.sh （合并索引， 每天的00点执行）

#!/bin/bash

/usr/bin/printf "\n----- ----- %s ----- -----\n\n" $(date "+%Y-%m-%d_%H:%M:%S")

/usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/mysql.conf --merge article added --merge-dst-range deleted 0 0 --rotate
	#更改maxid
	/usr/bin/mysql -h10.0.1.11 -ucoreseek -pcoreseek -Dwwwdb -e 'REPLACE INTO coreseek_counter SELECT 1, MAX(id) FROM cms_bringin' 
	#重建增量索引
	/usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/mysql.conf added --rotate

#crontab配置
	5 00 00 * * * /Server/scripts/coreseek_merge.sh >> /Server/scripts/logs/coreseek_merge.log 2>&1

开机启动coreseek脚本
autostart.sh 
#!/bin/bash
/usr/bin/printf "\n----- ----- %s ----- -----\n\n" $(date "+%Y-%m-%d_%H:%M:%S")

/usr/local/coreseek/bin/searchd --config /usr/local/coreseek/etc/mysql.conf >> /Server/scripts/logs/searchd.log 2>&1

在/etc/rc.local
里面加一句:
/Server/scripts/autostart.sh

对于是否设置sql_attr_*的理解，官方文档里面说：属性是可以在搜索的时候用于过滤和排序，
所以凡是以后要排序或者过滤用到的字段都应该设置一个sql_attr

作者：狂奔的蜗牛 出处： http://www.host900.com/

本文版权归作者所有，欢迎转载。未经作者同意下，必须在文章页面明显标出原文链接及作者，否则保留追究法律责任的权利。

如果您认为这篇文章还不错或者有所收获，可以分享给您的小伙伴，因为你的支持是我继续写作，分享的最大动力！

上一篇使ssh-copy-id支持指定端口

下一篇Ubuntu 防火墙命令ufw学习