coreseek服务的控制脚本和配置示例
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
|
一、索引服务的目录：
[root@test101 coreseek]
# pwd
/data/coreseek
[root@test101 coreseek]
# ls
bin conf data log var
[root@test101 coreseek]
# ls bin/
coreseek_ctl.sh indexer_rotate.sh
[root@test101 coreseek]
# ls conf/
bak conf.d main.conf
[root@test101 coreseek]
# ls conf/bak/
searchd.conf
[root@test101 coreseek]
# ls conf/conf.d/
IndexName111.conf IndexName222.conf
注：配置文件通过
"索引名.conf"
来分类管理，后续将通过控制脚本，将conf文件整合到一个
"main.conf"
中供服务来调用。
二、脚本
1）索引服务的控制脚本：
[root@test101 coreseek]
# cat bin/coreseek_ctl.sh
#!/bin/bash
#
# 2015/5/5
d_base=
'/home/coreseek'
d_base_conf=
"${d_base}/conf"
f_main_conf=
"${d_base_conf}/main.conf"
f_tmp_conf=
"${d_base_conf}/bak/tmp.conf"
searchd_bin=
'/usr/local/coreseek/bin/searchd'
#############################init
#
init() {
echo
"[+] 添加到计划任务中/var/spool/cron/$(whoami)"
cat
<<_CORESEEK >>
/var/spool/cron/
$(
whoami
)
# [coreseek]
#
#0 5 * * * ${d_base}/bin/indexer_rotate.sh main >/dev/null 2>&1 &
#0 7 * * * ${d_base}/bin/indexer_rotate.sh day >/dev/null 2>&1 &
#*/2 * * * * ${d_base}/bin/indexer_rotate.sh merge >/dev/null 2>&1 &
_CORESEEK
echo
'[-] 操作完成！'
echo
'[-] 注：启用时，取消被注释的任务项目即可。'
echo
echo
"#################"
echo
'[-] crontab的内容变成：'
crontab
-l
echo
"#################"
echo
"[+] 创建目录：data,log,var"
mkdir
-p ${d_base}/{data,log,var}
mkdir
-p ${d_base}
/data/
{bak,conf.d}
chmod
770 ${d_base}
}
#############################更新配置文件，将各模块的配置文件整合到main.conf中
#
update() {
echo
"[+] 更新配置文件，将各模块的配置文件整合到main.conf中"
echo
"# 配置文件更新时间：[`date`]"
>${f_tmp_conf}
for
f
in
`
ls
${d_base_conf}
/conf
.d/*.conf`;
do
cat
$f >>${f_tmp_conf}
done
cat
${d_base_conf}
/bak/searchd
.conf >>${f_tmp_conf}
mv
${f_main_conf} ${d_base_conf}
/bak/main
.conf.`
date
+%F` >
/dev/null
2>&1
mv
${f_tmp_conf} ${f_main_conf}
echo
"[-] 操作完成！"
}
#############################备份到： var/xxx.tar.gz
#
backup() {
echo
"[+] 备份到： var/xxx.tar.gz"
cd
${d_base}
local
f_backup=
"${d_base}/var/coreseek_ctl_$(date +%F).tar.gz"
tar
zcvf ${f_backup} bin/ conf/
chmod
o-r ${f_backup} &&
ls
-lh ${f_backup}
}
#############################start|stop|restart|status|debug
#
stop() {
echo
"[+] 停止服务："
${searchd_bin} --config ${f_main_conf} --stop
}
start() {
echo
"[+] 启动服务："
${searchd_bin} --config ${f_main_conf}
}
restart() {
echo
"[+] 重启服务："
stop
sleep
2s
start
}
debug() {
echo
"[+] 以debug模式启动服务："
${searchd_bin} --config ${f_main_conf} --logdebug
}
status() {
echo
"[+] 查看进程状态："
ps
-ef |
grep
-
v
grep
|
grep
searchd --color
}
case
$1
in
init|update|backup|start|stop|restart|status|debug)
$1
;;
*)
cat
<<_EOF
Usage: $0 [init|update|backup|start|stop|restart|status|debug]"
init: 初始化
crontab
和文件夹
update: 将conf
/conf
.d/*.conf所有模块的配置文件整合到main.conf
backup: 压缩备份bin和conf目录到var目录下
_EOF
;;
esac
2）索引数据文件的控制脚本：
[root@test101 coreseek]
# cat bin/indexer_rotate.sh
#!/bin/bash
#
# 2015/4/29
d_base=
'/home/coreseek'
d_data=
"${d_base}/data"
d_var=
"${d_base}/var"
d_log=
"${d_base}/log"
f_main_conf=
"${d_base}/conf/main.conf"
indexer_bin=
'/usr/local/coreseek/bin/indexer'
# 此处是重点。延迟执行
function
wait_rotate(){
local
idx=$1
echo
'##############################################################'
echo
'## 等待 索引数据文件 的滚动'
echo
'##############################################################'
while
[ 1 ]
do
sleep
.1
ls
${d_data}/${idx}.new* >
/dev/null
2>&1
if
[ $? -
ne
0 ];
then
break
fi
done
}
function
start_rotate() {
local
idx=$1
echo
'##############################################################'
echo
'## 开始滚动数据'
echo
'##############################################################'
echo
"[-] 开始滚动main索引： ${idx}"
${indexer_bin} --config ${f_main_conf} ${idx} --rotate
}
function
start_merge() {
local
idx_day=$1
local
idx_delta=$2
f_proc=
"${idx_delta}"
f_lock=
"${d_var}/${f_proc}.lock"
echo
'##############################################################'
echo
'## 开始合并迁移数据'
echo
'##############################################################'
# 检查脚本是否已经运行，避免因为searchd服务进程退出而crontab仍然在执行，从而陷入死循环
if
[ -f ${f_lock} ];
then
echo
"[+] 错误，存在锁: ${f_lock} ！可能存在异常。"
exit
1
fi
touch
${f_lock}
echo
"[+] `date` 正在操作中，加锁。"
#重建增量索引
echo
"[+] `date` 开始滚动delta索引 ${idx_delta}"
${indexer_bin} --config ${f_main_conf} ${idx_delta} --rotate
echo
"[-] `date` 等待delta索引 ${idx_delta}"
wait_rotate ${idx_delta}
#合并增量索引到当天索引
echo
"[-] `date` 合并delta到day索引： ${idx_day}"
${indexer_bin} --config ${f_main_conf} --merge ${idx_day} ${idx_delta} --rotate --merge-dst-range deleted 0 0
echo
"[-] `date` 等待day索引 ${idx_day}"
wait_rotate ${idx_day}
rm
${f_lock} &&
echo
"[-] `date` 操作完成，释放锁。"
||
echo
"[-] `date` 锁 ${f_lock} 未释放。"
}
# 控制main, day, merge操作
function
goto_main() {
echo
"`date` 建立数据文件main：$1"
start_rotate idx_main_$1 >${d_log}
/main
.$1.log 2>&1 &
echo
"`date` 完成。"
}
function
goto_day() {
echo
"`date` 建立数据文件day：$1"
start_rotate idx_day_$1 >${d_log}
/day
.$1.log 2>&1 &
echo
"`date` 操作完成。"
}
function
goto_merge() {
echo
"`date` 建立数据文件delta：$1"
start_merge idx_day_$1 idx_delta_$1 >${d_log}
/merge
.$1.log 2>&1 &
echo
"`date` 操作完成"
}
function
do_job() {
echo
"[+] `date` 建立数据文件："
goto_$1 corpdetail_0
goto_$1 corpdetail_1
goto_$1 crmdetail_0
goto_$1 crmdetail_1
goto_$1 feedback
goto_$1 userblog
echo
"[-] `date` 全部操作完成。"
}
function
do_all() {
echo
'##############################################################'
echo
'## 重建全部的索引数据'
echo
'##############################################################'
[ -d ${d_data} ] ||
mkdir
-p ${d_data}
${indexer_bin} --config ${f_main_conf} --all --rotate
}
function
do_idx() {
echo
'##############################################################'
echo
'## 建立指定索引名的数据'
echo
'##############################################################'
echo
"[+] `date` 建立数据文件，根据指定的索引名：$1"
start_rotate idx_main_$1 >${d_log}
/main
.$1.log 2>&1 &
start_rotate idx_day_$1 >${d_log}
/day
.$1.log 2>&1 &
echo
"[-] `date` 操作完成。"
}
function
usage() {
cat
<<_USAGE
usage: $0 [all|idx|main|day|merge]
all： 重建数据
idx： 根据指定的index名来重建数据
main： 主索引
day： 日索引
merge： 增量索引
_USAGE
}
case
$1
in
main|day|merge)
do_job $1
;;
all|idx)
do_$1 $2
;;
*)
usage
;;
esac
三、配置示例
示例索引：IndexName111
1）
/data/coreseek/conf/conf
.d
/IndexName111
.conf
################################################################################################
############################ IndexName111 start
################################################################################################
#
#######################################source IndexName111.conf
#用户日志主索引，每天重建
source
main_IndexName111
{
type
= mysql
sql_host = 192.168.1.123
sql_user = dbuser
sql_pass = dbpass
sql_db = d_IndexName111
sql_port = 3306
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_pre = SQL 语句
sql_query = SQL 语句
sql_attr_uint = 字段1
sql_attr_string = 字段2
sql_attr_timestamp = 字段3
}
#用户日志当天范围内索引，每天重建
source
day_IndexName111
{
type
= mysql
sql_host = 192.168.1.123
sql_user = dbuser
sql_pass = dbpass
sql_db = d_IndexName111
sql_port = 3306
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_pre = SQL 语句
sql_query = SQL 语句
sql_attr_uint = 字段1
sql_attr_string = 字段2
sql_attr_timestamp = 字段3
}
#用户日志当天范围内增量索引
source
delta_IndexName111 : day_IndexName111
{
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query = SQL 语句
sql_query_post_index = SQL 语句
}
#######################################index IndexName111
#用户日志主索引，每天重建
index idx_main_IndexName111
{
source
= main_IndexName111
path =
/data/coreseek/data/idx_main_IndexName111
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
min_infix_len = 1
html_strip = 0
charset_dictpath =
/usr/local/mmseg3/etc/
charset_type = zh_cn.utf-8
ngram_len = 0
}
#用户日志当天范围内索引，每天重建
index idx_day_IndexName111
{
source
= day_IndexName111
path =
/data/coreseek/data/idx_day_IndexName111
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
min_infix_len = 1
html_strip = 0
charset_dictpath =
/usr/local/mmseg3/etc/
charset_type = zh_cn.utf-8
ngram_len = 0
}
#用户日志当天范围内增量索引
index idx_delta_IndexName111 : idx_day_IndexName111
{
source
= delta_IndexName111
path =
/data/coreseek/data/idx_delta_IndexName111
}
################################################################################################
############################ IndexName111 end
################################################################################################
示例索引：IndexName222
2）
/data/coreseek/conf/conf
.d
/IndexName222
.conf
################################################################################################
############################ IndexName222 start
################################################################################################
#
#######################################source IndexName222.conf
#用户日志主索引，每天重建
source
main_IndexName222
{
type
= mysql
sql_host = 192.168.1.123
sql_user = dbuser
sql_pass = dbpass
sql_db = d_IndexName222
sql_port = 3306
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_pre = SQL 语句
sql_query = SQL 语句
sql_attr_uint = 字段1
sql_attr_string = 字段2
sql_attr_timestamp = 字段3
}
#用户日志当天范围内索引，每天重建
source
day_IndexName222
{
type
= mysql
sql_host = 192.168.1.123
sql_user = dbuser
sql_pass = dbpass
sql_db = d_IndexName222
sql_port = 3306
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_info_pre = SET NAMES latin1
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query_pre = SQL 语句
sql_query = SQL 语句
sql_attr_uint = 字段1
sql_attr_string = 字段2
sql_attr_timestamp = 字段3
}
#用户日志当天范围内增量索引
source
delta_IndexName222 : day_IndexName222
{
sql_query_pre = SET NAMES latin1
sql_query_pre = SET SESSION query_cache_type=OFF
sql_query = SQL 语句
sql_query_post_index = SQL 语句
}
#######################################index IndexName222
#用户日志主索引，每天重建
index idx_main_IndexName222
{
source
= main_IndexName222
path =
/data/coreseek/data/idx_main_IndexName222
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
min_infix_len = 1
html_strip = 0
charset_dictpath =
/usr/local/mmseg3/etc/
charset_type = zh_cn.utf-8
ngram_len = 0
}
#用户日志当天范围内索引，每天重建
index idx_day_IndexName222
{
source
= day_IndexName222
path =
/data/coreseek/data/idx_day_IndexName222
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
min_infix_len = 1
html_strip = 0
charset_dictpath =
/usr/local/mmseg3/etc/
charset_type = zh_cn.utf-8
ngram_len = 0
}
#用户日志当天范围内增量索引
index idx_delta_IndexName222 : idx_day_IndexName222
{
source
= delta_IndexName222
path =
/data/coreseek/data/idx_delta_IndexName222
}
################################################################################################
############################ IndexName222 end
################################################################################################
示例服务配置：indexer, searchd
3）
/data/coreseek/conf/bak/searchd
.conf
#############################################################################
## indexer settings
#############################################################################
indexer
{
mem_limit = 1024M
}
#############################################################################
## searchd settings
#############################################################################
searchd
{
listen = 9312
log =
/data/coreseek/log/searchd
.log
query_log =
/data/coreseek/log/query
.log
binlog_path =
# empty to close binlog.
read_timeout = 5
client_timeout = 300
max_children = 30
pid_file =
/data/coreseek/var/searchd
.pid
max_matches = 5000
compat_sphinxql_magics = 0
seamless_rotate = 1
preopen_indexes = 1
unlink_old = 1
mva_updates_pool = 1M
max_packet_size = 8M
max_filters = 256
max_filter_values = 4096
max_batch_queries = 32
workers = threads
# use threads for RT to work
}
# --eof--
|
本文转自 pcnk 51CTO博客，原文链接：http://blog.51cto.com/nosmoking/1640425，如需转载请自行联系原作者
