Linux 应用程序开发入门
版权 © 2011, 2012 http://netkiller.github.com
摘要
我会实现一个守护进程，从这个程序你将了解，Linux 应用程序开发基本流程
我们将实现一个远程shell的功能，可以通过tcp协议，运行远程机器上的命令或shell脚本
通过这个命令可以实现批量操作，管理上千台服务器。需要发挥你的想象力，灵活使用它。
写这个脚本，我是为了替代SSH远程操作，因为SSH不能控制运行命令，操作风险大，也不安全。
程序还不完善，还需要很多后续改进工作，比如通过SSL建立Socket链接，用户认证，ACL访问控制等等.
下面是我多年积累下来的经验总结，整理成文档供大家参考:
目录
1. 环境 2. nodekeeper 主程序 2.1. 帮助信息 2.2. 参数处理 2.3. 后台运行 2.4. 日志记录 2.5. 多线程 3. 配置文件 4. init.d 脚本 4.1. start/stop 4.2. service start/stopOS: Ubuntu 10.10
Python: 3.2.2
程序目录: /srv/nodekeeper
目录与相关文件
$ cd /srv $ find nodekeeper | grep -v .svn nodekeeper nodekeeper/nodekeeper.ubuntu nodekeeper/nodekeeper.cenos nodekeeper/etc nodekeeper/etc/commands.cfg nodekeeper/etc/protocol.cfg nodekeeper/bin nodekeeper/bin/nodekeeper nodekeeper/bin/console
$ cat nodekeeper/bin/nodekeeper
#!/usr/bin/env python3
#/bin/env python3
#-*- coding: utf-8 -*-
##############################################
# Home : http://netkiller.sf.net
# Author: Neo <openunix@163.com>
##############################################
import asyncore, asynchat, socket, threading
import subprocess, os, sys, getopt, configparser, logging
import string, re
from multiprocessing import Process
class Backend(asyncore.dispatcher):
queue = []
def __init__(self, host, port,config):
asyncore.dispatcher.__init__(self)
self.host = host
self.port = port
self.config = config
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind((host,port))
self.listen(10)
try:
cfg = Protocol(config['protocol'], self.host)
#self.protocols = cfg.items(self.host)
self.protocols = cfg.all()
self.sections = cfg.sections()
except configparser.NoSectionError as err:
print("Error: %s %s" %(err, config['protocol']))
sys.exit(2)
try:
logging.basicConfig(level=logging.NOTSET,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
filename=config['logfile'],
filemode='a')
self.logging = logging.getLogger()
#self.logging.debug('Test')
except AttributeError as err:
print("Error: %s %s" %(err, config['logfile']))
sys.exit(2)
def handle_accept (self):
conn, addr = self.accept()
self.queue.append(addr)
request_handler(conn, self)
def handle_connect(self):
pass
def handle_expt(self):
self.close()
def handle_close(self):
self.close()
class request_handler(asynchat.async_chat):
def __init__(self, sock, resource):
asynchat.async_chat.__init__(self, sock=sock)
self.sessions = resource
self.buffer = b''
self.set_terminator(b"\r\n")
self.logging = resource.logging
self.protocols = resource.protocols
self.sections = resource.sections
self.host = self.sessions.host
def handle_connect(self):
# connection succeeded
#self.logging.info('')
pass
def handle_expt(self):
# connection failed
self.close()
def collect_incoming_data(self, data):
"""Buffer the data"""
#self.buffer.append(data)
self.buffer = data
def found_terminator(self):
try:
buffer = bytes.decode(self.buffer)
except UnicodeDecodeError:
print("\r\nError: ",err)
buffer = ''
try:
execute = re.split(' ', buffer)
command = execute[0]
parameter = ' '.join( execute[1:])
response = b''
screen = ''
if self.buffer == b'quit' or self.buffer == b'exit' :
self.push(b'shutdown!!!\r\n')
self.close_when_done()
elif self.buffer == b'help' or self.buffer == b'?':
screen = "Help may be requested at any point in a command by entering a question mark '?' or 'help'. the help list will be showing the available options.\r\n"
for cmd,v in self.protocols :
screen += cmd + "\r\n"
elif self.buffer == b'sections' :
for sect in self.sections :
screen += sect + "\r\n"
elif self.buffer == b'help.html' :
for cmd,v in self.protocols :
screen += '<a href="?host='+self.host+'&cmd='+cmd+'">'+ cmd +'</a><br />' + "\r\n"
elif self.buffer == b'enable':
self.prompt = b'#'
elif self.buffer == b'end' or self.buffer == b'^z':
self.prompt = b'>'
else:
proto = dict(self.protocols)
if command in proto :
run = proto[command] + ' ' + parameter
screen = subprocess.getoutput(run)
if screen :
response = bytes(screen + "\r\n",'utf8')
self.push(response)
self.logging.info(bytes.decode(self.buffer))
self.buffer = b''
self.close_when_done()
except :
self.close_when_done()
sys.exit(2)
class Protocol():
config = None
agreement = None
def __init__(self,cfg = 'protocol.cfg',sections = ''):
self.config = configparser.SafeConfigParser()
self.config.read(cfg)
#self.agreement = self.config.items('common')
def sections(self):
return self.config.sections()
def items(self, sections):
self.agreement = self.config.items(sections)
return self.agreement
def dicts(self):
return dict(self.agreement)
def all(self):
self.agreement = []
for section in self.config.sections():
self.agreement += self.config.items(section)
return self.agreement
def main():
daemon = False
host = 'localhost'
port = 7800
pidfile = ''
logfile = ''
cfgfile = ''
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:d?v", [ "daemon","host=","port=", 'help',"h=","p=", "basedir=", "pidfile=", "config=", "protocol=", "logfile="])
if not opts :
usage()
sys.exit()
for o, a in opts :
if o in ('-?', '--help') :
usage()
sys.exit()
elif o in ("-v", "--verbose"):
usage()
sys.exit()
elif o in ("-d", "--daemon"):
daemon = True
elif o in ("-h", "--host"):
host = a
elif o in ("-p", "--port"):
port = int(a)
elif o in ("--basedir"):
BASEDIR = a
elif o in ("--pidfile"):
pidfile = a
elif o in ("--config"):
cfgfile = a
elif o in ("--protocol"):
protocol = a
elif o in ("--logfile"):
logfile = a
else:
assert False, "unhandled option"
except getopt.GetoptError as err:
# print help information and exit:
usage()
sys.exit(2)
try:
if daemon :
pid = os.fork()
if pid > 0:
#exit first parent
sys.exit(0)
myself = str(sys.argv[0].split('/')[-1:][0])
#pidfile = os.getpid()
if not pidfile :
pidfile = '/var/run/'+myself+'.pid'
file = open(pidfile,'w')
file.write(str(os.getpid()))
file.close()
if not cfgfile :
cfgfile = ''+myself+'.cfg'
if not logfile :
logfile = '/var/log/'+myself+'.log'
config = dict({'cfgfile':cfgfile, 'pidfile':pidfile, 'logfile':logfile, 'protocol':protocol})
Backend(host,port,config)
asyncore.loop(timeout=30, use_poll=True)
except socket.error as err:
print("\r\nError: ",err)
sys.exit(2)
except IOError as err:
print("\r\nError: ",err)
sys.exit(2)
def usage():
myself = str(sys.argv[0].split('/')[-1:][0])
print("Usage: %s -d -h <ip address> -p <7800>" % myself );
print("Development and deployment administration platform")
print("\r\nMandatory arguments to long options are mandatory for short options too.")
print("\t-?, --help")
print("\t-v, --verbose")
print("\t-d, --daemon")
print("\t-h, --host \t\t(default localhost)")
print("\t-p, --port")
print("\t --config \t\t(default %s.cfg)" % myself)
print("\t --protocol \t\t(default %s.cfg)" % "protocol.cfg")
print("\t --pidfile \t\t(default /var/run/%s.pid)" % myself)
print("\t --logfile \t\t(default /var/log/%s.log)" % myself)
print("\r\nExample:")
print("\t%s --daemon --host localhost --port 7800" % myself)
print("\t%s -d -h localhost -p 7800" % myself)
print("\r\nSee http://netkiller.sf.net/ for updates, bug reports, and answers, \r\nif you have no web access, by sending email to Neo Chan<openunix@163.com>. ")
# Exit status is 0 if OK, 1 if minor problems, 2 if serious trouble.
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print ("Crtl+C Pressed. Shutting down.")
usage()
Usage: nodekeeper -d -h <ip address> -p <7800> Development and deployment administration platform Mandatory arguments to long options are mandatory for short options too. -?, --help -v, --verbose -d, --daemon -h, --host (default localhost) -p, --port --config (default nodekeeper.cfg) --protocol (default protocol.cfg.cfg) --pidfile (default /var/run/nodekeeper.pid) --logfile (default /var/log/nodekeeper.log) Example: nodekeeper --daemon --host localhost --port 7800 nodekeeper -d -h localhost -p 7800 See http://netkiller.sf.net/ for updates, bug reports, and answers, if you have no web access, by sending email to Neo Chan<openunix@163.com>.
getopt.getopt 实现Unix风格的命令参数，例如:
nodekeeper --daemon --host localhost --port 7800 --host localhost --port 7800 IP地址与端口参数 --daemon 参数实现后台运行
具体实现代码
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:d?v", [ "daemon","host=","port=", 'help',"h=","p=", "basedir=", "pidfile=", "config=", "protocol=", "logfile="])
if not opts :
usage()
sys.exit()
for o, a in opts :
if o in ('-?', '--help') :
usage()
sys.exit()
elif o in ("-v", "--verbose"):
usage()
sys.exit()
elif o in ("-d", "--daemon"):
daemon = True
elif o in ("-h", "--host"):
host = a
elif o in ("-p", "--port"):
port = int(a)
elif o in ("--basedir"):
BASEDIR = a
elif o in ("--pidfile"):
pidfile = a
elif o in ("--config"):
cfgfile = a
elif o in ("--protocol"):
protocol = a
elif o in ("--logfile"):
logfile = a
else:
assert False, "unhandled option"
except getopt.GetoptError as err:
# print help information and exit:
usage()
sys.exit(2)
--daemon 参数实现后台运行，原理是首先通过os.fork()克隆一个进程，然后退出当前进程，克隆的新进程继续运行
如果是Shell程序，你可使用“&”符号后台运行，但作为一个应用程序，使用“&”显得不专业。
具体实现的代码如下
if daemon :
pid = os.fork()
if pid > 0:
#exit first parent
sys.exit(0)
程序一旦进入后台，当前进程即将关闭，所以你必须保存PID，为后面的推出程序操作使用，这里我们可以通过 --pidfile 指定一个pid文件
程序一旦进入后台，你只能通过ps，pstree, top 等命令查看状态，运行情况必须通过日志的形式，打印出来
具体实现代码如下：
logging.basicConfig(level=logging.NOTSET,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
filename=config['logfile'],
filemode='a')
self.logging = logging.getLogger()
self.logging.debug('Test')
继承 asynchat.async_chat 实现多线程
class request_handler(asynchat.async_chat):
def __init__(self, sock, resource):
asynchat.async_chat.__init__(self, sock=sock)
连接数限制
self.listen(10)
可以将这个参数提出来，然后通过命令行设置。
nodekeeper --daemon --maxconn 100 --host localhost --port 7800 self.max_connect = maxconn self.listen(self.max_connect)
$ cat nodekeeper/etc/protocol.cfg
[system]
ls = ls
os.hosts = cat /etc/hosts
os.issue = cat /etc/issue
os.memory = free
os.who = who
os.harddisk = df -h
os.uptime = uptime
os.cpuinfo = cat /proc/cpuinfo
os.meminfo = cat /proc/meminfo
os.dmesg = dmesg
os.process = ps aux
os.summary = echo
network.status = netstat -n | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}'
network.netstat = netstat -nlp
network.ifconfig = ifconfig
network.route = ip route
[apache]
apache.start = /usr/local/apache/bin/apachectl start
apache.stop = /usr/local/apache/bin/apachectl stop
apache.restart = /usr/local/apache/bin/apachectl restart
apache.status = ps ax |grep httpd
apache.conf = cat /usr/local/apache/conf/httpd.conf
apache.conf.vhost = cat /usr/local/apache/conf/extra/httpd-vhosts.conf
apache.logs.now =
apache.logs.tail =
[resin]
resin.start = /usr/local/resin/bin/httpd.sh start
resin.stop = /usr/local/resin/bin/httpd.sh stop
resin.restart = /usr/local/resin/bin/httpd.sh restart
resin.status = /usr/local/resin/bin/httpd.sh status
resin.conf = cat /usr/local/resin/conf/resin.conf
[www]
www.list = ls -1 /www
www.permission = find /www -type d -exec chmod 755 {} \; find /www -type f -exec chmod 644 {} \;
www.permission.777 = chmod 777 -R /www/*
lamp.status = ps ax |grep -E "mysqld|httpd|resin"
[samba]
samba.start = /etc/init.d/smb start
samba.stop = /etc/init.d/smb stop
samba.restart = /etc/init.d/smb restart
samba.status = /etc/init.d/smb status
[mysql]
mysql.start = /etc/init.d/mysql start
mysql.stop = /etc/init.d/mysql stop
mysql.restart = /etc/init.d/mysql restart
[memcache]
memcache.start = /etc/init.d/memcache start
memcache.stop = /etc/init.d/memcache stop
memcache.restart = /etc/init.d/memcache restart
[vsftpd]
vsftpd.start = /etc/init.d/vsftpd start
vsftpd.stop = /etc/init.d/vsftpd stop
vsftpd.restart = /etc/init.d/vsftpd restart
vsftpd.status = /etc/init.d/vsftpd status
Linux 所有守护进程都是用init.d下面的脚本来管理
当人你也可以直接运行命令：
nodekeeper --daemon --host localhost --port 7800
但这样只能算是一个半成品，也不够专业，我们写的是linux运用程序，必须遵循Linux规范，所有要实现一个init.d脚本
$ cat nodekeeper
#! /bin/sh
### BEGIN INIT INFO
# Provides: nodekeeper
# Required-Start: $local_fs $remote_fs $network $syslog
# Required-Stop: $local_fs $remote_fs $network $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: starts the nodekeeper web server
# Description: starts nodekeeper using start-stop-daemon
### END INIT INFO
PATH=/srv/nodekeeper/bin:$PATH
DAEMON=/srv/nodekeeper/bin/nodekeeper
NAME=nodekeeper
DESC=nodekeeper
BASEDIR="/srv/nodekeeper"
HOST=$(ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}'|head -n 1)
PORT=7800
CONFIG=$BASEDIR/etc/$NAME.cfg
LOGFILE=$BASEDIR/log/$NAME.log
PIDFILE=$BASEDIR/run/$NAME.pid
PIDFILE=/var/run/$NAME.pid
PROTOCOL=$BASEDIR/etc/protocol.cfg
DAEMON_OPTS="--daemon --host $HOST --port $PORT --config=$CONFIG --protocol=$PROTOCOL --pidfile=$PIDFILE --logfile=$LOGFILE"
test -x $DAEMON || exit 0
# Include nodekeeper defaults if available
if [ -f /etc/default/nodekeeper ] ; then
. /etc/default/nodekeeper
fi
set -e
. /lib/lsb/init-functions
#test_nodekeeper_config() {
# if $DAEMON -t $DAEMON_OPTS >/dev/null 2>&1
# then
# return 0
# else
# $DAEMON -t $DAEMON_OPTS
# return $?
# fi
#}
case "$1" in
start)
echo -n "Starting $DESC: "
start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
--exec $DAEMON -- $DAEMON_OPTS || true
echo "$NAME."
;;
stop)
echo -n "Stopping $DESC: "
start-stop-daemon --stop --quiet --pidfile /var/run/$NAME.pid \
--exec $DAEMON || true
echo "$NAME."
;;
restart|force-reload)
echo -n "Restarting $DESC: "
start-stop-daemon --stop --quiet --pidfile \
/var/run/$NAME.pid --exec $DAEMON || true
sleep 1
start-stop-daemon --start --quiet --pidfile \
/var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS || true
echo "$NAME."
;;
reload)
echo -n "Reloading $DESC configuration: "
start-stop-daemon --stop --signal HUP --quiet --pidfile /var/run/$NAME.pid \
--exec $DAEMON || true
echo "$NAME."
;;
configtest)
echo -n "Testing $DESC configuration: "
if test_nodekeeper_config
then
echo "$NAME."
else
exit $?
fi
;;
status)
status_of_proc -p /var/run/$NAME.pid "$DAEMON" nodekeeper && exit 0 || exit $?
;;
*)
echo "Usage: $NAME {start|stop|restart|reload|force-reload|status|configtest}" >&2
exit 1
;;
esac
exit 0
我们将使用DAEMON_OPTS变量，提供所有需要的参数
DAEMON_OPTS="--daemon --host $HOST --port $PORT --config=$CONFIG --protocol=$PROTOCOL --pidfile=$PIDFILE --logfile=$LOGFILE"
