SVN与ReviewBoard联动(3)

前面做测试,生成了很多错误的review request。
但没有找到可以批量删除的工具,于是自己写了一个。

deleterequest.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Created on 2016-11-24
@author: Hansen
获取指定svn文件夹下,指定svn版本的注释
'''

import os
import sys

#获取指定svn版本的注释
def del_rb_request(rid1,rid2):
    for rid in range(rid1,rid2):
        cmd='curl -X DELETE --header "Authorization: token API_TOKEN" http://127.0.0.1/api/review-requests/'+str(rid)+'/'
        print(cmd)
        logs = os.popen(cmd.encode()).readlines()
        for log in logs:
            print(log)

#start here
del_rb_request(0,86)

SVN与ReviewBoard联动(2)

接上一篇。

1、脚本写了没多久,但郁闷的是测通上面的脚本我花了很久,后来发现主要是字符集的问题。

我用的是Debian默认字符集为UTF8,python2.7默认字符集为ASCII。
后面,设置了python的字符集默认为UTF8,第一篇的脚本才测通了。
/usr/lib/python2.7/sitecustomize.py

# install the apport exception handler if available
import sys

sys.setdefaultencoding('UTF8')

try:
    import apport_python_hook
except ImportError:
    pass
else:
    apport_python_hook.install()

2、然后发现RBT貌似不支持UTF8的中文参数,反正我没有测通,好吧,自己写了一个脚本将SVN上的中文注释更新到mysql中。
说明一下,RBT的这个问题,我已经向社区提交Bug了,亲测补丁有效,大家等待下一版本就好了。

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Created on 2016-11-24
@author: Hansen
用于更新rb的说明
'''

import sys
import datetime
import MySQLdb
import nsvnutils

#记录日志
def nlog(logpath,msg):
    fo = open(logpath,'a+')
    try:
        fo.write(msg+'\n')
    finally:
        fo.close()

#更新rb的reviewrequest说明
def update_rb_comment():
    #获取连接
    try:
        connection=MySQLdb.connect(user='user',passwd="passwd",host='127.0.0.1',port=3306,db='db',charset='utf8')
        subconnection=MySQLdb.connect(user='user',passwd="passwd",host='127.0.0.1',port=3306,db='db',charset='utf8')
        #print('mysql connection ok')
        nlog('/mnt/rb/sql.log','mysql connection ok')
    except Exception as ex:
        #print(str(ex))
        nlog('/mnt/rb/sql.log',str(ex))
        return

    #获取需要处理的数据
    try:
        cursor=connection.cursor()
	subcursor=subconnection.cursor()
        sql="select id,summary from reviews_reviewrequest r where r.description='' or r.description is null"
        cursor.execute(sql)
        #print('mysql select ok')
        nlog('/mnt/rb/sql.log','mysql select ok')
        for row in cursor.fetchall():
            rid=str(row[0])
            ss=row[1].split('_Rev')
	    if len(ss)==2:
                nsvnutils.get_repo_name(ss[0])
                comment=nsvnutils.get_svn_log(ss[1])
		commiter=nsvnutils.get_svn_commiter(ss[1])
                subsql="update reviews_reviewrequest r set r.description='"+commiter+"_"+comment+"' where r.id="+rid
	        #print(subsql)
                nlog('/mnt/rb/sql.log',subsql)
		subcursor.execute(subsql)
            else:
                #print('skiping id='+rid+' summary='+row[1])
                nlog('/mnt/rb/sql.log','skiping id='+rid+' summary='+row[1])
        subconnection.commit()
    except Exception as ex:
        #print(str(ex))
        nlog('/mnt/rb/sql.log',str(ex))
	return
    finally:
        cursor.close()
	subcursor.close()

    connection.close()
    subconnection.close()

#start here
nlog("/mnt/rb/sql.log", 'Starting job on '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
update_rb_comment()
nlog("/mnt/rb/sql.log", 'Ending job on '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

3、建立定时任务,执行脚本

crontab -l

#10分钟运行一次
*/10 * * * * /home/neohope/scripts/updatedb.py

SVN与ReviewBoard联动(1)

公司的SVN服务器是在Windows上的,ReviewBoard是在Linux虚拟服务器上的。
好吧,主要是历史原因了哦。。。

解决思路是,利用SVN的post-commit钩子,在提交后,将版本号存到一个文件中。
在Linux服务器中,定时去扫描这些文件,有变动的话,则利用RBTools提交请求。

1、Windows中用post-commit钩子获取版本号
post-commit.bat

@echo off
SET REPOS=%1%
SET REV=%2%
SET RPATH=PATH_TO_STORE_FILE

CALL :getfilename %REPOS%

echo %REV% >> %RPATH%\%FNAME%.txt

:success
exit 0

:getfilename
set FNAME=%~nx1

2、Linux中用python脚本扫描文件变动
runme.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Created on 2016-11-24
@author: Hansen
遍历监视文件夹下所有的txt文件
当文件多余两行时,对于每一行,调用rbt生成review请求
最后,只保留最后一行
'''

import os
import sys
import datetime
import nsvnutils

#记录日志
def nlog(logpath,msg):
    fo = open(logpath,'a+')
    try:
        fo.write(msg)
    finally:
        fo.close()

#创建rbt命令,并调用命令
def create_rbt_command(repo, revision1, revision2, logpath):
    title = repo+"_Rev"+revision1
    comment = nsvnutils.get_svn_log(revision1)
    commiter = nsvnutils.get_svn_commiter(revision1)
    #rbt好像不支持utf8参数,因此用更新db的方式添加注释
    cmd = 'rbt post -p --repository ' +repo+" --repository-type svn --server http://127.0.0.1 --api-token API_TOKEN --summary "+title+' '+revision1 
    fo = open(logpath,'a+')
    try:
        fo.write('adding rbt job on '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'\n')
	fo.write('PWD is '+os.getcwd()+'\n')
	fo.write('CMD is '+cmd+'\n')
	logs = os.popen(cmd.encode('UTF8')).readlines()
        for log in logs:
	   fo.write(log+'\n')
	fo.write('\n')
    finally:
        fo.close()

#写回最后一行
def write_last_line(txtpath,line):
    fo = open(txtpath,'w+')
    try:
	fo.write(line)
    finally:
        fo.close()

#遍历行
#多于两行则自动生成post review
def enum_line(txtpath,logpath,repo):
    nlog('/mnt/rb/rb.log','Trying repositor '+repo+'\n')
    fi = open(txtpath)
    try:
	lines=fi.readlines()
	linesize=len(lines)
	if linesize <= 1:
            nlog('/mnt/rb/rb.log','Skiping repositor '+repo+'\n')
            return

        for i in range(0, linesize):
            if i+1<linesize:
		rbRepo = nsvnutils.get_repo_name(repo)
                if len(rbRepo)==0 :
                    nlog('/mnt/rb/rb.log','Skiping repositor '+repo+'\n')
                    return

                nlog('/mnt/rb/rb.log','Parsing repository '+rbRepo+'\n')
		create_rbt_command(rbRepo,lines[i].replace('\n','').replace('\r',''),lines[i+1].replace('\n','').replace('\r',''),logpath)
    except Exception as ex:
        nlog('/mnt/rb/rb.log',str(ex))
    finally:
        fi.close()

    write_last_line(txtpath,lines[linesize-1])

#递归遍历文件
def enum_file(targetdir,sufix,logpath):
    for root, dirs, files in os.walk(targetdir, True):
        for fname in files:
            fsufix = os.path.splitext(fname)[1][1:]
	    repo = os.path.splitext(fname)[0]
            if sufix == fsufix:
                txtpath = root+"/"+fname
                #txtpath = root+"\\"+fname
		enum_line(txtpath,logpath,repo)


#start here
nlog("/mnt/rb/rb.log", 'Starting job on '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'\n')
enum_file("/mnt/rb","txt", "/mnt/rb/rb.log")
#enum_file("D:/MyProducts/Python/ReviewBoard/rb","txt", "D:/MyProducts/Python/ReviewBoard/rb/rb.log")
nlog("/mnt/rb/rb.log", 'Ending job on '+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'\n')

nsvnutils.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Created on 2016-11-24
@author: Hansen
获取指定svn文件夹下,指定svn版本的注释
'''

import os
import sys

#切换路径,并获取repository名称
def get_repo_name(repo):
    mydict = {'REPO1':'REPO1','REPO2':'REPO2'}
    if not mydict.get(repo):
        return '' 
    os.chdir('/home/neohope/repository/'+mydict.get(repo))
    #os.chdir('D:/MyProducts/Python/ReviewBoard/repo/'+mydict(repo))
    return mydict[repo]

#获取指定svn版本的注释
def get_svn_log(revision):
    cmd = 'svn log -r '+revision
    logs = os.popen(cmd).readlines()
    #for log in logs:
    #    print(log)
    if len(logs)>3:
        print(logs[3].replace('\r','').replace('\n',''))
        return logs[3].replace('\r','').replace('\n','')
    else:
        print('no svn comment') 
        return 'no svn comment'

#获取指定svn版本的提交者
def get_svn_commiter(revision):
    cmd = 'svn log -r '+revision
    logs = os.popen(cmd).readlines()
    if len(logs)>2:
        ss=logs[1].split('|')
        if len(ss)>2:
            print(ss[1].strip().replace('\r','').replace('\n',''))
            return ss[1].strip().replace('\r','').replace('\n','')
        else:
            #print('no svn commiter') 
            return 'no svn commiter'
    else:
        #print('no svn commiter') 
        return 'no svn commiter'

#start here
#get_svn_log('193')
#get_svn_commiter('193')

3、建立定时任务,执行脚本

crontab -l

#半小时运行一次
*/30 * * * * /home/neohope/scripts/runme.py

比较麻烦的是,仍需要在Linux下面,先把SVN的repository检出才可以。
同时要注意,要做好SVN及ReviewBoard之间名称的映射。。。

Debain8安装ReviewBoard

1、安装需要的软件及软件包

#更新apt软件列表
apt-get update

#安装mysql
apt-get install mysql-server

#安装apache2
apt-get install apache2

#安装mod_wsgi
apt-get install libapache2-mod-wsgi

#安装memcached
apt-get install memcached

#安装patch
apt-get install patch

#安排subversion
apt-get install subversion

2、配置MySQL

#修改配置文件
vi /etc/mysql/my.cnf
#添加下面内容
[client]
default-character-set=utf8
[mysqld]
character-set-server=utf8


#重启mysql
/etc/init.d/mysql restart


#新建数据库、用户并授权
mysql -u root -p
mysql> CREATE DATABASE reviewboard CHARACTER SET utf8;
mysql> CREATE USER 'reviewboard'@'localhost' IDENTIFIED BY 'reviewboard';
mysql> GRANT ALL PRIVILEGES ON reviewboard.* to 'reviewboard'@'localhost';

3、配置svn(我的svn和reviewboard不是在一台机器上的,不需要重新配置)

4、配置Python2.7环境

#安装python-setuptools
apt-get install python-setuptools

#安装python-dev
apt-get install python-dev

#安装python-mysqldb
apt-get install python-mysqldb
#替代方案
#easy_install mysql-python

#安装python-svn
apt-get install python-svn

#安装 libffi-dev
apt-get install libffi-dev

#安装ReviewBoard
easy_install ReviewBoard

#你可以用pip安装
#eays_install pip
#pip install pipdeptree
#pip install ReviewBoard

5、安装网站

#这句话会报错,因为依赖的Django版本冲突所导致的
rb-site install /var/www/reviewboard

#查看插件依赖
pipdeptree

#可以看到
#ReviewBoard依赖Django [required: <1.7,>=1.6.11, installed: 1.8]
#django-haystack依赖Django [required: >=1.8, installed: 1.8]
argparse==1.2.1
chardet==2.3.0
lxml==3.4.0
MySQL-python==1.2.3
numpy==1.8.2
pycups==1.9.63
pycurl==7.19.5
pygobject==3.14.0
pysmbc==1.0.15.3
python-apt==0.9.3.11
python-debian==0.1.27
  - six [required: None, installed: 1.8.0]
python-debianbts==1.11
pyxdg==0.25
reportbug==6.6.3
ReviewBoard==2.5.1
  - Django [required: <1.7,>=1.6.11, installed: 1.8]
  - django-evolution [required: <=0.7.999,>=0.7.5, installed: 0.7.6]
    - Django [required: <1.7.0,>=1.4.10, installed: 1.8]
  - django-haystack [required: >=2.3.1, installed: 2.5.0]
    - Django [required: >=1.8, installed: 1.8]
    - Django [required: <1.10, installed: 1.8]
  - django-multiselectfield [required: None, installed: 0.1.4]
    - django [required: >=1.4, installed: 1.8]
  - Djblets [required: <=0.9.999,>=0.9, installed: 0.9.3]
    - Django [required: >=1.6.11,<1.8.999, installed: 1.8]
    - django-pipeline [required: <1.3.9999,>=1.3.23, installed: 1.3.27]
      - futures [required: >=2.1.3, installed: 3.0.5]
    - feedparser [required: >=5.1.2, installed: 5.2.1]
    - pillowfight [required: None, installed: 0.2]
      - Pillow [required: None, installed: 2.6.1]
    - pytz [required: None, installed: 2016.7]
  - docutils [required: None, installed: 0.12]
  - markdown [required: <2.4.999,>=2.4.0, installed: 2.4.1]
  - mimeparse [required: >=0.1.3, installed: 0.1.3]
  - paramiko [required: >=1.12, installed: 2.0.2]
    - cryptography [required: >=1.1, installed: 1.5.2]
      - cffi [required: >=1.4.1, installed: 1.8.3]
        - pycparser [required: None, installed: 2.14]
      - enum34 [required: None, installed: 1.1.6]
      - idna [required: >=2.0, installed: 2.1]
      - ipaddress [required: None, installed: 1.0.17]
      - pyasn1 [required: >=0.1.8, installed: 0.1.9]
      - setuptools [required: >=11.3, installed: 28.6.0]
      - six [required: >=1.4.1, installed: 1.8.0]
    - pyasn1 [required: >=0.1.7, installed: 0.1.9]
  - pycrypto [required: >=2.6, installed: 2.6.1]
  - Pygments [required: >=1.6, installed: 2.0.1]
  - python-dateutil [required: ==1.5, installed: 1.5]
  - python-memcached [required: None, installed: 1.58]
    - six [required: >=1.4.0, installed: 1.8.0]
  - pytz [required: None, installed: 2016.7]
  - recaptcha-client [required: None, installed: 1.0.6]
  - Whoosh [required: >=2.6, installed: 2.7.4]
roman==2.0.0
SOAPpy==0.12.22
  - defusedxml [required: None, installed: 0.4.1]
  - wstools [required: None, installed: 0.4.3]
    - docutils [required: None, installed: 0.12]
wsgiref==0.1.2

#所以降级django-haystack就好了
easy_install -m django
easy_install -m django-haystack
easy_install django-haystack==2.3.1
easy_install reviewboard

6、安装网站,并修改网站权限

#安装网站,按提示输入
rb-site install /var/www/reviewboard
chown -R www-data /var/www/reviewboard/htdocs/media/uploaded
chown -R www-data /var/www/reviewboard/data
chown -R www-data /var/www/reviewboard/logs
chown -R www-data /var/www/reviewboard/htdocs/media/ext
chown -R www-data /var/www/reviewboard/htdocs/static/ext

7、配置访问权限

#配置访问权限
vi /var/www/reviewboard/conf/settings_local.py

#修改下面一行,这是不限制任何访问
ALLOW_HOSTS=['*']

8、配置apache2虚拟目录

cp /var/www/reviewboard/conf/apache-wsgi.conf /etc/apache2/sites-available/reviewboard.conf
#按实际需要的部署情况,编辑reviewboard.conf
cd /etc/apache2/sites-enabled
ln -s ../sites-avaiable/reviewboard.conf

9、重启apache2

/etc/init.d/apache2 restart

10、打开浏览器就可以登录啦

11、另外,我是在虚拟机中部署的reviewboard,我尝试了用nginx反向代理reviewboard,但没有成功。
用firebug看到,json中的ip地址没有改写,最后发现是因为django没有正确的处理absolute_path。
好像增加一些django的配置就好了,但实在是没时间处理了。
最后用bridege方式将虚拟机映射出来,完成部署。