mysql 从库配置优化

pull/144/head
midoks 3 years ago
parent 25875add11
commit 13386926cd
  1. 12
      plugins/mysql/conf/mysql.sql
  2. 2
      plugins/mysql/index.html
  3. 242
      plugins/mysql/index.py
  4. 163
      plugins/mysql/js/mysql.js

@ -25,4 +25,16 @@ CREATE TABLE IF NOT EXISTS `master_replication_user` (
`addtime` TEXT
);
-- 从库配置主库的[ssh private key]
-- drop table `slave_id_rsa`;
CREATE TABLE IF NOT EXISTS `slave_id_rsa` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`ip` TEXT,
`port` TEXT,
`user` TEXT,
`id_rsa` TEXT,
`ps` TEXT,
`addtime` TEXT
);

@ -16,7 +16,7 @@
<p onclick="masterOrSlaveConf($('.plugin_version').attr('version'))">主从配置</p>
</div>
<div class="bt-w-con pd15">
<div class="soft-man-con"></div>
<div class="soft-man-con" style="height: 530px; overflow: auto;"></div>
</div>
</div>

@ -1574,8 +1574,7 @@ def getMasterRepSlaveList(version=''):
def addMasterRepSlaveUser(version=''):
args = getArgs()
data = checkArgs(args,
['username', 'password'])
data = checkArgs(args, ['username', 'password'])
if not data[0]:
return data[1]
@ -1696,11 +1695,110 @@ def updateMasterRepSlaveUser(version=''):
return mw.returnJson(True, '更新成功!')
def getSlaveSSHList(version=''):
args = getArgs()
data = checkArgs(args, ['page', 'page_size'])
if not data[0]:
return data[1]
page = int(args['page'])
page_size = int(args['page_size'])
conn = pSqliteDb('slave_id_rsa')
limit = str((page - 1) * page_size) + ',' + str(page_size)
field = 'id,ip,port,id_rsa,ps,addtime'
clist = conn.field(field).limit(limit).order('id desc').select()
count = conn.count()
data = {}
_page = {}
_page['count'] = count
_page['p'] = page
_page['row'] = page_size
_page['tojs'] = args['tojs']
data['page'] = mw.getPage(_page)
data['data'] = clist
return mw.getJson(data)
def getSlaveSSHByIp(version=''):
args = getArgs()
data = checkArgs(args, ['ip'])
if not data[0]:
return data[1]
ip = args['ip']
conn = pSqliteDb('slave_id_rsa')
data = conn.field('ip,port,id_rsa').where("ip=?", (ip,)).select()
return mw.returnJson(True, 'ok', data)
def addSlaveSSH(version=''):
import base64
args = getArgs()
data = checkArgs(args, ['ip'])
if not data[0]:
return data[1]
ip = args['ip']
if ip == "":
return mw.returnJson(True, 'ok')
data = checkArgs(args, ['port', 'id_rsa'])
if not data[0]:
return data[1]
id_rsa = args['id_rsa']
port = args['port']
user = 'root'
addTime = time.strftime('%Y-%m-%d %X', time.localtime())
conn = pSqliteDb('slave_id_rsa')
data = conn.field('ip,id_rsa').where("ip=?", (ip,)).select()
if len(data) > 0:
res = conn.where("ip=?", (ip,)).save('port,id_rsa', (port, id_rsa,))
else:
conn.add('ip,port,user,id_rsa,ps,addtime',
(ip, port, user, id_rsa, '', addTime))
return mw.returnJson(True, '设置成功!')
def delSlaveSSH(version=''):
args = getArgs()
data = checkArgs(args, ['ip'])
if not data[0]:
return data[1]
ip = args['ip']
conn = pSqliteDb('slave_id_rsa')
conn.where("ip=?", (ip,)).delete()
return mw.returnJson(True, 'ok')
def updateSlaveSSH(version=''):
args = getArgs()
data = checkArgs(args, ['ip', 'id_rsa'])
if not data[0]:
return data[1]
ip = args['ip']
id_rsa = args['id_rsa']
conn = pSqliteDb('slave_id_rsa')
conn.where("ip=?", (ip,)).save('id_rsa', (id_rsa,))
return mw.returnJson(True, 'ok')
def getSlaveList(version=''):
db = pMysqlDb()
dlist = db.query('show slave status')
dlist = list(dlist)
# print(dlist)
ret = []
for x in range(0, len(dlist)):
@ -1718,6 +1816,14 @@ def getSlaveList(version=''):
return mw.getJson(data)
def getSlaveSyncCmd(version=''):
root = mw.getRunDir()
cmd = 'cd ' + root + ' && python ' + root + \
'/plugins/mysql/index.py do_full_sync {"db":"all"}'
return mw.returnJson(True, 'ok', cmd)
def setSlaveStatus(version=''):
db = pMysqlDb()
dlist = db.query('show slave status')
@ -1774,6 +1880,16 @@ def mw_async(f):
return wrapper
############### --- 重要 同步---- ###########
def writeDbSyncStatus(data):
path = '/tmp/db_async_status.txt'
# status_data['code'] = 1
# status_data['msg'] = '主服务器备份完成...'
# status_data['progress'] = 30
mw.writeFile(path, json.dumps(data))
def doFullSync():
args = getArgs()
@ -1781,53 +1897,54 @@ def doFullSync():
if not data[0]:
return data[1]
arg_db_select = args['db']
status_data = {}
status_data['progress'] = 5
db = pMysqlDb()
dlist = db.query('show slave status')
dlist = list(dlist)
if len(dlist) == 0:
status_data['code'] = -1
status_data['msg'] = '没有启动...'
ip = dlist[0][1]
print(ip)
print("master ip:", ip)
status_file = '/tmp/db_async_status.txt'
id_rsa_conn = pSqliteDb('slave_id_rsa')
data = id_rsa_conn.field('ip,port,id_rsa').where("ip=?", (ip,)).select()
SSH_PRIVATE_KEY = "/tmp/mysql_sync_id_rsa.txt"
id_rsa_key = data[0]['id_rsa']
id_rsa_key = id_rsa_key.replace('\\n', '\n')
master_port = int(data[0]['port'])
mw.writeFile(SSH_PRIVATE_KEY, id_rsa_key)
status_data['code'] = 0
status_data['msg'] = '运行中...'
mw.writeFile(status_file, json.dumps(status_data))
writeDbSyncStatus({'code': 0, 'msg': '开始同步...', 'progress': 0})
import paramiko
paramiko.util.log_to_file('paramiko.log')
ssh = paramiko.SSHClient()
SSH_PRIVATE_KEY = '/root/.ssh/id_rsa'
if mw.getOs() == 'darwin':
user = mw.execShell(
"who | sed -n '2, 1p' |awk '{print $1}'")[0].strip()
SSH_PRIVATE_KEY = '/Users/' + user + '/.ssh/id_rsa'
print(SSH_PRIVATE_KEY)
if not os.path.exists(SSH_PRIVATE_KEY):
status_data['code'] = 0
status_data['msg'] = '需要配置免登录...'
mw.writeFile(status_file, json.dumps(status_data))
return
writeDbSyncStatus({'code': 0, 'msg': '需要配置SSH......', 'progress': 0})
return 'fail'
try:
key = paramiko.RSAKey.from_private_key_file(SSH_PRIVATE_KEY)
# ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=ip, port=22, username='root', pkey=key)
ssh.connect(hostname=ip, port=master_port, username='root', pkey=key)
except Exception as e:
status_data['code'] = 0
status_data['msg'] = '需要配置免登录....'
mw.writeFile(status_file, json.dumps(status_data))
return
writeDbSyncStatus(
{'code': 0, 'msg': 'SSH配置错误:' + str(e), 'progress': 0})
return 'fail'
writeDbSyncStatus({'code': 0, 'msg': '登录Master成功...', 'progress': 5})
cmd = "cd /www/server/mdserver-web && python /www/server/mdserver-web/plugins/mysql/index.py dump_mysql_data {\"db\":'" + args[
'db'] + "'}"
@ -1835,51 +1952,49 @@ def doFullSync():
result = stdout.read()
result_err = stderr.read()
if result == 'ok':
status_data['code'] = 1
status_data['msg'] = '主服务器备份完成...'
status_data['progress'] = 30
mw.writeFile(status_file, json.dumps(status_data))
result = result.decode('utf-8')
# print(result)
if result.strip() == 'ok':
writeDbSyncStatus({'code': 1, 'msg': '主服务器备份完成...', 'progress': 30})
else:
writeDbSyncStatus({'code': 1, 'msg': '主服务器备份失败...', 'progress': 30})
return 'fail'
r = mw.execShell('scp root@' + ip + ':/tmp/dump.sql /tmp')
if r[0] == '':
status_data['code'] = 2
status_data['msg'] = '数据同步本地完成...'
status_data['progress'] = 40
mw.writeFile(status_file, json.dumps(status_data))
writeDbSyncStatus({'code': 2, 'msg': '数据同步本地完成...', 'progress': 40})
cmd = 'cd /www/server/mdserver-web && python /www/server/mdserver-web/plugins/mysql/index.py get_master_rep_slave_user_cmd {"username":"","db":""}'
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stdout.read()
result_err = stderr.read()
cmd_data = json.loads(result)
# print(cmd_data)
db.query('stop slave')
status_data['code'] = 3
status_data['msg'] = '停止从库完成...'
status_data['progress'] = 45
mw.writeFile(status_file, json.dumps(status_data))
writeDbSyncStatus({'code': 3, 'msg': '停止从库完成...', 'progress': 45})
dlist = db.query(cmd_data['data'])
status_data['code'] = 4
status_data['msg'] = '刷新库信息完成...'
status_data['progress'] = 50
mw.writeFile(status_file, json.dumps(status_data))
writeDbSyncStatus({'code': 4, 'msg': '刷新从库同步信息完成...', 'progress': 50})
pwd = pSqliteDb('config').where('id=?', (1,)).getField('mysql_root')
cmd = getServerDir() + "/bin/mysql -uroot -p" + pwd + " < /tmp/dump.sql"
print(mw.execShell(cmd))
status_data['code'] = 5
status_data['msg'] = '同步数据完成...'
status_data['progress'] = 90
mw.writeFile(status_file, json.dumps(status_data))
root_dir = getServerDir()
msock = root_dir + "/mysql.sock"
cmd = root_dir + "/bin/mysql -S " + msock + \
" -uroot -p" + pwd + " < /tmp/dump.sql"
import_data = mw.execShell(cmd)
print(import_data[0])
print(import_data[1])
if import_data[0] == '':
writeDbSyncStatus({'code': 5, 'msg': '导入数据完成...', 'progress': 90})
else:
writeDbSyncStatus({'code': 5, 'msg': '导入数据失败...', 'progress': 90})
return 'fail'
db.query('start slave')
status_data['code'] = 6
status_data['msg'] = '从库重启完成...'
status_data['progress'] = 100
mw.writeFile(status_file, json.dumps(status_data))
writeDbSyncStatus({'code': 6, 'msg': '从库重启完成...', 'progress': 100})
os.system("rm -rf " + SSH_PRIVATE_KEY)
return True
@ -1894,15 +2009,20 @@ def fullSync(version=''):
cmd = 'cd ' + mw.getRunDir() + ' && python ' + \
getPluginDir() + \
'/index.py do_full_sync {"db":"' + args['db'] + '"} &'
# print(cmd)
mw.execShell(cmd)
return json.dumps({'code': 0, 'msg': '同步数据中!', 'progress': 0})
if os.path.exists(status_file):
c = mw.readFile(status_file)
d = json.loads(c)
if d['code'] == 6:
os.remove(status_file)
tmp = json.loads(c)
if tmp['code'] == 1:
dump_size = os.path.getsize("/tmp/dump.sql")
tmp['msg'] = tmp['msg'] + ":" + "同步文件:" + mw.toSize(dump_size)
c = json.dumps(tmp)
# if tmp['code'] == 6:
# os.remove(status_file)
return c
return json.dumps({'code': 0, 'msg': '点击开始,开始同步!', 'progress': 0})
@ -2021,6 +2141,18 @@ if __name__ == "__main__":
print(getMasterRepSlaveUserCmd(version))
elif func == 'get_slave_list':
print(getSlaveList(version))
elif func == 'get_slave_sync_cmd':
print(getSlaveSyncCmd(version))
elif func == 'get_slave_ssh_list':
print(getSlaveSSHList(version))
elif func == 'get_slave_ssh_by_ip':
print(getSlaveSSHByIp(version))
elif func == 'add_slave_ssh':
print(addSlaveSSH(version))
elif func == 'del_slave_ssh':
print(delSlaveSSH(version))
elif func == 'update_slave_ssh':
print(updateSlaveSSH(version))
elif func == 'set_slave_status':
print(setSlaveStatus(version))
elif func == 'delete_slave':

@ -1002,7 +1002,7 @@ function dbList(page, search){
</tbody></table>\
</div>\
<div id="databasePage" class="dataTables_paginate paging_bootstrap page"></div>\
<div class="table_toolbar">\
<div class="table_toolbar" style="left:0px;">\
<span class="sync btn btn-default btn-sm" style="margin-right:5px" onclick="syncToDatabase(1)" title="将选中数据库信息同步到服务器">同步选中</span>\
<span class="sync btn btn-default btn-sm" style="margin-right:5px" onclick="syncToDatabase(0)" title="将所有数据库信息同步到服务器">同步所有</span>\
<span class="sync btn btn-default btn-sm" onclick="syncGetDatabase()" title="从服务器获取数据库列表">从服务器获取</span>\
@ -1321,10 +1321,8 @@ function addMasterRepSlaveUser(){
function updateMasterRepSlaveUser(username){
var index = layer.open({
type: 1,
skin: 'demo-class',
area: '500px',
title: '更新账户',
closeBtn: 1,
@ -1347,7 +1345,6 @@ function updateMasterRepSlaveUser(username){
var data = $("#update_master").serialize();
data = decodeURIComponent(data);
var dataObj = str2Obj(data);
// console.log(dataObj);
myPost('update_master_rep_slave_user', data, function(data){
var rdata = $.parseJSON(data.data);
showMsg(rdata.msg,function(){
@ -1435,7 +1432,7 @@ function getMasterRepSlaveList(){
}
var page = '<div class="dataTables_paginate_4 dataTables_paginate paging_bootstrap page" style="margin-top:0px;"></div>';
page += '<div class="table_toolbar"><span class="sync btn btn-default btn-sm" onclick="addMasterRepSlaveUser()" title="">添加同步账户</span></div>';
page += '<div class="table_toolbar" style="left:0px;"><span class="sync btn btn-default btn-sm" onclick="addMasterRepSlaveUser()" title="">添加同步账户</span></div>';
var loadOpen = layer.open({
type: 1,
@ -1444,7 +1441,7 @@ function getMasterRepSlaveList(){
content:"<div class='bt-form pd20 c6'>\
<div class='divtable mtb10'>\
<div><table class='table table-hover'>\
<thead><tr><th>用户</th><th></th><th></th></tr></thead>\
<thead><tr><th>用户</th><th></th><th></th></tr></thead>\
<tbody>" + list + "</tbody>\
</table></div>\
"+page +"\
@ -1470,8 +1467,9 @@ function deleteSlave(){
function getFullSyncStatus(db){
var timeId = null;
var btn = '<div class="table_toolbar"><span class="sync btn btn-default btn-sm" id="begin_full_sync" title="">开始</span></div>';
var btn = '<div class="table_toolbar" style="left:0px;"><span data-status="init" class="sync btn btn-default btn-sm" id="begin_full_sync" title="">开始</span></div>';
var loadOpen = layer.open({
type: 1,
title: '全量同步['+db+']',
@ -1484,13 +1482,12 @@ function getFullSyncStatus(db){
</div>\
</div>\
"+btn+"\
</div>"
</div>",
cancel: function(){
clearInterval(timeId);
}
});
var timeId = setInterval(function(){
fullSync(db,0);
}, 1000);
function fullSync(db,begin){
myPostN('full_sync', {db:db,begin:begin}, function(data){
@ -1502,26 +1499,150 @@ function getFullSyncStatus(db){
if (rdata['code']==6 ||rdata['code']<0){
layer.msg(rdata['msg']);
clearInterval(timeId);
$("#begin_full_sync").attr('data-status','init');
}
});
}
fullSync(db,0);
$('#begin_full_sync').click(function(){
var val = $(this).attr('data-status');
if (val == 'init'){
fullSync(db,1);
timeId= setTimeout(function(){
timeId = setInterval(function(){
fullSync(db,0);
}, 1000);
$(this).attr('data-status','starting');
} else {
layer.msg("正在同步中..");
}
});
}
$('.layui-layer-close1').click(function(){
clearInterval(timeId);
function addSlaveSSH(ip=''){
myPost('get_slave_ssh_by_ip', {ip:ip}, function(rdata){
var rdata = $.parseJSON(rdata.data);
var ip = '127.0.0.1';
var port = "22";
var id_rsa = '';
if (rdata.data.length>0){
ip = rdata.data[0]['ip'];
port = rdata.data[0]['port'];
id_rsa = rdata.data[0]['id_rsa'];
}
var index = layer.open({
type: 1,
area: ['500px','400px'],
title: '添加SSH',
closeBtn: 1,
shift: 5,
shadeClose: true,
btn:["确认","取消"],
content: "<form class='bt-form pd20'>\
<div class='line'><span class='tname'>IP</span><div class='info-r'><input name='ip' class='bt-input-text mr5' type='text' style='width:330px;' value='"+ip+"'></div></div>\
<div class='line'><span class='tname'>端口</span><div class='info-r'><input name='port' class='bt-input-text mr5' type='text' style='width:330px;' value='"+port+"'></div></div>\
<div class='line'>\
<span class='tname'>ID_RSA</span>\
<div class='info-r'><textarea class='bt-input-text mr5' row='20' cols='50' name='id_rsa' style='width:330px;height:200px;'>"+id_rsa+"</textarea></div>\
</div>\
<input type='hidden' name='ps' value='' />\
</form>",
success:function(){},
yes:function(index){
var ip = $('input[name="ip"]').val();
var port = $('input[name="port"]').val();
var id_rsa = $('textarea[name="id_rsa"]').val();
var data = {ip:ip,port:port,id_rsa:id_rsa};
myPost('add_slave_ssh', data, function(data){
layer.close(index);
var rdata = $.parseJSON(data.data);
showMsg(rdata.msg,function(){
if (rdata.status){
getSlaveSSHPage();
}
},{icon: rdata.status ? 1 : 2},600);
});
}
});
});
}
function delSlaveSSH(ip){
myPost('del_slave_ssh', {ip:ip}, function(rdata){
var rdata = $.parseJSON(rdata.data);
layer.msg(rdata.msg, {icon: rdata.status ? 1 : 2});
getSlaveSSHPage();
});
}
function getSlaveSSHPage(page=1){
var _data = {};
_data['page'] = page;
_data['page_size'] = 5;
_data['tojs'] ='getSlaveSSHPage';
myPost('get_slave_ssh_list', _data, function(data){
var layerId = null;
var rdata = [];
try {
rdata = $.parseJSON(data.data);
} catch(e) {
console.log(e);
}
var list = '';
var ssh_list = rdata['data'];
for (i in ssh_list) {
var ip = ssh_list[i]['ip'];
var port = ssh_list[i]['port'];
list += '<tr><td>'+ip+'</td>\
<td>'+port+'</td>\
<td>OK</td>\
<td>\
<a class="btlink" onclick="addSlaveSSH(\''+ip+'\');">修改</a> | \
<a class="btlink" onclick="delSlaveSSH(\''+ip+'\');">删除</a>\
</td>\
</tr>';
}
$('.get-slave-ssh-list tbody').html(list);
$('.dataTables_paginate_4').html(rdata['page']);
});
}
function getSlaveSSHList(page=1){
var page = '<div class="dataTables_paginate_4 dataTables_paginate paging_bootstrap page" style="margin-top:0px;"></div>';
page += '<div class="table_toolbar" style="left:0px;"><span class="sync btn btn-default btn-sm" onclick="addSlaveSSH()" title="">添加SSH</span></div>';
layerId = layer.open({
type: 1,
title: 'SSH列表',
area: '500px',
content:"<div class='bt-form pd20 c6'>\
<div class='divtable mtb10'>\
<div><table class='table table-hover get-slave-ssh-list'>\
<thead><tr><th>IP</th><th>PORT</th><th>SSH</th><th></th></tr></thead>\
<tbody></tbody>\
</table></div>\
"+page +"\
</div>\
</div>",
success:function(){
}
});
getSlaveSSHPage(1);
}
function handlerRun(){
cmd = 'cd /www/server/mdserver-web && python /www/server/mdserver-web/plugins/mysql/index.py do_full_sync {"db":"all"}';
myPostN('get_slave_sync_cmd', {}, function(data){
var rdata = $.parseJSON(data.data);
var cmd = rdata.data;
var loadOpen = layer.open({
type: 1,
title: '手动执行',
@ -1537,6 +1658,7 @@ function handlerRun(){
$('.class-copy-cmd').click(function(){
copyPass(cmd);
});
});
}
function masterOrSlaveConf(version=''){
@ -1576,7 +1698,7 @@ function masterOrSlaveConf(version=''){
</tbody></table>\
</div>\
<div id="databasePage" class="dataTables_paginate paging_bootstrap page"></div>\
<div class="table_toolbar">\
<div class="table_toolbar" style="left:0px;">\
<span class="sync btn btn-default btn-sm" onclick="getMasterRepSlaveList()" title="">同步账户列表</span>\
</div>\
</div>';
@ -1669,9 +1791,10 @@ function masterOrSlaveConf(version=''){
</tbody></table>\
</div>\
<div id="databasePage" class="dataTables_paginate paging_bootstrap page"></div>\
<div class="table_toolbar">\
<div class="table_toolbar" style="left:0px;">\
<span class="sync btn btn-default btn-sm" onclick="handlerRun()" title="免登录设置后,需要手动执行一下!">手动命令</span>\
<span class="sync btn btn-default btn-sm" onclick="getFullSyncStatus(\'ALL\')" title="全量同步">全量同步</span>\
<span class="sync btn btn-default btn-sm" onclick="getSlaveSSHList()" title="[主]SSH配置">[]SSH配置</span>\
</div>\
</div>';

Loading…
Cancel
Save