Merge pull request #503 from midoks/dev

mysql binlog日志查看功能及同步12306错误，尝试修复功能。

plugins/clean/tool_task.py

@@ -52,8 +52,8 @@ def createBgTask():
     removeBgTask()
     args = {
         "period": "day",
-        "hour": "2",
-        "minute": "30",
+        "hour": "0",
+        "minute": "15",
     }
     createBgTaskByName(getPluginName(), args)
 

plugins/mysql/conf/my5.5.cnf

@@ -53,7 +53,7 @@ long_query_time=10
 #log_queries_not_using_indexes=on
 #log_slow_admin_statements=1
 #log_slow_slave_statements=1
-expire_logs_days=30
+expire_logs_days=7
 
 relay-log=mdserver
 relay-log-index=mdserver

plugins/mysql/conf/my5.6.cnf

@@ -56,7 +56,7 @@ long_query_time=10
 #log_queries_not_using_indexes=on
 #log_slow_admin_statements=1
 #log_slow_slave_statements=1
-expire_logs_days=30
+expire_logs_days=7
 
 relay-log=mdserver
 relay-log-index=mdserver

plugins/mysql/conf/my5.7.cnf

@@ -57,7 +57,7 @@ long_query_time=10
 #log_queries_not_using_indexes=1
 #log_slow_admin_statements=1
 #log_slow_slave_statements=1
-expire_logs_days=30
+expire_logs_days=7
 
 relay-log=mdserver
 relay-log-index=mdserver

plugins/mysql/index.html

@@ -12,6 +12,7 @@
 			<p onclick="myPerfOpt();">性能优化</p>
 			<p onclick="myLogs();">日志</p>
 			<p onclick="pluginLogs('mysql',$('.plugin_version').attr('version'),'show_log');">慢日志</p>
+			<p onclick="myBinLogs();">BINLOG</p>
 			<p onclick="dbList()">管理列表</p>
 			<p onclick="masterOrSlaveConf($('.plugin_version').attr('version'))">主从配置</p>
 		</div>
@@ -57,12 +58,14 @@
     text-align: right;
 }
 </style>
+
+<!-- <script type="text/javascript"  src="/plugins/file?name=mysql&f=js/mysql.js"></script> -->
 <script type="text/javascript">
 resetPluginWinWidth(800);
 
-var loading = layer.msg("资源加载中...", { icon: 16, time: 0, shade: 0.3 });
+// var loading = layer.msg("资源加载中...", { icon: 16, time: 0, shade: 0.3 });
 $.getScript( "/plugins/file?name=mysql&f=js/mysql.js", function(){
-	layer.close(loading);
+	// layer.close(loading);
 	pluginService('mysql',$('.plugin_version').attr('version'));
 });
 </script>

plugins/mysql/index.py

@@ -293,6 +293,14 @@ def getDataDir():
     return tmp.groups()[0].strip()
 
 
+def getLogBinName():
+    file = getConf()
+    content = mw.readFile(file)
+    rep = 'log-bin\s*=\s*(.*)'
+    tmp = re.search(rep, content)
+    return tmp.groups()[0].strip()
+
+
 def getPidFile():
     file = getConf()
     content = mw.readFile(file)
@@ -333,6 +341,54 @@ def binLog():
     return mw.returnJson(True, '设置成功!')
 
 
+def binLogList():
+    args = getArgs()
+    data = checkArgs(args, ['page', 'page_size', 'tojs'])
+    if not data[0]:
+        return data[1]
+
+    page = int(args['page'])
+    page_size = int(args['page_size'])
+
+    data_dir = getDataDir()
+    log_bin_name = getLogBinName()
+
+    alist = os.listdir(data_dir)
+    log_bin_l = []
+    for x in range(len(alist)):
+        f = alist[x]
+        t = {}
+        if f.startswith(log_bin_name) and not f.endswith('.index'):
+            abspath = data_dir + '/' + f
+            t['name'] = f
+            t['size'] = os.path.getsize(abspath)
+            t['time'] = mw.getDataFromInt(os.path.getctime(abspath))
+            log_bin_l.append(t)
+
+    log_bin_l = sorted(log_bin_l, key=lambda x: x['time'], reverse=True)
+
+    # print(log_bin_l)
+    # print(data_dir, log_bin_name)
+
+    count = len(log_bin_l)
+
+    page_start = (page - 1) * page_size
+    page_end = page_start + page_size
+    if page_end > count:
+        page_end = count
+
+    data = {}
+    page_args = {}
+    page_args['count'] = count
+    page_args['p'] = page
+    page_args['row'] = page_size
+    page_args['tojs'] = args['tojs']
+    data['page'] = mw.getPage(page_args)
+    data['data'] = log_bin_l[page_start:page_end]
+
+    return mw.getJson(data)
+
+
 def cleanBinLog():
     db = pMysqlDb()
     cleanTime = time.strftime('%Y-%m-%d %H:%i:%s', time.localtime())
@@ -2431,11 +2487,65 @@ def getSlaveList(version=''):
 
     db = pMysqlDb()
     dlist = db.query('show slave status')
+
+    # print(dlist)
     data = {}
     data['data'] = dlist
     return mw.getJson(data)
 
 
+def trySlaveSyncBugfix(version=''):
+    if status(version) == 'stop':
+        return mw.returnJson(False, 'MySQL未启动', [])
+
+    mode_file = getSyncModeFile()
+    if not os.path.exists(mode_file):
+        return mw.returnJson(False, '需要先设置同步配置')
+
+    mode = mw.readFile(mode_file)
+    if mode != 'sync-user':
+        return mw.returnJson(False, '仅支持【同步账户】模式')
+
+    conn = pSqliteDb('slave_sync_user')
+    slave_sync_data = conn.field('ip,port,user,pass,mode,cmd').select()
+    if len(slave_sync_data) < 1:
+        return mw.returnJson(False, '需要先添加【同步用户】配置!')
+
+    # print(slave_sync_data)
+    # 本地从库
+    sdb = pMysqlDb()
+
+    gtid_purged = ''
+
+    for i in range(len(slave_sync_data)):
+        port = slave_sync_data[i]['port']
+        password = slave_sync_data[i]['pass']
+        host = slave_sync_data[i]['ip']
+        user = slave_sync_data[i]['user']
+
+        # print(port, password, host)
+
+        mdb = mw.getMyORM()
+        mdb.setHost(host)
+        mdb.setPort(port)
+        mdb.setUser(user)
+        mdb.setPwd(password)
+        mdb.setSocket('')
+
+        var_gtid = mdb.query('show VARIABLES like "%gtid_purged%"')
+        if len(var_gtid) > 0:
+            gtid_purged += var_gtid[0]['Value'] + ','
+
+    gtid_purged = gtid_purged.strip(',')
+    sql = "set @@global.gtid_purged='" + gtid_purged + "'"
+
+    sdb.query('stop slave')
+    # print(sql)
+    sdb.query(sql)
+    sdb.query('start slave')
+    return mw.returnJson(True, '修复成功!')
+
+
 def getSlaveSyncCmd(version=''):
     root = mw.getRunDir()
     cmd = 'cd ' + root + ' && python3 ' + root + \
@@ -3021,6 +3131,10 @@ if __name__ == "__main__":
         print(getConf())
     elif func == 'bin_log':
         print(binLog())
+    elif func == 'binlog_list':
+        print(binLogList())
+    elif func == 'binlog_look':
+        print(binLogListLook())
     elif func == 'clean_bin_log':
         print(cleanBinLog())
     elif func == 'error_log':
@@ -3115,6 +3229,8 @@ if __name__ == "__main__":
         print(getMasterRepSlaveUserCmd(version))
     elif func == 'get_slave_list':
         print(getSlaveList(version))
+    elif func == 'try_slave_sync_bugfix':
+        print(trySlaveSyncBugfix(version))
     elif func == 'get_slave_sync_cmd':
         print(getSlaveSyncCmd(version))
     elif func == 'get_slave_ssh_list':

plugins/mysql/index_mysql.py

@@ -0,0 +1,186 @@
+# coding:utf-8
+
+import sys
+import io
+import os
+import time
+import subprocess
+import re
+import json
+
+sys.path.append(os.getcwd() + "/class/core")
+import mw
+
+
+if mw.isAppleSystem():
+    cmd = 'ls /usr/local/lib/ | grep python  | cut -d \\  -f 1 | awk \'END {print}\''
+    info = mw.execShell(cmd)
+    p = "/usr/local/lib/" + info[0].strip() + "/site-packages"
+    sys.path.append(p)
+
+
+app_debug = False
+if mw.isAppleSystem():
+    app_debug = True
+
+
+def getPluginName():
+    return 'mysql'
+
+
+def getPluginDir():
+    return mw.getPluginDir() + '/' + getPluginName()
+
+
+def getSPluginDir():
+    return '/www/server/mdserver-web/plugins/' + getPluginName()
+
+
+def getServerDir():
+    return mw.getServerDir() + '/' + getPluginName()
+
+
+def getConf():
+    path = getServerDir() + '/etc/my.cnf'
+    return path
+
+
+def getDataDir():
+    file = getConf()
+    content = mw.readFile(file)
+    rep = 'datadir\s*=\s*(.*)'
+    tmp = re.search(rep, content)
+    return tmp.groups()[0].strip()
+
+
+def getRelayLogName():
+    file = getConf()
+    content = mw.readFile(file)
+    rep = 'relay-log\s*=\s*(.*)'
+    tmp = re.search(rep, content)
+    return tmp.groups()[0].strip()
+
+
+def getLogBinName():
+    file = getConf()
+    content = mw.readFile(file)
+    rep = 'log-bin\s*=\s*(.*)'
+    tmp = re.search(rep, content)
+    return tmp.groups()[0].strip()
+
+
+def binLogListLook(args):
+
+    file = args['file']
+    line = args['line']
+
+    data_dir = getDataDir()
+    my_bin = getServerDir() + '/bin'
+    my_binlog_cmd = my_bin + '/mysqlbinlog'
+
+    cmd = my_binlog_cmd + ' --no-defaults ' + \
+        data_dir + '/' + file + '|tail -' + str(line)
+
+    data = mw.execShell(cmd)
+
+    rdata = {}
+    rdata['cmd'] = cmd
+    rdata['data'] = data[0]
+
+    return rdata
+
+
+def binLogListLookDecode(args):
+
+    file = args['file']
+    line = args['line']
+
+    data_dir = getDataDir()
+    my_bin = getServerDir() + '/bin'
+    my_binlog_cmd = my_bin + '/mysqlbinlog'
+
+    cmd = my_binlog_cmd + ' --no-defaults --base64-output=decode-rows -vvvv ' + \
+        data_dir + '/' + file + '|tail -' + str(line)
+
+    data = mw.execShell(cmd)
+
+    rdata = {}
+    rdata['cmd'] = cmd
+    rdata['data'] = data[0]
+
+    return rdata
+
+
+def binLogListTraceRelay(args):
+    rdata = {}
+    file = args['file']
+    line = args['line']
+
+    relay_name = getRelayLogName()
+    data_dir = getDataDir()
+    alist = os.listdir(data_dir)
+    relay_list = []
+    for x in range(len(alist)):
+        f = alist[x]
+        t = {}
+        if f.startswith(relay_name) and not f.endswith('.index'):
+            relay_list.append(f)
+
+    relay_list = sorted(relay_list, reverse=True)
+    if len(relay_list) == 0:
+        rdata['cmd'] = ''
+        rdata['data'] = '无Relay日志'
+        return rdata
+
+    file = relay_list[0]
+
+    my_bin = getServerDir() + '/bin'
+    my_binlog_cmd = my_bin + '/mysqlbinlog'
+
+    cmd = my_binlog_cmd + ' --no-defaults --base64-output=decode-rows -vvvv ' + \
+        data_dir + '/' + file + '|tail -' + str(line)
+
+    data = mw.execShell(cmd)
+
+    rdata['cmd'] = cmd
+    rdata['data'] = data[0]
+
+    return rdata
+
+
+def binLogListTraceBinLog(args):
+    rdata = {}
+    file = args['file']
+    line = args['line']
+
+    data_dir = getDataDir()
+    log_bin_name = getLogBinName()
+
+    alist = os.listdir(data_dir)
+    log_bin_l = []
+    for x in range(len(alist)):
+        f = alist[x]
+        t = {}
+        if f.startswith(log_bin_name) and not f.endswith('.index'):
+            log_bin_l.append(f)
+
+    if len(log_bin_l) == 0:
+        rdata['cmd'] = ''
+        rdata['data'] = '无BINLOG'
+        return rdata
+
+    log_bin_l = sorted(log_bin_l, reverse=True)
+    file = log_bin_l[0]
+
+    my_bin = getServerDir() + '/bin'
+    my_binlog_cmd = my_bin + '/mysqlbinlog'
+
+    cmd = my_binlog_cmd + ' --no-defaults --base64-output=decode-rows -vvvv ' + \
+        data_dir + '/' + file + '|tail -' + str(line)
+
+    data = mw.execShell(cmd)
+
+    rdata['cmd'] = cmd
+    rdata['data'] = data[0]
+
+    return rdata

plugins/mysql/js/mysql.js

@@ -59,6 +59,63 @@ function myAsyncPost(method,args){
     return syncPost('/plugins/run', {name:'mysql', func:method, args:_args}); 
 }
 
+
+function myPostCallbak(method, version, args,callback){
+    var loadT = layer.msg('正在获取...', { icon: 16, time: 0, shade: 0.3 });
+
+    var req_data = {};
+    req_data['name'] = 'mysql';
+    req_data['func'] = method;
+    req_data['script']='index_mysql';
+    args['version'] = version;
+
+ 
+    if (typeof(args) == 'string' && args == ''){
+        req_data['args'] = JSON.stringify(toArrayObject(args));
+    } else {
+        req_data['args'] = JSON.stringify(args);
+    }
+
+    $.post('/plugins/callback', req_data, function(data) {
+        layer.close(loadT);
+        if (!data.status){
+            layer.msg(data.msg,{icon:0,time:2000,shade: [0.3, '#000']});
+            return;
+        }
+
+        if(typeof(callback) == 'function'){
+            callback(data);
+        }
+    },'json'); 
+}
+
+function myPostCallbakN(method, version, args,callback){
+
+    var req_data = {};
+    req_data['name'] = 'mysql';
+    req_data['func'] = method;
+    req_data['script']='index_mysql';
+    args['version'] = version;
+
+ 
+    if (typeof(args) == 'string' && args == ''){
+        req_data['args'] = JSON.stringify(toArrayObject(args));
+    } else {
+        req_data['args'] = JSON.stringify(args);
+    }
+
+    $.post('/plugins/callback', req_data, function(data) {
+        if (!data.status){
+            layer.msg(data.msg,{icon:0,time:2000,shade: [0.3, '#000']});
+            return;
+        }
+
+        if(typeof(callback) == 'function'){
+            callback(data);
+        }
+    },'json'); 
+}
+
 function vaildPhpmyadmin(url,username,password){
     // console.log("Authorization: Basic " + btoa(username + ":" + password));
     $.ajax({
@@ -1163,6 +1220,146 @@ function dbList(page, search){
 }
 
 
+function myBinRollingLogs(_name, func, _args, line){
+
+    var file_line = 100;
+    if ( typeof(line) != 'undefined' ){
+        file_line = line;
+    }
+
+    var reqTimer = null;
+
+    function requestLogs(func,file,line){
+        myPostCallbakN(func,'',{'file':file,"line":line}, function(rdata){
+            var data = rdata.data.data;
+            var cmd = rdata.data.cmd;
+            if(data == '') {
+                data = '当前没有日志!';
+            }
+
+            $('#my_rolling_cmd').html(cmd);
+
+            $('#my_rolling_copy').click(function(){
+                copyText(cmd);
+            });
+
+            var ebody = '<textarea readonly="readonly" style="margin: 0px;width: 100%;height: 570px;background-color: #333;color:#fff; padding:0 5px" id="roll_info_log">'+data+'</textarea>';
+            $("#my_rolling_logs").html(ebody);
+            var ob = document.getElementById('roll_info_log');
+            ob.scrollTop = ob.scrollHeight;
+        });
+    }
+
+
+    layer.open({
+        type: 1,
+        title: _name + '日志',
+        area: ['800px','700px'],
+        end: function(){
+            if (reqTimer){
+                clearInterval(reqTimer);
+            }
+        },
+        content:'<div class="change-default" style="padding:0px 20px 0px;">\
+                    <div class="divtable mtb10">\
+                    <table class="table table-hover"><tr>\
+                    <td id="my_rolling_cmd">cmd</td>\
+                    <td id="my_rolling_copy" style="width:35px;"><span class="ico-copy cursor btcopy" title="复制密码"></span></td>\
+                    <tr>\
+                    </table>\
+                    </div>\
+                </div>\
+                <div class="change-default" style="padding:0px 20px 0px;" id="my_rolling_logs">\
+                    <textarea readonly="readonly" style="margin: 0px;width: 100%;height: 570px;background-color: #333;color:#fff; padding:0 5px" id="roll_info_log"></textarea>\
+                </div>',
+        success:function(){
+            var fileName = _args['file'];
+            requestLogs(func,fileName,file_line);
+            reqTimer = setInterval(function(){
+                requestLogs(func,fileName,file_line);
+            },1000);
+        }
+    });
+}
+function myBinLogsRender(page){
+    var _data = {};
+    if (typeof(page) =='undefined'){
+        var page = 1;
+    }
+    
+    _data['page'] = page;
+    _data['page_size'] = 10;
+    _data['tojs'] = 'myBinLogsRender';
+    myPost('binlog_list', _data, function(data){
+        var rdata = $.parseJSON(data.data);
+        // console.log(rdata);
+        var list = '';
+        for(i in rdata.data){
+            list += '<tr>';
+
+            list += '<td>' + rdata.data[i]['name'] +'</td>';
+            list += '<td>' + toSize(rdata.data[i]['size'])+'</td>';
+            list += '<td>' + rdata.data[i]['time'] +'</td>';
+            
+
+            list += '<td style="text-align:right">';
+            list += '<a href="javascript:;" data-index="'+i+'" class="btlink look" class="btlink">查看</a> | ';
+            list += '<a href="javascript:;" data-index="'+i+'" class="btlink look_decode" class="btlink">解码查看</a>';
+            list += '</td></tr>';
+        }
+
+        if (rdata.data.length ==0){
+            list = '<tr><td colspan="4">无数据</td</tr>';
+        }
+
+        $("#binlog_list tbody").html(list);
+        $('#binlog_page').html(rdata.page);
+
+
+        $('#binlog_list .look').click(function(){
+            var i = $(this).data('index');
+            var file = rdata.data[i]['name'];
+            myBinRollingLogs('查看BINLOG','binLogListLook',{'file':file },100);
+        });
+
+        $('#binlog_list .look_decode').click(function(){
+            var i = $(this).data('index');
+            var file = rdata.data[i]['name'];
+            myBinRollingLogs('查看解码BINLOG','binLogListLookDecode',{'file':file },100);
+        });
+    });
+}
+
+function myBinLogs(){
+    var con = '<div class="safe bgw">\
+            <button class="btn btn-success btn-sm relay_trace" type="button" style="margin-right: 5px;">中继日志跟踪</button>\
+            <button class="btn btn-default btn-sm binlog_trace" type="button" style="margin-right: 5px;">最新BINLOG日志跟踪</button>\
+            <div id="binlog_list" class="divtable mtb10">\
+                <div class="tablescroll">\
+                    <table class="table table-hover" width="100%" cellspacing="0" cellpadding="0" border="0" style="border: 0 none;">\
+                    <thead><tr>\
+                    <th>文件名称</th>\
+                    <th>大小</th>\
+                    <th>时间</th>\
+                    <th style="text-align:right;">操作</th>\
+                    </tr></thead>\
+                    <tbody></tbody></table>\
+                </div>\
+                <div id="binlog_page" class="dataTables_paginate paging_bootstrap page"></div>\
+            </div>\
+        </div>';
+    $(".soft-man-con").html(con);
+    myBinLogsRender(1);
+
+    $('.soft-man-con .relay_trace').click(function(){
+        myBinRollingLogs('中继日志跟踪','binLogListTraceRelay',{'file':''},100);
+    });
+
+    $('.soft-man-con .binlog_trace').click(function(){
+        myBinRollingLogs('最新BINLOG日志跟踪','binLogListTraceBinLog',{'file':''},100);
+    });
+}
+
 function myLogs(){
     
     myPost('bin_log', {status:1}, function(data){
@@ -2346,11 +2543,16 @@ function masterOrSlaveConf(version=''){
                     <td>"+(info['Slave_SQL_Running_State'] == '' ? '无':info['Slave_SQL_Running_State']) +"</td>\
                 </tr>";
 
+
+                var btn_list = ['复制错误',"取消"];
+                if (info['Last_IO_Error'].search(/1236/i)>0){
+                    btn_list = ['复制错误',"取消","尝试修复"];
+                }
                 layer.open({
                     type: 1,
                     title: '同步异常信息',
                     area: ['600px','300px'],
-                    btn:['复制错误',"取消"],
+                    btn:btn_list,
                     content:"<form class='bt-form pd15'>\
                         <div class='divtable mtb10'>\
                         <div class='tablescroll'>\
@@ -2385,6 +2587,14 @@ function masterOrSlaveConf(version=''){
                             copyText(info['Slave_SQL_Running_State']);
                             return;
                         }
+                    },
+                    btn3:function(){
+                        myPost('try_slave_sync_bugfix', {}, function(data){
+                            var rdata = $.parseJSON(data.data);
+                            showMsg(rdata.msg, function(){
+                                masterOrSlaveConf();
+                            },{ icon: rdata.status ? 1 : 5 },2000);
+                        });
                     }
                 });
             });

plugins/php/versions/83/install.sh

@@ -52,7 +52,7 @@ if [ ! -d $sourcePath/php/php${PHP_VER} ];then
 	
 
 	if [ ! -f $sourcePath/php/php-${version}.tar.xz ];then
-		wget --no-check-certificate -O $sourcePath/php/php-${version}.tar.xz https://downloads.php.net/~eric/php-${version}.tar.xz
+		wget --no-check-certificate -O $sourcePath/php/php-${version}.tar.xz https://downloads.php.net/~jakub/php-${version}.tar.xz
 	fi
 
 	#检测文件是否损坏.

scripts/install/debian.sh

@@ -54,7 +54,8 @@ fi
 
 apt-get update -y
 apt install -y wget curl lsof unzip tar cron expect locate lrzsz
-apt install -y rar unrar
+apt install -y rar 
+apt install -y unrar
 apt install -y python3-pip python3-dev python3-venv
 
 
@@ -178,7 +179,9 @@ apt install -y libmagickwand-dev
 
 apt install -y libxml2 libxml2-dev libbz2-dev libmcrypt-dev libpspell-dev librecode-dev
 apt install -y libgmp-dev libgmp3-dev libreadline-dev libxpm-dev
-apt install -y dia pkg-config
+apt install -y dia
+
+apt install -y pkg-config
 apt install -y zlib1g-dev
 
 apt install -y libevent-dev libncurses5-dev libldap2-dev

scripts/install/ubuntu.sh

@@ -103,8 +103,11 @@ apt install -y libmagickwand-dev
 
 apt install -y libxml2 libxml2-dev libbz2-dev libmcrypt-dev libpspell-dev librecode-dev
 apt install -y libgmp-dev libgmp3-dev libreadline-dev libxpm-dev
-apt install -y dia pkg-config
+apt install -y dia
+
+apt install -y pkg-config
 apt install -y zlib1g-dev
+
 apt install -y libjpeg-dev libpng-dev
 apt install -y libfreetype6
 apt install -y libjpeg62-turbo-dev