From cf5b986d2eab4ad004496bddda5945ff0081fa05 Mon Sep 17 00:00:00 2001 From: Mr Chen Date: Mon, 20 May 2024 14:11:27 +0800 Subject: [PATCH] update --- plugins/sphinx/class/sphinx_make.py | 372 ++++++++++++++-------------- plugins/sphinx/index.py | 8 +- plugins/sphinx/install.sh | 2 + scripts/install/debian.sh | 1 + scripts/install/euler.sh | 1 + 5 files changed, 193 insertions(+), 191 deletions(-) diff --git a/plugins/sphinx/class/sphinx_make.py b/plugins/sphinx/class/sphinx_make.py index e8a86f6fc..62aa9f6ef 100644 --- a/plugins/sphinx/class/sphinx_make.py +++ b/plugins/sphinx/class/sphinx_make.py @@ -54,42 +54,51 @@ def pMysqlDb(): db.setPwd(pSqliteDb('config').where('id=?', (1,)).getField('mysql_root')) return db -def getTablePk(pdb, db, table): - # SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY' - pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,); - pkey_data = pdb.query(pkey_sql) - - # print(db, table) - # print(pkey_data) - - if len(pkey_data) == 1: - pkey_name = pkey_data[0]['Column_name'] - sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';" - sql = sql.format(db,table,pkey_name,) - # print(sql) - fields = pdb.query(sql) - - if len(fields) == 1: - # print(fields[0]['DATA_TYPE']) - if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']): - return pkey_name - - return '' - -def getTableFieldStr(pdb, db, table): - sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" - sql = sql.format(db,table,) - fields = pdb.query(sql) - - field_str = '' - for x in range(len(fields)): - field_str += '`'+fields[x]['COLUMN_NAME']+'`,' - - field_str = field_str.strip(',') - return field_str - -def makeSphinxHeader(): - conf = ''' +class sphinxMake(): + + pdb = None + psdb = None + def __init__(self): + self.pdb = pMysqlDb() + + + def getTablePk(self, db, table): + + # SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY' + pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,); + pkey_data = self.pdb.query(pkey_sql) + + # print(db, table) + # print(pkey_data) + + if len(pkey_data) == 1: + pkey_name = pkey_data[0]['Column_name'] + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';" + sql = sql.format(db,table,pkey_name,) + # print(sql) + fields = self.pdb.query(sql) + + if len(fields) == 1: + # print(fields[0]['DATA_TYPE']) + if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']): + return pkey_name + return '' + + + def getTableFieldStr(self, db, table): + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + fields = self.pdb.query(sql) + + field_str = '' + for x in range(len(fields)): + field_str += '`'+fields[x]['COLUMN_NAME']+'`,' + + field_str = field_str.strip(',') + return field_str + + def makeSphinxHeader(self): + conf = ''' indexer { mem_limit = 128M @@ -110,32 +119,28 @@ searchd #workers = threads # for RT to work binlog_path = {$server_dir}/sphinx/index/binlog } - ''' - conf = conf.replace("{$server_dir}", mw.getServerDir()) - return conf - -def makeSphinxDbSourceRangeSql(pdb, db, table, pkey_name): - # pkey_name = getTablePk(pdb, db, table) - sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table - return sql - -def makeSphinxDbSourceQuerySql(pdb, db, table,pkey_name): - field_str = getTableFieldStr(pdb, db,table) - # print(field_str) - if pkey_name == 'id': - sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end" - else: - sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end" - return sql - -def makeSphinxDbSource(pdb, db, table, pkey_name): - - db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find() - port = getDbPort() - - - conf = ''' - + ''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + return conf + + def makeSphinxDbSourceRangeSql(self, db, table, pkey_name): + sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table + return sql + + def makeSphinxDbSourceQuerySql(self, db, table,pkey_name): + field_str = self.getTableFieldStr(db,table) + # print(field_str) + if pkey_name == 'id': + sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end" + else: + sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end" + return sql + + def makeSphinxDbSource(self, db, table, pkey_name): + db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find() + port = getDbPort() + + conf = ''' source {$DB_NAME}_{$TABLE_NAME} { type = mysql @@ -162,153 +167,144 @@ index {$DB_NAME}_{$TABLE_NAME} ngram_len = 1 ngram_chars = U+3000..U+2FA1F } -''' - conf = conf.replace("{$server_dir}", mw.getServerDir()) + ''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + conf = conf.replace("{$DB_USER}", db_info['username']) + conf = conf.replace("{$DB_PASS}", db_info['password']) + conf = conf.replace("{$DB_PORT}", port) + + range_sql = self.makeSphinxDbSourceRangeSql(db, table,pkey_name) + conf = conf.replace("{$DB_RANGE_SQL}", range_sql) - conf = conf.replace("{$DB_NAME}", db) - conf = conf.replace("{$TABLE_NAME}", table) - conf = conf.replace("{$DB_USER}", db_info['username']) - conf = conf.replace("{$DB_PASS}", db_info['password']) - conf = conf.replace("{$DB_PORT}", port) + query_sql = self.makeSphinxDbSourceQuerySql(db, table, pkey_name) + conf = conf.replace("{$DB_QUERY_SQL}", query_sql) - range_sql = makeSphinxDbSourceRangeSql(pdb, db, table,pkey_name) - conf = conf.replace("{$DB_RANGE_SQL}", range_sql) + sph_field = self.makeSqlToSphinxTable(db, table, pkey_name) + conf = conf.replace("{$SPH_FIELD}", sph_field) - query_sql = makeSphinxDbSourceQuerySql(pdb, db, table, pkey_name) - conf = conf.replace("{$DB_QUERY_SQL}", query_sql) + return conf - sph_field = makeSqlToSphinxTable(pdb, db, table, pkey_name) - conf = conf.replace("{$SPH_FIELD}", sph_field) + def makeSqlToSphinxDb(self, db, table = []): + conf = '' + + for t in table: + pkey_name = self.pdb.getTablePk(db, t) + if pkey_name == '': + continue + conf += self.makeSphinxDbSource(db, t, pkey_name) + + if len(table) == 0: + tables = self.pdb.query("show tables in "+ db) + for x in range(len(tables)): + key = 'Tables_in_'+db + table_name = tables[x][key] + pkey_name = self.getTablePk(db, table_name) + + if pkey_name == '': + continue + + if self.makeSqlToSphinxTableIsHaveFulltext(db, table_name): + conf += self.makeSphinxDbSource(db, table_name, pkey_name) + return conf + + def makeSqlToSphinxTableIsHaveFulltext(self, db, table): + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + cols = self.pdb.query(sql) + cols_len = len(cols) + + for x in range(cols_len): + data_type = cols[x]['DATA_TYPE'] + column_name = cols[x]['COLUMN_NAME'] + + if mw.inArray(['varchar'], data_type): + return True + if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): + return True + return False + + def makeSqlToSphinxTable(self,db,table,pkey_name): + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + cols = self.pdb.query(sql) + cols_len = len(cols) + conf = '' + run_pos = 0 + for x in range(cols_len): + data_type = cols[x]['DATA_TYPE'] + column_name = cols[x]['COLUMN_NAME'] + # print(column_name+":"+data_type) + + # if mw.inArray(['tinyint'], data_type): + # conf += 'sql_attr_bool = '+ column_name + "\n" + + if pkey_name == column_name: + # run_pos += 1 + # conf += '\tsql_attr_bigint = '+column_name+"\n" + continue - return conf + if mw.inArray(['enum'], data_type): + run_pos += 1 + conf += '\tsql_attr_string = '+ column_name + "\n" + continue -def makeSqlToSphinxTableIsHaveFulltext(pdb, db, table): - sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" - sql = sql.format(db,table,) - cols = pdb.query(sql) - cols_len = len(cols) + if mw.inArray(['decimal'], data_type): + run_pos += 1 + conf += '\tsql_attr_float = '+ column_name + "\n" - for x in range(cols_len): - data_type = cols[x]['DATA_TYPE'] - column_name = cols[x]['COLUMN_NAME'] + if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type): + run_pos += 1 + conf += '\tsql_attr_bigint = '+ column_name + "\n" + continue - if mw.inArray(['varchar'], data_type): - return True - if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): - return True + if mw.inArray(['float'], data_type): + run_pos += 1 + conf += '\tsql_attr_float = '+ column_name + "\n" + continue - return False + if mw.inArray(['char'], data_type): + conf += '\tsql_attr_string = '+ column_name + "\n" + continue -def makeSqlToSphinxAll(): - filter_db = ['information_schema','performance_schema','sys','mysql'] + if mw.inArray(['varchar'], data_type): + run_pos += 1 + conf += '\tsql_field_string = '+ column_name + "\n" + continue - pdb = pMysqlDb() - dblist = pdb.query('show databases') + if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): + run_pos += 1 + conf += '\tsql_field_string = '+ column_name + "\n" + continue - conf = '' - conf += makeSphinxHeader() + if mw.inArray(['datetime','date'], data_type): + run_pos += 1 + conf += '\tsql_attr_timestamp = '+ column_name + "\n" + continue - # conf += makeSqlToSphinxDb(pdb, 'bbs') - for x in range(len(dblist)): - dbname = dblist[x]['Database'] - if mw.inArray(filter_db, dbname): - continue - conf += makeSqlToSphinxDb(pdb, dbname) - return conf + return conf + def makeSqlToSphinxAll(self): + filter_db = ['information_schema','performance_schema','sys','mysql'] -def makeSqlToSphinxDb(pdb, db, table = []): - conf = '' + dblist = self.pdb.query('show databases') - for t in table: - pkey_name = getTablePk(pdb, db, t) - if pkey_name == '': - continue - conf += makeSphinxDbSource(pdb, db, t, pkey_name) + conf = '' + conf += self.makeSphinxHeader() - if len(table) == 0: - tables = pdb.query("show tables in "+ db) - for x in range(len(tables)): - key = 'Tables_in_'+db - table_name = tables[x][key] - pkey_name = getTablePk(pdb, db, table_name) + # conf += makeSqlToSphinxDb(pdb, 'bbs') + for x in range(len(dblist)): + dbname = dblist[x]['Database'] + if mw.inArray(filter_db, dbname): + continue + conf += self.makeSqlToSphinxDb(dbname) + return conf - if pkey_name == '': - continue - if makeSqlToSphinxTableIsHaveFulltext(pdb, db, table_name): - conf += makeSphinxDbSource(pdb, db, table_name, pkey_name) - return conf - -def makeSqlToSphinxTable(pdb,db,table,pkey_name): - - sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" - sql = sql.format(db,table,) - cols = pdb.query(sql) - cols_len = len(cols) - conf = '' - run_pos = 0 - for x in range(cols_len): - data_type = cols[x]['DATA_TYPE'] - column_name = cols[x]['COLUMN_NAME'] - # print(column_name+":"+data_type) - - # if mw.inArray(['tinyint'], data_type): - # conf += 'sql_attr_bool = '+ column_name + "\n" - - if pkey_name == column_name: - # run_pos += 1 - # conf += '\tsql_attr_bigint = '+column_name+"\n" - continue - - if mw.inArray(['enum'], data_type): - run_pos += 1 - conf += '\tsql_attr_string = '+ column_name + "\n" - continue - - if mw.inArray(['decimal'], data_type): - run_pos += 1 - conf += '\tsql_attr_float = '+ column_name + "\n" - - if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type): - run_pos += 1 - conf += '\tsql_attr_bigint = '+ column_name + "\n" - continue - - - if mw.inArray(['float'], data_type): - run_pos += 1 - conf += '\tsql_attr_float = '+ column_name + "\n" - continue - - if mw.inArray(['char'], data_type): - conf += '\tsql_attr_string = '+ column_name + "\n" - continue - - if mw.inArray(['varchar'], data_type): - run_pos += 1 - conf += '\tsql_field_string = '+ column_name + "\n" - continue - - if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): - run_pos += 1 - conf += '\tsql_field_string = '+ column_name + "\n" - continue - - if mw.inArray(['datetime','date'], data_type): - run_pos += 1 - conf += '\tsql_attr_timestamp = '+ column_name + "\n" - continue - - # if cols_len != run_pos: - # print(db,table) - # print(cols_len,run_pos) - # print(conf) - return conf - - diff --git a/plugins/sphinx/index.py b/plugins/sphinx/index.py index 894cf22da..915ea1331 100755 --- a/plugins/sphinx/index.py +++ b/plugins/sphinx/index.py @@ -319,12 +319,14 @@ def sphinxCmd(): return mw.returnJson(False, 'no index') def makeDbToSphinx(): - sphinx_file = getConf() + conf_file = getConf() import sphinx_make - conf = sphinx_make.makeSqlToSphinxAll() - mw.writeFile(sphinx_file,conf) + sph_make = sphinx_make.sphinxMake() + conf = sph_make.makeSqlToSphinxAll() + + mw.writeFile(conf_file,conf) print(conf) # makeSqlToSphinxTable() return True diff --git a/plugins/sphinx/install.sh b/plugins/sphinx/install.sh index cda14b952..9a1954fb3 100755 --- a/plugins/sphinx/install.sh +++ b/plugins/sphinx/install.sh @@ -18,6 +18,8 @@ install_tmp=${rootPath}/tmp/mw_install.pl # /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate # /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate +# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate + bash ${rootPath}/scripts/getos.sh # echo "bash ${rootPath}/scripts/getos.sh" OSNAME="macos" diff --git a/scripts/install/debian.sh b/scripts/install/debian.sh index 782168e08..104a2b53c 100644 --- a/scripts/install/debian.sh +++ b/scripts/install/debian.sh @@ -57,6 +57,7 @@ apt install -y wget curl lsof unzip tar cron expect locate lrzsz apt install -y rar apt install -y unrar apt install -y pv +apt install -y bc apt install -y python3-pip python3-dev python3-venv diff --git a/scripts/install/euler.sh b/scripts/install/euler.sh index 22b1d308a..320ab0700 100755 --- a/scripts/install/euler.sh +++ b/scripts/install/euler.sh @@ -32,6 +32,7 @@ yum install -y libmcrypt-devel yum install -y mysql-devel yum install -y expect yum install -y pv +yum install -y bc # if [ -f /usr/sbin/iptables ];then