diff --git a/plugins/sphinx/class/sphinx_make.py b/plugins/sphinx/class/sphinx_make.py index 94ff3bddf..5d2b9b1ea 100644 --- a/plugins/sphinx/class/sphinx_make.py +++ b/plugins/sphinx/class/sphinx_make.py @@ -54,12 +54,151 @@ def pMysqlDb(): db.setPwd(pSqliteDb('config').where('id=?', (1,)).getField('mysql_root')) return db +def getTablePk(db, table): + pdb = pMysqlDb() + pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,); + pkey_data = pdb.query(pkey_sql) + + if len(pkey_data) == 1: + return pkey_data[0]['Column_name'] + + return '' + +def getTableFieldStr(db, table): + pdb = pMysqlDb() + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + fields = pdb.query(sql) + + field_str = '' + for x in range(len(fields)): + field_str += fields[x]['COLUMN_NAME']+',' + + field_str = field_str.strip(',') + return field_str + + def makeSqlToSphinx(): pass -def makeSqlToSphinxDb(db, table = []): +def makeSphinxHeader(): + conf = ''' +indexer +{ + mem_limit = 128M +} + + searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$server_dir}/sphinx/index/searchd.log + query_log = {$server_dir}/sphinx/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$server_dir}/sphinx/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$server_dir}/sphinx/index/binlog +} + ''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + return conf + +def makeSphinxDbSourceRangeSql(db, table): + pdb = pMysqlDb() + pkey_name = getTablePk(db, table) + sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table + return sql + +def makeSphinxDbSourceQuerySql(db, table): pdb = pMysqlDb() + pkey_name = getTablePk(db, table) + field_str = getTableFieldStr(db,table) + # print(field_str) + sql = "SELECT "+field_str+" FROM "+table+ " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end" + return sql + +def makeSphinxDbSource(db, table): + + db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find() + port = getDbPort() + + conf = ''' + +source {$DB_NAME}_{$TABLE_NAME} +{ + type = mysql + sql_host = 127.0.0.1 + sql_user = {$DB_USER} + sql_pass = {$DB_PASS} + sql_db = {$DB_NAME} + sql_port = {$DB_PORT} + + sql_query_range = {$DB_RANGE_SQL} + sql_range_step = 1000 + + sql_query_pre = SET NAMES utf8bm4 + sql_query = {$DB_QUERY_SQL} + +{$SPH_FIELD} +} + +index {$DB_NAME}_{$TABLE_NAME} +{ + source = {$DB_NAME}_{$TABLE_NAME} + path = {$server_dir}/sphinx/index/db/{$DB_NAME}.{$TABLE_NAME}/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} +''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + conf = conf.replace("{$DB_USER}", db_info['username']) + conf = conf.replace("{$DB_PASS}", db_info['password']) + conf = conf.replace("{$DB_PORT}", port) + + range_sql = makeSphinxDbSourceRangeSql(db,table) + conf = conf.replace("{$DB_RANGE_SQL}", range_sql) + query_sql = makeSphinxDbSourceQuerySql(db,table) + conf = conf.replace("{$DB_QUERY_SQL}", query_sql) + + sph_field = makeSqlToSphinxTable(db,table) + conf = conf.replace("{$SPH_FIELD}", sph_field) + + + + return conf + + +def makeSqlToSphinxAll(): + + filter_db = ['information_schema','performance_schema','sys','mysql'] + + pdb = pMysqlDb() + dblist = pdb.query('show databases') + + conf = '' + conf += makeSphinxHeader() + for x in range(len(dblist)): + dbname = dblist[x]['Database'] + if mw.inArray(filter_db, dbname): + continue + conf += makeSqlToSphinxDb(dbname) + return conf + + + +def makeSqlToSphinxDb(db, table = []): + conf = '' + + pdb = pMysqlDb() tables = pdb.query("show tables in "+ db) for x in range(len(tables)): key = 'Tables_in_'+db @@ -69,19 +208,12 @@ def makeSqlToSphinxDb(db, table = []): if pkey_name == '': continue + conf += makeSphinxDbSource(db,table_name) + # print(conf) # print(table_name+':'+pkey_name) - makeSqlToSphinxTable(db,table_name) - - -def getTablePk(db, table): - pdb = pMysqlDb() - pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,); - pkey_data = pdb.query(pkey_sql) - - if len(pkey_data) == 1: - return pkey_data[0]['Column_name'] - - return '' + # db_field_str = makeSqlToSphinxTable(db,table_name) + # print(db_field_str) + return conf def makeSqlToSphinxTable(db,table): pdb = pMysqlDb() @@ -95,46 +227,43 @@ def makeSqlToSphinxTable(db,table): for x in range(cols_len): data_type = cols[x]['DATA_TYPE'] column_name = cols[x]['COLUMN_NAME'] - print(column_name+":"+data_type) + # print(column_name+":"+data_type) # if mw.inArray(['tinyint'], data_type): # conf += 'sql_attr_bool = '+ column_name + "\n" if mw.inArray(['enum'], data_type): run_pos += 1 - conf += 'sql_attr_string = '+ column_name + "\n" + conf += '\tsql_attr_string = '+ column_name + "\n" if mw.inArray(['decimal'], data_type): run_pos += 1 - conf += 'sql_attr_float = '+ column_name + "\n" + conf += '\tsql_attr_float = '+ column_name + "\n" if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type): run_pos += 1 - conf += 'sql_attr_bigint = '+ column_name + "\n" + conf += '\tsql_attr_bigint = '+ column_name + "\n" if mw.inArray(['float'], data_type): run_pos += 1 - conf += 'sql_attr_float = '+ column_name + "\n" + conf += '\tsql_attr_float = '+ column_name + "\n" if mw.inArray(['varchar','char'], data_type): run_pos += 1 - conf += 'sql_attr_string = '+ column_name + "\n" + conf += '\tsql_attr_string = '+ column_name + "\n" if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): run_pos += 1 - conf += 'sql_field_string = '+ column_name + "\n" + conf += '\tsql_field_string = '+ column_name + "\n" if mw.inArray(['datetime','date'], data_type): run_pos += 1 - conf += 'sql_attr_timestamp = '+ column_name + "\n" - - if cols_len != run_pos: - print(db,table) - - print(cols_len,run_pos) - + conf += '\tsql_attr_timestamp = '+ column_name + "\n" + # if cols_len != run_pos: + # print(db,table) + # print(cols_len,run_pos) # print(conf) return conf diff --git a/plugins/sphinx/index.py b/plugins/sphinx/index.py index 0903cd9c7..8c8ddb70b 100755 --- a/plugins/sphinx/index.py +++ b/plugins/sphinx/index.py @@ -210,6 +210,7 @@ def reload(): def rebuild(): file = initDreplace() cmd = file + ' rebuild &' + # print(cmd) data = mw.execShell(cmd) if data[0].find('successfully')<0: return data[0].replace("\n","
") @@ -319,25 +320,16 @@ def sphinxCmd(): def makeDbToSphinx(): + sphinx_file = getConf() - import sphinx_make - pdb = sphinx_make.pMysqlDb() - - filter_db = ['information_schema','performance_schema','sys','mysql'] - - db_list = pdb.query('show databases') - print(db_list) - for x in range(len(db_list)): - dbname = db_list[x]['Database'] - if mw.inArray(filter_db, dbname): - continue - - sphinx_make.makeSqlToSphinxDb(dbname) + import sphinx_make + conf = sphinx_make.makeSqlToSphinxAll() - print(dbname) + mw.writeFile(sphinx_file,conf) + print(conf) - # makeSqlToSphinxTable() + # makeSqlToSphinxTable() diff --git a/plugins/sphinx/install.sh b/plugins/sphinx/install.sh index 3b636167f..831b7e0c6 100755 --- a/plugins/sphinx/install.sh +++ b/plugins/sphinx/install.sh @@ -11,7 +11,7 @@ sysName=`uname` install_tmp=${rootPath}/tmp/mw_install.pl # cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py rebuild -# /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --rotate +# /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate # cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py db_to_sphinx