midoks
/
mdserver-web
mirror of https://github.com/midoks/mdserver-web
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
pull/577/head
Mr Chen 12 months ago
parent e1d0fc49bb
commit cf5b986d2e
5 changed files with 193 additions and 191 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 372
      plugins/sphinx/class/sphinx_make.py
  2. 8
      plugins/sphinx/index.py
  3. 2
      plugins/sphinx/install.sh
  4. 1
      scripts/install/debian.sh
  5. 1
      scripts/install/euler.sh

372
plugins/sphinx/class/sphinx_make.py
Unescape View File

@ -54,42 +54,51 @@ def pMysqlDb():
db.setPwd(pSqliteDb('config').where('id=?', (1,)).getField('mysql_root'))
return db
def getTablePk(pdb, db, table):
# SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY'
pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,);
pkey_data = pdb.query(pkey_sql)
# print(db, table)
# print(pkey_data)
if len(pkey_data) == 1:
pkey_name = pkey_data[0]['Column_name']
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';"
sql = sql.format(db,table,pkey_name,)
# print(sql)
fields = pdb.query(sql)
if len(fields) == 1:
# print(fields[0]['DATA_TYPE'])
if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']):
return pkey_name
return ''
def getTableFieldStr(pdb, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
fields = pdb.query(sql)
field_str = ''
for x in range(len(fields)):
field_str += '`'+fields[x]['COLUMN_NAME']+'`,'
field_str = field_str.strip(',')
return field_str
def makeSphinxHeader():
conf = '''
class sphinxMake():
pdb = None
psdb = None
def __init__(self):
self.pdb = pMysqlDb()
def getTablePk(self, db, table):
# SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY'
pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,);
pkey_data = self.pdb.query(pkey_sql)
# print(db, table)
# print(pkey_data)
if len(pkey_data) == 1:
pkey_name = pkey_data[0]['Column_name']
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';"
sql = sql.format(db,table,pkey_name,)
# print(sql)
fields = self.pdb.query(sql)
if len(fields) == 1:
# print(fields[0]['DATA_TYPE'])
if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']):
return pkey_name
return ''
def getTableFieldStr(self, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
fields = self.pdb.query(sql)
field_str = ''
for x in range(len(fields)):
field_str += '`'+fields[x]['COLUMN_NAME']+'`,'
field_str = field_str.strip(',')
return field_str
def makeSphinxHeader(self):
conf = '''
indexer
{
mem_limit = 128M
@ -110,32 +119,28 @@ searchd
#workers = threads # for RT to work
binlog_path = {$server_dir}/sphinx/index/binlog
}
'''
conf = conf.replace("{$server_dir}", mw.getServerDir())
return conf
def makeSphinxDbSourceRangeSql(pdb, db, table, pkey_name):
# pkey_name = getTablePk(pdb, db, table)
sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table
return sql
def makeSphinxDbSourceQuerySql(pdb, db, table,pkey_name):
field_str = getTableFieldStr(pdb, db,table)
# print(field_str)
if pkey_name == 'id':
sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end"
else:
sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end"
return sql
def makeSphinxDbSource(pdb, db, table, pkey_name):
db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find()
port = getDbPort()
conf = '''
'''
conf = conf.replace("{$server_dir}", mw.getServerDir())
return conf
def makeSphinxDbSourceRangeSql(self, db, table, pkey_name):
sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table
return sql
def makeSphinxDbSourceQuerySql(self, db, table,pkey_name):
field_str = self.getTableFieldStr(db,table)
# print(field_str)
if pkey_name == 'id':
sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end"
else:
sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end"
return sql
def makeSphinxDbSource(self, db, table, pkey_name):
db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find()
port = getDbPort()
conf = '''
source {$DB_NAME}_{$TABLE_NAME}
{
type = mysql
@ -162,153 +167,144 @@ index {$DB_NAME}_{$TABLE_NAME}
ngram_len = 1
ngram_chars = U+3000..U+2FA1F
}
'''
conf = conf.replace("{$server_dir}", mw.getServerDir())
'''
conf = conf.replace("{$server_dir}", mw.getServerDir())
conf = conf.replace("{$DB_NAME}", db)
conf = conf.replace("{$TABLE_NAME}", table)
conf = conf.replace("{$DB_USER}", db_info['username'])
conf = conf.replace("{$DB_PASS}", db_info['password'])
conf = conf.replace("{$DB_PORT}", port)
range_sql = self.makeSphinxDbSourceRangeSql(db, table,pkey_name)
conf = conf.replace("{$DB_RANGE_SQL}", range_sql)
conf = conf.replace("{$DB_NAME}", db)
conf = conf.replace("{$TABLE_NAME}", table)
conf = conf.replace("{$DB_USER}", db_info['username'])
conf = conf.replace("{$DB_PASS}", db_info['password'])
conf = conf.replace("{$DB_PORT}", port)
query_sql = self.makeSphinxDbSourceQuerySql(db, table, pkey_name)
conf = conf.replace("{$DB_QUERY_SQL}", query_sql)
range_sql = makeSphinxDbSourceRangeSql(pdb, db, table,pkey_name)
conf = conf.replace("{$DB_RANGE_SQL}", range_sql)
sph_field = self.makeSqlToSphinxTable(db, table, pkey_name)
conf = conf.replace("{$SPH_FIELD}", sph_field)
query_sql = makeSphinxDbSourceQuerySql(pdb, db, table, pkey_name)
conf = conf.replace("{$DB_QUERY_SQL}", query_sql)
return conf
sph_field = makeSqlToSphinxTable(pdb, db, table, pkey_name)
conf = conf.replace("{$SPH_FIELD}", sph_field)
def makeSqlToSphinxDb(self, db, table = []):
conf = ''
for t in table:
pkey_name = self.pdb.getTablePk(db, t)
if pkey_name == '':
continue
conf += self.makeSphinxDbSource(db, t, pkey_name)
if len(table) == 0:
tables = self.pdb.query("show tables in "+ db)
for x in range(len(tables)):
key = 'Tables_in_'+db
table_name = tables[x][key]
pkey_name = self.getTablePk(db, table_name)
if pkey_name == '':
continue
if self.makeSqlToSphinxTableIsHaveFulltext(db, table_name):
conf += self.makeSphinxDbSource(db, table_name, pkey_name)
return conf
def makeSqlToSphinxTableIsHaveFulltext(self, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = self.pdb.query(sql)
cols_len = len(cols)
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
if mw.inArray(['varchar'], data_type):
return True
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
return True
return False
def makeSqlToSphinxTable(self,db,table,pkey_name):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = self.pdb.query(sql)
cols_len = len(cols)
conf = ''
run_pos = 0
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
# print(column_name+":"+data_type)
# if mw.inArray(['tinyint'], data_type):
# conf += 'sql_attr_bool = '+ column_name + "\n"
if pkey_name == column_name:
# run_pos += 1
# conf += '\tsql_attr_bigint = '+column_name+"\n"
continue
return conf
if mw.inArray(['enum'], data_type):
run_pos += 1
conf += '\tsql_attr_string = '+ column_name + "\n"
continue
def makeSqlToSphinxTableIsHaveFulltext(pdb, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = pdb.query(sql)
cols_len = len(cols)
if mw.inArray(['decimal'], data_type):
run_pos += 1
conf += '\tsql_attr_float = '+ column_name + "\n"
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type):
run_pos += 1
conf += '\tsql_attr_bigint = '+ column_name + "\n"
continue
if mw.inArray(['varchar'], data_type):
return True
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
return True
if mw.inArray(['float'], data_type):
run_pos += 1
conf += '\tsql_attr_float = '+ column_name + "\n"
continue
return False
if mw.inArray(['char'], data_type):
conf += '\tsql_attr_string = '+ column_name + "\n"
continue
def makeSqlToSphinxAll():
filter_db = ['information_schema','performance_schema','sys','mysql']
if mw.inArray(['varchar'], data_type):
run_pos += 1
conf += '\tsql_field_string = '+ column_name + "\n"
continue
pdb = pMysqlDb()
dblist = pdb.query('show databases')
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
run_pos += 1
conf += '\tsql_field_string = '+ column_name + "\n"
continue
conf = ''
conf += makeSphinxHeader()
if mw.inArray(['datetime','date'], data_type):
run_pos += 1
conf += '\tsql_attr_timestamp = '+ column_name + "\n"
continue
# conf += makeSqlToSphinxDb(pdb, 'bbs')
for x in range(len(dblist)):
dbname = dblist[x]['Database']
if mw.inArray(filter_db, dbname):
continue
conf += makeSqlToSphinxDb(pdb, dbname)
return conf
return conf
def makeSqlToSphinxAll(self):
filter_db = ['information_schema','performance_schema','sys','mysql']
def makeSqlToSphinxDb(pdb, db, table = []):
conf = ''
dblist = self.pdb.query('show databases')
for t in table:
pkey_name = getTablePk(pdb, db, t)
if pkey_name == '':
continue
conf += makeSphinxDbSource(pdb, db, t, pkey_name)
conf = ''
conf += self.makeSphinxHeader()
if len(table) == 0:
tables = pdb.query("show tables in "+ db)
for x in range(len(tables)):
key = 'Tables_in_'+db
table_name = tables[x][key]
pkey_name = getTablePk(pdb, db, table_name)
# conf += makeSqlToSphinxDb(pdb, 'bbs')
for x in range(len(dblist)):
dbname = dblist[x]['Database']
if mw.inArray(filter_db, dbname):
continue
conf += self.makeSqlToSphinxDb(dbname)
return conf
if pkey_name == '':
continue
if makeSqlToSphinxTableIsHaveFulltext(pdb, db, table_name):
conf += makeSphinxDbSource(pdb, db, table_name, pkey_name)
return conf
def makeSqlToSphinxTable(pdb,db,table,pkey_name):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = pdb.query(sql)
cols_len = len(cols)
conf = ''
run_pos = 0
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
# print(column_name+":"+data_type)
# if mw.inArray(['tinyint'], data_type):
# conf += 'sql_attr_bool = '+ column_name + "\n"
if pkey_name == column_name:
# run_pos += 1
# conf += '\tsql_attr_bigint = '+column_name+"\n"
continue
if mw.inArray(['enum'], data_type):
run_pos += 1
conf += '\tsql_attr_string = '+ column_name + "\n"
continue
if mw.inArray(['decimal'], data_type):
run_pos += 1
conf += '\tsql_attr_float = '+ column_name + "\n"
if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type):
run_pos += 1
conf += '\tsql_attr_bigint = '+ column_name + "\n"
continue
if mw.inArray(['float'], data_type):
run_pos += 1
conf += '\tsql_attr_float = '+ column_name + "\n"
continue
if mw.inArray(['char'], data_type):
conf += '\tsql_attr_string = '+ column_name + "\n"
continue
if mw.inArray(['varchar'], data_type):
run_pos += 1
conf += '\tsql_field_string = '+ column_name + "\n"
continue
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
run_pos += 1
conf += '\tsql_field_string = '+ column_name + "\n"
continue
if mw.inArray(['datetime','date'], data_type):
run_pos += 1
conf += '\tsql_attr_timestamp = '+ column_name + "\n"
continue
# if cols_len != run_pos:
# print(db,table)
# print(cols_len,run_pos)
# print(conf)
return conf

8
plugins/sphinx/index.py
Unescape View File

@ -319,12 +319,14 @@ def sphinxCmd():
return mw.returnJson(False, 'no index')
def makeDbToSphinx():
sphinx_file = getConf()
conf_file = getConf()
import sphinx_make
conf = sphinx_make.makeSqlToSphinxAll()
mw.writeFile(sphinx_file,conf)
sph_make = sphinx_make.sphinxMake()
conf = sph_make.makeSqlToSphinxAll()
mw.writeFile(conf_file,conf)
print(conf)
# makeSqlToSphinxTable()
return True

2
plugins/sphinx/install.sh
Unescape View File

@ -18,6 +18,8 @@ install_tmp=${rootPath}/tmp/mw_install.pl
# /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate
# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate
# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate
bash ${rootPath}/scripts/getos.sh
# echo "bash ${rootPath}/scripts/getos.sh"
OSNAME="macos"

1
scripts/install/debian.sh
Unescape View File

@ -57,6 +57,7 @@ apt install -y wget curl lsof unzip tar cron expect locate lrzsz
apt install -y rar
apt install -y unrar
apt install -y pv
apt install -y bc
apt install -y python3-pip python3-dev python3-venv

1
scripts/install/euler.sh
Unescape View File

@ -32,6 +32,7 @@ yum install -y libmcrypt-devel
yum install -y mysql-devel
yum install -y expect
yum install -y pv
yum install -y bc
# if [ -f /usr/sbin/iptables ];then

Write Preview
Loading…
Cancel
Save