midoks
/
mdserver-web
mirror of https://github.com/midoks/mdserver-web
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
pull/577/head
Mr Chen 12 months ago
parent e1d0fc49bb
commit cf5b986d2e
5 changed files with 193 additions and 191 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 134
      plugins/sphinx/class/sphinx_make.py
  2. 8
      plugins/sphinx/index.py
  3. 2
      plugins/sphinx/install.sh
  4. 1
      scripts/install/debian.sh
  5. 1
      scripts/install/euler.sh

134
plugins/sphinx/class/sphinx_make.py
Unescape View File

@ -54,10 +54,19 @@ def pMysqlDb():
db.setPwd(pSqliteDb('config').where('id=?', (1,)).getField('mysql_root'))
return db
def getTablePk(pdb, db, table):
class sphinxMake():
pdb = None
psdb = None
def __init__(self):
self.pdb = pMysqlDb()
def getTablePk(self, db, table):
# SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY'
pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,);
pkey_data = pdb.query(pkey_sql)
pkey_data = self.pdb.query(pkey_sql)
# print(db, table)
# print(pkey_data)
@ -67,19 +76,19 @@ def getTablePk(pdb, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';"
sql = sql.format(db,table,pkey_name,)
# print(sql)
fields = pdb.query(sql)
fields = self.pdb.query(sql)
if len(fields) == 1:
# print(fields[0]['DATA_TYPE'])
if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']):
return pkey_name
return ''
def getTableFieldStr(pdb, db, table):
def getTableFieldStr(self, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
fields = pdb.query(sql)
fields = self.pdb.query(sql)
field_str = ''
for x in range(len(fields)):
@ -88,7 +97,7 @@ def getTableFieldStr(pdb, db, table):
field_str = field_str.strip(',')
return field_str
def makeSphinxHeader():
def makeSphinxHeader(self):
conf = '''
indexer
{
@ -114,13 +123,12 @@ searchd
conf = conf.replace("{$server_dir}", mw.getServerDir())
return conf
def makeSphinxDbSourceRangeSql(pdb, db, table, pkey_name):
# pkey_name = getTablePk(pdb, db, table)
def makeSphinxDbSourceRangeSql(self, db, table, pkey_name):
sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table
return sql
def makeSphinxDbSourceQuerySql(pdb, db, table,pkey_name):
field_str = getTableFieldStr(pdb, db,table)
def makeSphinxDbSourceQuerySql(self, db, table,pkey_name):
field_str = self.getTableFieldStr(db,table)
# print(field_str)
if pkey_name == 'id':
sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end"
@ -128,14 +136,11 @@ def makeSphinxDbSourceQuerySql(pdb, db, table,pkey_name):
sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end"
return sql
def makeSphinxDbSource(pdb, db, table, pkey_name):
def makeSphinxDbSource(self, db, table, pkey_name):
db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find()
port = getDbPort()
conf = '''
source {$DB_NAME}_{$TABLE_NAME}
{
type = mysql
@ -162,7 +167,7 @@ index {$DB_NAME}_{$TABLE_NAME}
ngram_len = 1
ngram_chars = U+3000..U+2FA1F
}
'''
'''
conf = conf.replace("{$server_dir}", mw.getServerDir())
conf = conf.replace("{$DB_NAME}", db)
@ -171,82 +176,60 @@ index {$DB_NAME}_{$TABLE_NAME}
conf = conf.replace("{$DB_PASS}", db_info['password'])
conf = conf.replace("{$DB_PORT}", port)
range_sql = makeSphinxDbSourceRangeSql(pdb, db, table,pkey_name)
range_sql = self.makeSphinxDbSourceRangeSql(db, table,pkey_name)
conf = conf.replace("{$DB_RANGE_SQL}", range_sql)
query_sql = makeSphinxDbSourceQuerySql(pdb, db, table, pkey_name)
query_sql = self.makeSphinxDbSourceQuerySql(db, table, pkey_name)
conf = conf.replace("{$DB_QUERY_SQL}", query_sql)
sph_field = makeSqlToSphinxTable(pdb, db, table, pkey_name)
sph_field = self.makeSqlToSphinxTable(db, table, pkey_name)
conf = conf.replace("{$SPH_FIELD}", sph_field)
return conf
def makeSqlToSphinxTableIsHaveFulltext(pdb, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = pdb.query(sql)
cols_len = len(cols)
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
if mw.inArray(['varchar'], data_type):
return True
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
return True
return False
def makeSqlToSphinxAll():
filter_db = ['information_schema','performance_schema','sys','mysql']
pdb = pMysqlDb()
dblist = pdb.query('show databases')
conf = ''
conf += makeSphinxHeader()
# conf += makeSqlToSphinxDb(pdb, 'bbs')
for x in range(len(dblist)):
dbname = dblist[x]['Database']
if mw.inArray(filter_db, dbname):
continue
conf += makeSqlToSphinxDb(pdb, dbname)
return conf
def makeSqlToSphinxDb(pdb, db, table = []):
def makeSqlToSphinxDb(self, db, table = []):
conf = ''
for t in table:
pkey_name = getTablePk(pdb, db, t)
pkey_name = self.pdb.getTablePk(db, t)
if pkey_name == '':
continue
conf += makeSphinxDbSource(pdb, db, t, pkey_name)
conf += self.makeSphinxDbSource(db, t, pkey_name)
if len(table) == 0:
tables = pdb.query("show tables in "+ db)
tables = self.pdb.query("show tables in "+ db)
for x in range(len(tables)):
key = 'Tables_in_'+db
table_name = tables[x][key]
pkey_name = getTablePk(pdb, db, table_name)
pkey_name = self.getTablePk(db, table_name)
if pkey_name == '':
continue
if makeSqlToSphinxTableIsHaveFulltext(pdb, db, table_name):
conf += makeSphinxDbSource(pdb, db, table_name, pkey_name)
if self.makeSqlToSphinxTableIsHaveFulltext(db, table_name):
conf += self.makeSphinxDbSource(db, table_name, pkey_name)
return conf
def makeSqlToSphinxTable(pdb,db,table,pkey_name):
def makeSqlToSphinxTableIsHaveFulltext(self, db, table):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = self.pdb.query(sql)
cols_len = len(cols)
for x in range(cols_len):
data_type = cols[x]['DATA_TYPE']
column_name = cols[x]['COLUMN_NAME']
if mw.inArray(['varchar'], data_type):
return True
if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type):
return True
return False
def makeSqlToSphinxTable(self,db,table,pkey_name):
sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';"
sql = sql.format(db,table,)
cols = pdb.query(sql)
cols = self.pdb.query(sql)
cols_len = len(cols)
conf = ''
run_pos = 0
@ -302,10 +285,23 @@ def makeSqlToSphinxTable(pdb,db,table,pkey_name):
conf += '\tsql_attr_timestamp = '+ column_name + "\n"
continue
# if cols_len != run_pos:
# print(db,table)
# print(cols_len,run_pos)
# print(conf)
return conf
def makeSqlToSphinxAll(self):
filter_db = ['information_schema','performance_schema','sys','mysql']
dblist = self.pdb.query('show databases')
conf = ''
conf += self.makeSphinxHeader()
# conf += makeSqlToSphinxDb(pdb, 'bbs')
for x in range(len(dblist)):
dbname = dblist[x]['Database']
if mw.inArray(filter_db, dbname):
continue
conf += self.makeSqlToSphinxDb(dbname)
return conf

8
plugins/sphinx/index.py
Unescape View File

@ -319,12 +319,14 @@ def sphinxCmd():
return mw.returnJson(False, 'no index')
def makeDbToSphinx():
sphinx_file = getConf()
conf_file = getConf()
import sphinx_make
conf = sphinx_make.makeSqlToSphinxAll()
mw.writeFile(sphinx_file,conf)
sph_make = sphinx_make.sphinxMake()
conf = sph_make.makeSqlToSphinxAll()
mw.writeFile(conf_file,conf)
print(conf)
# makeSqlToSphinxTable()
return True

2
plugins/sphinx/install.sh
Unescape View File

@ -18,6 +18,8 @@ install_tmp=${rootPath}/tmp/mw_install.pl
# /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate
# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate
# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate
bash ${rootPath}/scripts/getos.sh
# echo "bash ${rootPath}/scripts/getos.sh"
OSNAME="macos"

1
scripts/install/debian.sh
Unescape View File

@ -57,6 +57,7 @@ apt install -y wget curl lsof unzip tar cron expect locate lrzsz
apt install -y rar
apt install -y unrar
apt install -y pv
apt install -y bc
apt install -y python3-pip python3-dev python3-venv

1
scripts/install/euler.sh
Unescape View File

@ -32,6 +32,7 @@ yum install -y libmcrypt-devel
yum install -y mysql-devel
yum install -y expect
yum install -y pv
yum install -y bc
# if [ -f /usr/sbin/iptables ];then

Write Preview
Loading…
Cancel
Save