From 011949ba853d1a9af81f866a121faafdda2e1e53 Mon Sep 17 00:00:00 2001 From: midoks Date: Mon, 17 Oct 2022 00:44:04 +0800 Subject: [PATCH] up --- plugins/op_waf/waf/lua/common.lua | 10 + plugins/op_waf/waf/lua/init.lua | 9 +- plugins/webstats/index.py | 23 +- plugins/webstats/info.json | 2 +- plugins/webstats/install.sh | 22 +- plugins/webstats/lua/webstats_common.lua | 384 +++++++++++++++++++++++ plugins/webstats/lua/webstats_log.lua | 130 ++++---- plugins/webstats/t/bench/bench.sh | 24 ++ plugins/webstats/t/bench/simple.lua | 18 ++ plugins/webstats/t/bench/test_today.lua | 33 ++ plugins/webstats/t/index.py | 95 ++++++ plugins/webstats/t/test.sh | 6 + 12 files changed, 663 insertions(+), 93 deletions(-) create mode 100644 plugins/webstats/lua/webstats_common.lua create mode 100755 plugins/webstats/t/bench/bench.sh create mode 100755 plugins/webstats/t/bench/simple.lua create mode 100644 plugins/webstats/t/bench/test_today.lua create mode 100644 plugins/webstats/t/index.py create mode 100755 plugins/webstats/t/test.sh diff --git a/plugins/op_waf/waf/lua/common.lua b/plugins/op_waf/waf/lua/common.lua index 113455cc0..28aa39c20 100644 --- a/plugins/op_waf/waf/lua/common.lua +++ b/plugins/op_waf/waf/lua/common.lua @@ -30,6 +30,8 @@ function _M.new(self) end + + function _M.getInstance(self) if rawget(self, "instance") == nil then rawset(self, "instance", self.new()) @@ -258,6 +260,14 @@ function _M.write_to_file(self, logstr) return true end +-- 是否文件迁入数据库中 +function _M.is_migrating(self) + local migrating = self.waf_root +"/migrating" + local file = io.open(migrating, "rb") + if file then return true end + return false +end + function _M.continue_key(self,key) key = tostring(key) diff --git a/plugins/op_waf/waf/lua/init.lua b/plugins/op_waf/waf/lua/init.lua index 4a320b44f..10957f173 100644 --- a/plugins/op_waf/waf/lua/init.lua +++ b/plugins/op_waf/waf/lua/init.lua @@ -4,8 +4,6 @@ local ngx_match = ngx.re.find local __C = require "common" local C = __C:getInstance() -local waf_root = "{$WAF_ROOT}" - local config = require "config" local site_config = require "site" local config_domains = require "domains" @@ -13,9 +11,6 @@ local config_domains = require "domains" C:setConfData(config, site_config) C:setDebug(true) --- local ngx_os = require "os" --- C:("msss".. ngx_os.name) - local get_html = require "html_get" local post_html = require "html_post" @@ -335,9 +330,9 @@ end local function waf_url() if not config['get']['open'] or not C:is_site_config('get') then return false end --正则-- - if C:is_ngx_match(url_rules,params["uri"],'url') then + if C:is_ngx_match(url_rules, params["uri"], 'url') then C:write_log('url','regular') - C:return_html(config['get']['status'],get_html) + C:return_html(config['get']['status'], get_html) return true end return false diff --git a/plugins/webstats/index.py b/plugins/webstats/index.py index fd9d4c44a..e2e8f3eb1 100755 --- a/plugins/webstats/index.py +++ b/plugins/webstats/index.py @@ -77,11 +77,11 @@ def status(): return 'start' -def loadLuaLogFile(): +def loadLuaFile(name): lua_dir = getServerDir() + "/lua" - lua_dst = lua_dir + "/webstats_log.lua" + lua_dst = lua_dir + "/" + name - lua_tpl = getPluginDir() + '/lua/webstats_log.lua' + lua_tpl = getPluginDir() + '/lua/' + name content = mw.readFile(lua_tpl) content = content.replace('{$SERVER_APP}', getServerDir()) content = content.replace('{$ROOT_PATH}', mw.getServerDir()) @@ -98,7 +98,7 @@ def loadConfigFile(): dst_conf_json = getServerDir() + "/lua/config.json" mw.writeFile(dst_conf_json, json.dumps(content)) - dst_conf_lua = getServerDir() + "/lua/config.lua" + dst_conf_lua = getServerDir() + "/lua/webstats_config.lua" listToLuaFile(dst_conf_lua, content) @@ -127,7 +127,7 @@ def loadLuaSiteFile(): ddata["default"] = dlist[0] mw.writeFile(default_json, json.dumps(ddata)) - lua_site = lua_dir + "/sites.lua" + lua_site = lua_dir + "/webstats_sites.lua" listToLuaFile(lua_site, content) @@ -205,7 +205,14 @@ def initDreplace(): if not os.path.exists(log_path): mw.execShell('mkdir -p ' + log_path) - loadLuaLogFile() + file_list = [ + 'webstats_common.lua', + 'webstats_log.lua', + ] + + for fl in file_list: + loadLuaFile(fl) + loadConfigFile() loadLuaSiteFile() loadDebugLogFile() @@ -294,7 +301,7 @@ def setGlobalConf(): content['global']['exclude_url'] = exclude_url_val mw.writeFile(conf, json.dumps(content)) - conf_lua = getServerDir() + "/lua/config.lua" + conf_lua = getServerDir() + "/lua/webstats_config.lua" listToLuaFile(conf_lua, content) mw.restartWeb() return mw.returnJson(True, '设置成功') @@ -387,7 +394,7 @@ def setSiteConf(): content[domain] = site_conf mw.writeFile(conf, json.dumps(content)) - conf_lua = getServerDir() + "/lua/config.lua" + conf_lua = getServerDir() + "/lua/webstats_config.lua" listToLuaFile(conf_lua, content) mw.restartWeb() return mw.returnJson(True, '设置成功') diff --git a/plugins/webstats/info.json b/plugins/webstats/info.json index b710aa1d4..aa5ec163c 100755 --- a/plugins/webstats/info.json +++ b/plugins/webstats/info.json @@ -4,7 +4,7 @@ "name": "webstats", "title": "网站统计", "shell": "install.sh", - "versions":["0.2.0"], + "versions":["0.2.2"], "tip": "soft", "install_pre_inspection":true, "checks": "server/webstats", diff --git a/plugins/webstats/install.sh b/plugins/webstats/install.sh index b15db97b8..b965b92bc 100755 --- a/plugins/webstats/install.sh +++ b/plugins/webstats/install.sh @@ -28,8 +28,6 @@ Install_App() { echo '正在安装脚本文件...' > $install_tmp mkdir -p $serverPath/source/webstats - - mkdir -p $serverPath/webstats # 下载源码安装包 @@ -87,19 +85,25 @@ Install_App() fi # https://github.com/P3TERX/GeoLite.mmdb - pip install geoip2 - if [ ! -f $serverPath/webstats/GeoLite2-City.mmdb ];then - wget --no-check-certificate -O $serverPath/webstats/GeoLite2-City.mmdb https://git.io/GeoLite2-City.mmdb + # pip install geoip2 + # if [ ! -f $serverPath/webstats/GeoLite2-City.mmdb ];then + # wget --no-check-certificate -O $serverPath/webstats/GeoLite2-City.mmdb https://github.com/P3TERX/GeoLite.mmdb/releases/download/2022.10.16/GeoLite2-City.mmdb + # fi + + + # 缓存数据 + if [ ! -f $serverPath/source/webstats/GeoLite2-City.mmdb ];then + wget --no-check-certificate -O $serverPath/source/webstats/GeoLite2-City.mmdb https://github.com/P3TERX/GeoLite.mmdb/releases/download/2022.10.16/GeoLite2-City.mmdb fi - # GeoLite2-Country.mmdb + if [ -f $serverPath/source/webstats/GeoLite2-City.mmdb ];then + cp -rf $serverPath/source/webstats/GeoLite2-City.mmdb $serverPath/webstats/GeoLite2-City.mmdb + fi echo "${VERSION}" > $serverPath/webstats/version.pl echo '安装完成' > $install_tmp - if [ "$sys_os" != "Darwin" ];then - cd $rootPath && python3 ${rootPath}/plugins/webstats/index.py start - fi + cd $rootPath && python3 ${rootPath}/plugins/webstats/index.py start } Uninstall_App() diff --git a/plugins/webstats/lua/webstats_common.lua b/plugins/webstats/lua/webstats_common.lua new file mode 100644 index 000000000..20b3bfde9 --- /dev/null +++ b/plugins/webstats/lua/webstats_common.lua @@ -0,0 +1,384 @@ + +local setmetatable = setmetatable +local _M = { _VERSION = '1.0' } +local mt = { __index = _M } + +local json = require "cjson" +local sqlite3 = require "lsqlite3" +local config = require "webstats_config" +local sites = require "webstats_sites" + +local debug_mode = true +local total_key = "log_kv_total" +local cache = ngx.shared.mw_total +local today = ngx.re.gsub(ngx.today(),'-','') +local day = os.date("%d") +local number_day = tonumber(day) +local day_column = "day"..number_day +local flow_column = "flow"..number_day +local spider_column = "spider_flow"..number_day + +function _M.new(self) + local self = { + total_key = total_key, + params = nil, + site_config = nil, + config = nil, + + } + -- self.dbs = {} + return setmetatable(self, mt) +end + + +function _M.getInstance(self) + if rawget(self, "instance") == nil then + rawset(self, "instance", self.new()) + self:cron() + end + assert(self.instance ~= nil) + return self.instance +end + + +function _M.initDB(self, input_sn) + local log_dir = "{$SERVER_APP}/logs" + local path = log_dir .. '/' .. input_sn .. "/logs.db" + db, _ = sqlite3.open(path) + + return db +end + +function _M.getTotalKey(self) + return self.total_key +end + +function _M.setConfData( self, config, site_config ) + self.config = config + self.site_config = site_config +end + +function _M.setParams( self, params ) + self.params = params +end + +function _M.split(self, str, reps) + local arr = {} + string.gsub(str,'[^'..reps..']+',function(w) table.insert(arr,w) end) + return arr +end + +-- 后台任务 +function _M.cron(self) + local timer_every_get_data = function (premature) + local llen, _ = ngx.shared.mw_total:llen(total_key) + -- self:D("cron llen:"..tostring(llen)) + for i=1,llen do + local data, _ = ngx.shared.mw_total:lpop(total_key) + if data then + local info = json.decode(data) + self:store_logs(info) + end + end + -- local capacity_bytes = ngx.shared.mw_total:free_space() + -- self:D("cron free_capacity:"..tostring(capacity_bytes)) + end + + ngx.timer.every(1, timer_every_get_data) +end + + +function _M.store_logs_line(self, db, stmt, input_sn, info) + local logline = info['log_kv'] + + local time = logline["time"] + local id = logline["id"] + local protocol = logline["protocol"] + local client_port = logline["client_port"] + local status_code = logline["status_code"] + local uri = logline["uri"] + local request_uri = logline["request_uri"] + local method = logline["method"] + local body_length = logline["body_length"] + local referer = logline["referer"] + local ip = logline["ip"] + local ip_list = logline["ip_list"] + local request_time = logline["request_time"] + local is_spider = logline["is_spider"] + local domain = logline["domain"] + local server_name = logline["server_name"] + local user_agent = logline["user_agent"] + local request_headers = logline["request_headers"] + local excluded = logline["excluded"] + + local request_stat_fields = nil + local client_stat_fields = nil + local spider_stat_fields = nil + local stat_fields = info['stat_fields'] + if stat_fields == nil then + -- D("Log stat fields is nil.") + -- D("Logdata:"..logvalue) + else + stat_fields = self:split(stat_fields, ";") + request_stat_fields = stat_fields[1] + client_stat_fields = stat_fields[2] + spider_stat_fields = stat_fields[3] + + if "x" == client_stat_fields then + client_stat_fields = nil + end + + if "x" == spider_stat_fields then + spider_stat_fields = nil + end + end + + local time_key = logline["time_key"] + if not excluded then + stmt:bind_names { + time=time, + ip=ip, + domain=domain, + server_name=server_name, + method=method, + status_code=status_code, + uri=request_uri, + body_length=body_length, + referer=referer, + user_agent=user_agent, + protocol=protocol, + request_time=request_time, + is_spider=is_spider, + request_headers=request_headers, + ip_list=ip_list, + client_port=client_port, + } + + local res, err = stmt:step() + if tostring(res) == "5" then + self:D("step res:"..tostring(res)) + self:D("step err:"..tostring(err)) + self:D("the step database connection is busy, so it will be stored later.") + return false + end + stmt:reset() + + self:update_stat( db, "client_stat", time_key, client_stat_fields) + self:update_stat( db, "spider_stat", time_key, spider_stat_fields) + -- self:D("stat ok") + + -- only count non spider requests + local ok, err = self:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length) + local ok, err = self:statistics_ip(db, ip, body_length) + -- self:D("stat url ip ok") + end + + self:update_stat( db, "request_stat", time_key, request_stat_fields) + return true +end + +function _M.statistics_uri(self, db, uri, uri_md5, body_length) + -- count the number of URI requests and traffic + local open_statistics_uri = config['global']["statistics_uri"] + if not open_statistics_uri then return true end + + local stat_sql = nil + stat_sql = "INSERT INTO uri_stat(uri_md5,uri) SELECT \""..uri_md5.."\",\""..uri.."\" WHERE NOT EXISTS (SELECT uri_md5 FROM uri_stat WHERE uri_md5=\""..uri_md5.."\");" + local res, err = db:exec(stat_sql) + + stat_sql = "UPDATE uri_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE uri_md5=\""..uri_md5.."\"" + local res, err = db:exec(stat_sql) + return true +end + +function _M.statistics_ip(self, db, ip, body_length) + local open_statistics_ip = config['global']["statistics_ip"] + if not open_statistics_ip then return true end + + local stat_sql = nil + stat_sql = "INSERT INTO ip_stat(ip) SELECT \""..ip.."\" WHERE NOT EXISTS (SELECT ip FROM ip_stat WHERE ip=\""..ip.."\");" + local res, err = db:exec(stat_sql) + + stat_sql = "UPDATE ip_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE ip=\""..ip.."\"" + local res, err = db:exec(stat_sql) + return true +end + + +function _M.update_stat(self,db, stat_table, key, columns) + -- 根据指定表名，更新统计数据 + if not columns then return end + local stmt = db:prepare(string.format("INSERT INTO %s(time) SELECT :time WHERE NOT EXISTS(SELECT time FROM %s WHERE time=:time);", stat_table, stat_table)) + stmt:bind_names{time=key} + local res, err = stmt:step() + stmt:finalize() + local update_sql = "UPDATE ".. stat_table .. " SET " .. columns + update_sql = update_sql .. " WHERE time=" .. key + status, errorString = db:exec(update_sql) +end + +function _M.store_logs(self,info) + local input_sn = info["server_name"] + + -- 迁移合并时不执行 + if self:is_migrating(input_sn) == true then + return + end + + if self:is_working(input_sn) then + return true + end + + self:lock_working(input_sn) + local db = self:initDB(input_sn) + + local stmt2 = nil + if db ~= nil then + stmt2 = db:prepare[[INSERT INTO web_logs( + time, ip, domain, server_name, method, status_code, uri, body_length, + referer, user_agent, protocol, request_time, is_spider, request_headers, ip_list, client_port) + VALUES(:time, :ip, :domain, :server_name, :method, :status_code, :uri, + :body_length, :referer, :user_agent, :protocol, :request_time, :is_spider, + :request_headers, :ip_list, :client_port)]] + end + + if db == nil or stmt2 == nil then + if db and db:isopen() then + db:close() + end + return true + end + + db:exec([[PRAGMA synchronous = 0]]) + db:exec([[PRAGMA page_size = 4096]]) + db:exec([[PRAGMA journal_mode = wal]]) + db:exec([[PRAGMA journal_size_limit = 1073741824]]) + + status, errorString = db:exec([[BEGIN TRANSACTION]]) + + local update_day = self:load_update_day(input_sn) + if not update_day or update_day ~= today then + + local update_sql = "UPDATE uri_stat SET "..day_column.."=0,"..flow_column.."=0" + status, errorString = db:exec(update_sql) + + update_sql = "UPDATE ip_stat SET "..day_column.."=0,"..flow_column.."=0" + status, errorString = db:exec(update_sql) + self:write_update_day(input_sn) + end + + self:store_logs_line(db, stmt2, input_sn, info) + + local res, err = stmt2:finalize() + if tostring(res) == "5" then + -- self:D("Finalize res:"..tostring(res)) + -- self:D("Finalize err:"..tostring(err)) + end + + local now_date = os.date("*t") + local save_day = config['global']["save_day"] + local save_date_timestamp = os.time{year=now_date.year, + month=now_date.month, day=now_date.day-save_day, hour=0} + -- delete expire data + db:exec("DELETE FROM web_logs WHERE time<"..tostring(save_date_timestamp)) + + + local res, err = db:execute([[COMMIT]]) + if db and db:isopen() then + db:close() + end + self:unlock_working(input_sn) +end + + +-- debug func +function _M.D(self,msg) + if not debug_mode then return true end + local fp = io.open('{$SERVER_APP}/debug.log', 'ab') + if fp == nil then + return nil + end + local localtime = os.date("%Y-%m-%d %H:%M:%S") + if server_name then + fp:write(tostring(msg) .. "\n") + else + fp:write(localtime..":"..tostring(msg) .. "\n") + end + fp:flush() + fp:close() + return true +end + +function _M.is_migrating(self,input_sn) + local file = io.open("{$SERVER_APP}/migrating", "rb") + if file then return true end + local file = io.open("{$SERVER_APP}/logs/"..input_sn.."/migrating", "rb") + if file then return true end + return false +end + +function _M.is_working(self,input_sn) + local work_status = cache:get(input_sn.."_working") + if work_status ~= nil and work_status == true then + return true + end + return false +end + +function _M.lock_working(self, input_sn) + local working_key = input_sn.."_working" + cache:set(working_key, true, 60) +end + +function _M.unlock_working(self, input_sn) + local working_key = input_sn.."_working" + cache:set(working_key, false) +end + +function _M.write_file(self, filename, body, mode) + local fp = io.open(filename, mode) + if fp == nil then + return nil + end + fp:write(body) + fp:flush() + fp:close() + return true + end + +function _M.read_file_body(self, filename) + local fp = io.open(filename,'rb') + if not fp then + return nil + end + fbody = fp:read("*a") + fp:close() + if fbody == '' then + return nil + end + return fbody +end + +function _M.load_update_day(self, input_sn) + local _file = "{$SERVER_APP}/logs/"..input_sn.."/update_day.log" + return self:read_file_body(_file) +end + +function _M.write_update_day(self, input_sn) + local _file = "{$SERVER_APP}/logs/"..input_sn.."/update_day.log" + return self:write_file(_file, today, "w") +end + +function _M.lpop(self) + local cache = ngx.shared.mw_total + return cache:lpop(total_key) +end + +function _M.rpop(self) + local cache = ngx.shared.mw_total + return cache:rpop(total_key) +end + + + +return _M \ No newline at end of file diff --git a/plugins/webstats/lua/webstats_log.lua b/plugins/webstats/lua/webstats_log.lua index 78732f005..5d83e06e6 100644 --- a/plugins/webstats/lua/webstats_log.lua +++ b/plugins/webstats/lua/webstats_log.lua @@ -1,16 +1,5 @@ log_by_lua_block { - -- python3 ./plugins/webstats/index.py reload && ab -c 10 -n 1000 http://t1.cn/ - -- - -- - - local ver = '0.2.0' - local max_log_id = 99999999999999 - local debug_mode = true - - local unset_server_name = "unset" - - local cpath = "{$SERVER_APP}/lua/" if not package.cpath:find(cpath) then package.cpath = cpath .. "?.so;" .. package.cpath @@ -19,24 +8,14 @@ log_by_lua_block { package.path = cpath .. "?.lua;" .. package.path end - -- debug func - local function D(msg) - if not debug_mode then return true end - local fp = io.open('{$SERVER_APP}/debug.log', 'ab') - if fp == nil then - return nil - end - local localtime = os.date("%Y-%m-%d %H:%M:%S") - if server_name then - fp:write(tostring(msg) .. "\n") - else - fp:write(localtime..":"..tostring(msg) .. "\n") - end - fp:flush() - fp:close() - return true - end + local ver = '0.2.0' + local max_log_id = 99999999999999 + local debug_mode = true + + local unset_server_name = "unset" + local __C = require "webstats_common" + local C = __C:getInstance() -- cache start --- @@ -56,17 +35,23 @@ log_by_lua_block { local value = cache:get(line_kv) return value end + + -- cache end --- -- domain config is import local db = nil local json = require "cjson" local sqlite3 = require "lsqlite3" + local config = require "webstats_config" + local sites = require "webstats_sites" + + C:setConfData(config, sites) local server_name local request_header local method - local config + local auto_config local excluded @@ -78,14 +63,19 @@ log_by_lua_block { local spider_column --- default common var end --- + + local function to_json(msg) + return json.encode(msg) + end + local function init_var() - config = require "config" request_header = ngx.req.get_headers() method = ngx.req.get_method() day = os.date("%d") - today = os.date("%Y%m%d") + -- today = os.date("%Y%m%d") + today = ngx.re.gsub(ngx.today(),'-','') number_day = tonumber(day) day_column = "day"..number_day @@ -204,16 +194,21 @@ log_by_lua_block { end end end - if type(client_ip) == 'table' then client_ip = "" end - if client_ip ~= "unknown" and string.match(client_ip,"^[%w:]+$") then - return client_ip - end - if string.match(client_ip,"%d+%.%d+%.%d+%.%d+") == nil or not is_ipaddr_bylog(client_ip) then - client_ip = ngx.var.remote_addr - if client_ip == nil then - client_ip = "unknown" - end - end + + -- ipv6 + if type(client_ip) == 'table' then client_ip = "" end + if client_ip ~= "unknown" and ngx.re.match(client_ip,"^([a-fA-F0-9]*):") then + return client_ip + end + + -- ipv4 + if not ngx.re.match(client_ip,"\\d+\\.\\d+\\.\\d+\\.\\d+") == nil or not is_ipaddr_bylog(client_ip) then + client_ip = ngx.var.remote_addr + if client_ip == nil then + client_ip = "unknown" + end + end + return client_ip end @@ -292,26 +287,20 @@ log_by_lua_block { end - local function get_server_name(c_name) local my_name = cache:get(c_name) if my_name then return my_name end - - -- D("get_server_name start") - local determined_name = nil - local sites = require "sites" - -- D("get_server_name"..json.encode(sites)) for _,v in ipairs(sites) do if c_name == v["name"] then - cache:set(c_name, v['name'],3600) + cache:set(c_name, v['name'],86400) return v["name"] end for _,d_name in ipairs(v['domains']) do if c_name == d_name then - cache:set(c_name,v['name'],3600) + cache:set(c_name, v['name'], 86400) return v['name'] elseif string.find(d_name, "*") then new_domain = string.gsub(d_name, '*', '.*') @@ -322,12 +311,11 @@ log_by_lua_block { end end - -- D("get_server_name end") if determined_name then - cache:set(c_name, determined_name,3600) + cache:set(c_name, determined_name,86400) return determined_name end - cache:set(c_name, unset_server_name, 3600) + cache:set(c_name, unset_server_name, 86400) return unset_server_name end @@ -543,7 +531,7 @@ log_by_lua_block { local ip_token = input_server_name..'_'..ip if not cache:get(ip_token) then ipc = 1 - cache:set(ip_token,1,get_end_time()) + cache:set(ip_token,1, get_end_time()) end return ipc end @@ -569,7 +557,7 @@ log_by_lua_block { local uv_token = ngx.md5(ip .. request_header['user-agent'] .. today) if not cache:get(uv_token) then uvc = 1 - cache:set(uv_token,1,get_end_time()) + cache:set(uv_token,1, get_end_time()) end end end @@ -762,7 +750,7 @@ log_by_lua_block { local domain = get_domain() local referer = ngx.var.http_referer - kv = { + local kv = { id=new_id, time_key=time_key, time=os.time(), @@ -792,21 +780,19 @@ log_by_lua_block { if not excluded then - if status_code == 500 or (method=="POST" and config["record_post_args"]==true) or (status_code==403 and config["record_get_403_args"]==true) then + if status_code == 500 or (method=="POST" and config["record_post_args"] == true) or (status_code==403 and config["record_get_403_args"]==true) then local data = "" local ok, err = pcall(function() data=get_http_original() end) if ok and not err then kv["request_headers"] = data end - -- D("Get http orgininal ok:"..tostring(ok)) - -- D("Get http orgininal res:"..tostring(data)) end - if ngx.re.find("500,501,502,503,504,505,506,507,509,510,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,421,422,423,424,425,426,449,451", tostring(status_code), "jo") then local field = "status_"..status_code request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1" end + -- D("method:"..method) local lower_method = string.lower(method) if ngx.re.find("get,post,put,patch,delete", lower_method, "ijo") then @@ -834,8 +820,7 @@ log_by_lua_block { spider_stat_fields = request_spider.."="..request_spider.."+"..1 request_stat_fields = request_stat_fields .. ","..field.."="..field.."+"..1 end - -- D("Is spider:"..tostring(is_spider==true)) - -- D("Request spider:".. tostring(request_spider)) + if ipc > 0 then request_stat_fields = request_stat_fields..",ip=ip+1" end @@ -848,9 +833,19 @@ log_by_lua_block { end local stat_fields = request_stat_fields..";"..client_stat_fields..";"..spider_stat_fields - -- D("stat_fields:"..stat_fields) - cache_set(server_name, new_id, "stat_fields", stat_fields) - cache_set(server_name, new_id, "log_kv", json.encode(kv)) + + local data = { + server_name = server_name, + stat_fields = stat_fields, + log_kv = kv, + } + + local push_data = json.encode(data) + local key = C:getTotalKey() + ngx.shared.mw_total:rpush(key, push_data) + + -- cache_set(server_name, new_id, "stat_fields", stat_fields) + -- cache_set(server_name, new_id, "log_kv", json.encode(kv)) end local function store_logs_line(db, stmt, input_server_name, lineno) @@ -970,7 +965,7 @@ log_by_lua_block { lock_working(input_server_name) local log_dir = "{$SERVER_APP}/logs" - local db_path= log_dir .. '/' .. input_server_name .. "/logs.db" + local db_path = log_dir .. '/' .. input_server_name .. "/logs.db" local db, err = sqlite3.open(db_path) -- if tostring(err) ~= 'nil' then @@ -1025,7 +1020,6 @@ log_by_lua_block { end local res, err = stmt2:finalize() - if tostring(res) == "5" then -- D("Finalize res:"..tostring(res)) -- D("Finalize err:"..tostring(err)) @@ -1061,7 +1055,7 @@ log_by_lua_block { load_exclude_ip(server_name) cache_logs() - store_logs(server_name) + -- store_logs(server_name) -- D("------------ debug end -------------") end @@ -1076,7 +1070,7 @@ log_by_lua_block { end ) if not presult then - D("debug error on :"..tostring(err)) + C:D("debug error on :"..tostring(err)) return true end end diff --git a/plugins/webstats/t/bench/bench.sh b/plugins/webstats/t/bench/bench.sh new file mode 100755 index 000000000..7d6c3d0b6 --- /dev/null +++ b/plugins/webstats/t/bench/bench.sh @@ -0,0 +1,24 @@ +#!/bin/bash +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin + +curPath=`pwd` +rootPath=$(dirname "$curPath") +rootPath=$(dirname "$rootPath") +rootPath=$(dirname "$rootPath") +rootPath=$(dirname "$rootPath") +rootPath=$(dirname "$rootPath") + +# echo $rootPath + +resty=$rootPath/openresty/bin/resty + +RUN_CMD=$resty +if [ ! -f $resty ];then + RUN_CMD=/www/server/openresty/bin/resty +fi + + +# test +# $RUN_CMD simple.lua + +$RUN_CMD test_today.lua diff --git a/plugins/webstats/t/bench/simple.lua b/plugins/webstats/t/bench/simple.lua new file mode 100755 index 000000000..7f52523a0 --- /dev/null +++ b/plugins/webstats/t/bench/simple.lua @@ -0,0 +1,18 @@ +local function target() + ngx.re.find("hello, world.", [[\w+\.]], "jo") +end +for i = 1, 100 do + target() +end + +collectgarbage() + +ngx.update_time() +local begin = ngx.now() +local N = 1e7 +for i = 1, N do + target() +end +ngx.update_time() + +ngx.say("elapsed: ", (ngx.now() - begin) / N) \ No newline at end of file diff --git a/plugins/webstats/t/bench/test_today.lua b/plugins/webstats/t/bench/test_today.lua new file mode 100644 index 000000000..433710420 --- /dev/null +++ b/plugins/webstats/t/bench/test_today.lua @@ -0,0 +1,33 @@ + +local function target() + ngx.re.find("hello, world.", [[\w+\.]], "jo") +end +for i = 1, 100 do + target() +end +-- 以上为预热操作 +collectgarbage() + + +ngx.update_time() +local begin = ngx.now() +local N = 1e6 +for i = 1, N do + os.date("%Y%m%d") + -- ngx.say(t) +end +ngx.update_time() + +ngx.say("os.date elapsed: ", (ngx.now() - begin) / N) + + +ngx.update_time() +local begin = ngx.now() +local N = 1e6 +for i = 1, N do + ngx.re.gsub(ngx.today(),'-','') + -- ngx.say(t) +end +ngx.update_time() + +ngx.say("ngx.today() elapsed: ", (ngx.now() - begin) / N) diff --git a/plugins/webstats/t/index.py b/plugins/webstats/t/index.py new file mode 100644 index 000000000..2c67f3865 --- /dev/null +++ b/plugins/webstats/t/index.py @@ -0,0 +1,95 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import json + +import os +import sys +import time +import string +import json +import hashlib +import shlex +import datetime +import subprocess +import re +from random import Random + + +TEST_URL = "http://t1.cn/" +# TEST_URL = "https://www.zzzvps.com/" + + +def httpGet(url, timeout=10): + import urllib.request + + try: + req = urllib.request.urlopen(url, timeout=timeout) + result = req.read().decode('utf-8') + return result + + except Exception as e: + return str(e) + + +def httpPost(url, data, timeout=10): + """ + 发送POST请求 + @url 被请求的URL地址(必需) + @data POST参数，可以是字符串或字典(必需) + @timeout 超时时间默认60秒 + return string + """ + if sys.version_info[0] == 2: + try: + import urllib + import urllib2 + import ssl + ssl._create_default_https_context = ssl._create_unverified_context + data = urllib.urlencode(data) + req = urllib2.Request(url, data) + response = urllib2.urlopen(req, timeout=timeout) + return response.read() + except Exception as ex: + return str(ex) + else: + try: + import urllib.request + import ssl + try: + ssl._create_default_https_context = ssl._create_unverified_context + except: + pass + data = urllib.parse.urlencode(data).encode('utf-8') + req = urllib.request.Request(url, data) + response = urllib.request.urlopen(req, timeout=timeout) + result = response.read() + if type(result) == bytes: + result = result.decode('utf-8') + return result + except Exception as ex: + return str(ex) + + +def test_OK(): + ''' + 目录保存 + ''' + url = TEST_URL + "ok.txt" + print("ok test start") + url_val = httpGet(url, 10) + print(url_val) + print("ok test end") + + +def test_start(): + test_OK() + + +if __name__ == "__main__": + os.system('cd /Users/midoks/Desktop/mwdev/server/mdserver-web/plugins/webstats && sh install.sh uninstall 0.2.2 && sh install.sh install 0.2.2') + os.system('cd /Users/midoks/Desktop/mwdev/server/mdserver-web/ && python3 plugins/openresty/index.py stop && python3 plugins/openresty/index.py start') + test_start() diff --git a/plugins/webstats/t/test.sh b/plugins/webstats/t/test.sh new file mode 100755 index 000000000..bf823a160 --- /dev/null +++ b/plugins/webstats/t/test.sh @@ -0,0 +1,6 @@ +#!/bin/bash +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin +export PATH + +python3 index.py +