up

plugins/webstats/index.py

@@ -1051,6 +1051,8 @@ def getUriStatList():
         ftime = time.localtime(time.time())
         day = ftime.tm_mday
 
+        # print(day)
+
         field_day = "day" + str(day)
         field_flow = "flow" + str(day)
         # print(field_day, field_flow)

plugins/webstats/lua/webstats_common.lua

@@ -412,9 +412,11 @@ function _M.store_logs_line(self, db, stmt, input_sn, info)
         end
         stmt:reset()
 
-        self:update_stat( db, "client_stat", time_key, client_stat_fields)
-        self:update_stat( db, "spider_stat", time_key, spider_stat_fields)
-        -- self:D("stat ok")
+        local res ,err = self:update_stat( db, "client_stat", time_key, client_stat_fields)
+        -- self:D("step res:"..tostring(res) ..",step err:"..tostring(err))
+        local res ,err = self:update_stat( db, "spider_stat", time_key, spider_stat_fields)
+        -- self:D("step res:"..tostring(res) ..",step err:"..tostring(err))
+        -- self:D("stat ok"..)
 
         -- only count non spider requests
         local ok, err = self:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length)
@@ -478,7 +480,7 @@ function _M.statistics_uri(self, db, uri, uri_md5, body_length)
     stat_sql = "INSERT INTO uri_stat(uri_md5,uri) SELECT \""..uri_md5.."\",\""..uri.."\" WHERE NOT EXISTS (SELECT uri_md5 FROM uri_stat WHERE uri_md5=\""..uri_md5.."\");"
     local res, err = db:exec(stat_sql)
     
-    stat_sql = "UPDATE uri_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE uri_md5=\""..uri_md5.."\""
+    stat_sql = "UPDATE uri_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE uri_md5=\""..uri_md5.."\";"
     local res, err = db:exec(stat_sql)
     return true
 end
@@ -590,6 +592,20 @@ function _M.write_update_day(self, input_sn)
     return self:write_file(_file, today, "w")
 end
 
+function _M.clean_stats(self, db, input_sn)
+    -- 清空 uri,ip 汇总的信息[昨日]
+    local update_day = self:load_update_day(input_sn)
+    if not update_day or update_day ~= today then
+
+        local update_sql = "UPDATE uri_stat SET "..day_column.."=0,"..flow_column.."=0"
+        db:exec(update_sql)
+
+        update_sql = "UPDATE ip_stat SET "..day_column.."=0,"..flow_column.."=0"
+        db:exec(update_sql)
+        self:write_update_day(input_sn)
+    end
+end
+
 function _M.lpop(self)
     local cache = ngx.shared.mw_total
     return cache:lpop(total_key)

plugins/webstats/lua/webstats_log.lua

@@ -14,11 +14,24 @@ log_by_lua_block {
 	local __C = require "webstats_common"
 	local C = __C:getInstance()
 
+	-- local redis = require "resty.redis"
+ --    local red = redis:new()
+
+ --    local ok, err = red:connect("127.0.0.1", 6379)
+ --    if not ok then
+ --        ngx.say("failed to connect: ", err)
+ --        return
+ --    end
+
+ --    red:auth("admin")
+
 
 	-- cache start ---
 	local cache = ngx.shared.mw_total
 	local function cache_set(server_name, id ,key, val)
 		local line_kv = "log_kv_"..server_name..'_'..id.."_"..key
+		-- cache:set(line_kv, val)
+
 		cache:set(line_kv, val)
 	end
 
@@ -49,10 +62,12 @@ log_by_lua_block {
 
 	local auto_config = C:setInputSn(server_name)
 
-	local request_header = ngx.req.get_method()
-	local method = ngx.req.get_headers()
-	local excluded
+	local request_header = ngx.req.get_headers()
+	local method = ngx.req.get_method()
+	local excluded = false
+
 
+	local today = ngx.re.gsub(ngx.today(),'-','')
 	local day = os.date("%d")
 	local number_day = tonumber(day)
 	local day_column = "day"..number_day
@@ -223,7 +238,7 @@ log_by_lua_block {
 		local kv = {
 			id=new_id,
 			time_key=time_key,
-			time=os.time(),
+			time=ngx.time(),
 			ip=ip,
 			domain=domain,
 			server_name=server_name,
@@ -317,7 +332,8 @@ log_by_lua_block {
 		-- local key = C:getTotalKey()
 		-- ngx.shared.mw_total:rpush(key, push_data)
 
-		-- C:D("ddd")
+		-- C:D("stat_fields:"..stat_fields)
+		-- C:D("log_kv:"..json.encode(kv))
 		cache_set(server_name, new_id, "stat_fields", stat_fields)
 		cache_set(server_name, new_id, "log_kv", json.encode(kv))
 
@@ -406,13 +422,14 @@ log_by_lua_block {
 			end
 			stmt:reset()
 			-- D("store_logs_line ok")
+
 			C:update_stat( db, "client_stat", time_key, client_stat_fields)
 			C:update_stat( db, "spider_stat", time_key, spider_stat_fields)
-			-- D("stat ok")
+			-- C:D("stat ok")
 
 			-- only count non spider requests
-			local ok, err = pcall(function() C:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length) end)
-			local ok, err = pcall(function() C:statistics_ip(db, ip, body_length) end)
+			local ok, err = C:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length)
+			local ok, err = C:statistics_ip(db, ip, body_length)
 		end
 
 		C:update_stat( db, "request_stat", time_key, request_stat_fields)
@@ -454,6 +471,11 @@ log_by_lua_block {
 			return true
 		end 
 
+		db:exec([[PRAGMA synchronous = 0]])
+		db:exec([[PRAGMA page_size = 4096]])
+		db:exec([[PRAGMA journal_mode = wal]])
+		db:exec([[PRAGMA journal_size_limit = 1073741824]])
+
 		local stmt2 = nil
 		if db ~= nil then
 			stmt2 = db:prepare[[INSERT INTO web_logs(
@@ -472,23 +494,10 @@ log_by_lua_block {
 			end
 			return true
 		end
-		db:exec([[PRAGMA synchronous = 0]])
-		db:exec([[PRAGMA page_size = 4096]])
-		db:exec([[PRAGMA journal_mode = wal]])
-		db:exec([[PRAGMA journal_size_limit = 1073741824]])
-
+		
 		status, errorString = db:exec([[BEGIN TRANSACTION]])
 
-		update_day = C:load_update_day(input_sn)
-		if not update_day or update_day ~= today then
-
-			local update_sql = "UPDATE uri_stat SET "..day_column.."=0,"..flow_column.."=0"
-			status, errorString = db:exec(update_sql)
-
-			update_sql = "UPDATE ip_stat SET "..day_column.."=0,"..flow_column.."=0"
-			status, errorString = db:exec(update_sql)
-			C:write_update_day(input_sn)
-		end
+		C:clean_stats(db, input_sn)
 
 		if store_end >= store_start then
 			for i=store_start, store_end, 1 do

plugins/webstats/t/index.py

@@ -74,13 +74,29 @@ def httpPost(url, data, timeout=10):
             return str(ex)
 
 
+def httpGet__UA(url, ua, timeout=10):
+    import urllib.request
+    headers = {'user-agent': ua}
+    try:
+        req = urllib.request.Request(url, headers=headers)
+        response = urllib.request.urlopen(req)
+        result = response.read().decode('utf-8')
+        return result
+
+    except Exception as e:
+        return str(e)
+
+
 def test_OK():
     '''
     目录保存
     '''
     url = TEST_URL + "ok.txt"
     print("ok test start")
-    url_val = httpGet(url, 10)
+    # Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36
+    # (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36
+    url_val = httpGet__UA(
+        url, "Mozilla / 5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit / 537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36")
     print(url_val)
     print("ok test end")