Update webstats_log.lua

plugins/webstats/lua/webstats_log.lua

@@ -1,6 +1,6 @@
 log_by_lua_block {
 
-	-- python3 ./plugins/webstats/index.py reload && echo "" > /Users/midoks/Desktop/mwdev/server/webstats/debug.log && wget http://t1.cn/
+	-- python3 ./plugins/webstats/index.py reload && ab -c 10 -n 1000 http://t1.cn/
 	-- 
     -- 
 
@@ -196,7 +196,6 @@ log_by_lua_block {
 			end
 
 			if "table" == type(data) then
-				D("get_http_original:"..data)
 				headers = table.concat(headers, data)
 			end
 		end
@@ -229,11 +228,11 @@ log_by_lua_block {
 		local my_name = cache:get(c_name)
 		if my_name then return my_name end
 
-		D("get_server_name start")
+		-- D("get_server_name start")
 
 		local determined_name = nil
 		local sites = require "sites"
-		D("get_server_name"..json.encode(sites))
+		-- D("get_server_name"..json.encode(sites))
 		for _,v in ipairs(sites)
 		do
 			if c_name == v["name"] then
@@ -247,18 +246,14 @@ log_by_lua_block {
 					return v['name']
 	            elseif string.find(d_name, "*") then
 					new_domain = string.gsub(d_name, '*', '.*')
-					D("ngx server name:"..ngx.var.server_name.."/new_domain:"..new_domain)
 	            	if string.find(c_name, new_domain) then
-	                	-- cache:set(c_name, v['name'],3600)
-	                	-- return v['name']
-						-- debug("find deter name:"..v['name'])
 						determined_name = v['name']
 	            	end
 				end
 			end
 		end
 
-		D("get_server_name end")
+		-- D("get_server_name end")
 		if determined_name then
 	        cache:set(c_name, determined_name,3600)
 			return determined_name
@@ -276,7 +271,6 @@ log_by_lua_block {
 		local res, err = stmt:step()
 		stmt:finalize()
 		local update_sql = "UPDATE ".. stat_table .. " SET " .. columns
-		D("update_stat::key::"..tostring(key))
 		update_sql = update_sql .. " WHERE time=" .. key
 		status, errorString = db:exec(update_sql)
 	end
@@ -358,8 +352,8 @@ log_by_lua_block {
 			else
 				cls_pc = string.lower(pc_res[0])
 			end
-			D("UA:"..ua)
-			D("PC cls:"..tostring(cls_pc))
+			-- D("UA:"..ua)
+			-- D("PC cls:"..tostring(cls_pc))
 			if cls_pc then
 				client_stat_fields = client_stat_fields..","..get_update_field(clients_map[cls_pc], 1)
 			else
@@ -525,8 +519,8 @@ log_by_lua_block {
 			for i, _ip in pairs(global_exclude_ip)
 			do 
 				-- global
+				-- D("set global exclude ip: ".._ip)
 				if not cache:get("global_exclude_ip_".._ip) then
-					D("set global exclude ip: ".._ip)
 					cache:set("global_exclude_ip_".._ip, true)
 				end
 			end
@@ -621,15 +615,15 @@ log_by_lua_block {
 				break
 			end
 		end
-		D("server[" ..input_server_name.."]check exclude ip : "..tostring(check_server_exclude_ip))
+		-- D("server[" ..input_server_name.."]check exclude ip : "..tostring(check_server_exclude_ip))
 		if check_server_exclude_ip then
 			if cache:get(input_server_name .. "_exclude_ip_"..ip) then 
-				D("-Exclude server ip:"..ip)
+				-- D("-Exclude server ip:"..ip)
 				return true
 			end
 		else
 			if cache:get("global_exclude_ip_"..ip) then 
-				D("*Excluded global ip:"..ip)
+				-- D("*Excluded global ip:"..ip)
 				return true
 			end
 		end
@@ -647,8 +641,6 @@ log_by_lua_block {
 		local ip = get_client_ip_bylog()
 		excluded = filter_status() or exclude_extension() or exclude_url() or exclude_ip(server_name, ip)
 
-		D("new_id:"..new_id)
-
 		local ip_list = request_header["x-forwarded-for"]
 		if ip and not ip_list then
 			ip_list = ip
@@ -704,8 +696,8 @@ log_by_lua_block {
 				if ok and not err then
 					kv["request_headers"] = data
 				end
-				D("Get http orgininal ok:"..tostring(ok))
-				D("Get http orgininal res:"..tostring(data))
+				-- D("Get http orgininal ok:"..tostring(ok))
+				-- D("Get http orgininal res:"..tostring(data))
 			end
 
 
@@ -713,8 +705,7 @@ log_by_lua_block {
 				local field = "status_"..status_code
 				request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1"
 			end
-			
-			D("method:"..method)
+			-- D("method:"..method)
 			local lower_method = string.lower(method)
 			if ngx.re.find("get,post,put,patch,delete", lower_method, "ijo") then
 				local field = "http_"..lower_method
@@ -728,8 +719,7 @@ log_by_lua_block {
 
      		is_spider, request_spider, spider_index = match_spider(ip)
      		if not is_spider then
-     			client_stat_fields = match_client()	
-				D("Client stat fields:"..tostring(client_stat_fields))
+     			client_stat_fields = match_client()
 				if not client_stat_fields or #client_stat_fields == 0 then
 					client_stat_fields = request_stat_fields..",other=other+1"
 				end
@@ -742,8 +732,8 @@ log_by_lua_block {
 				spider_stat_fields = request_spider.."="..request_spider.."+"..1
 				request_stat_fields = request_stat_fields .. ","..field.."="..field.."+"..1
 			end
-     		D("Is spider:"..tostring(is_spider==true))
-			D("Request spider:".. tostring(request_spider))
+     		-- D("Is spider:"..tostring(is_spider==true))
+			-- D("Request spider:".. tostring(request_spider))
 			if ipc > 0 then 
 				request_stat_fields = request_stat_fields..",ip=ip+1"
 			end
@@ -756,7 +746,7 @@ log_by_lua_block {
 		end
 
 		local stat_fields = request_stat_fields..";"..client_stat_fields..";"..spider_stat_fields
-		D("stat_fields:"..stat_fields)
+		-- D("stat_fields:"..stat_fields)
 		cache_set(server_name, new_id, "stat_fields", stat_fields)
 		cache_set(server_name, new_id, "log_kv", json.encode(kv))
  	end
@@ -790,8 +780,8 @@ log_by_lua_block {
 		local spider_stat_fields = nil
 		local stat_fields = cache_get(input_server_name, id, "stat_fields")
 		if stat_fields == nil then
-			D("Log stat fields is nil.")
-			D("Logdata:"..logvalue)
+			-- D("Log stat fields is nil.")
+			-- D("Logdata:"..logvalue)
 		else
 			stat_fields = split_bylog(stat_fields, ";")
 			request_stat_fields = stat_fields[1]
@@ -830,21 +820,18 @@ log_by_lua_block {
 
 			local res, err = stmt:step()
 			if tostring(res) == "5" then
-				D("Step res:"..tostring(res))
-				D("Step err:"..tostring(err))
-				D("Step数据库连接繁忙，稍候存储。")
+				-- D("Step res:"..tostring(res))
+				-- D("Step err:"..tostring(err))
+				-- D("Step数据库连接繁忙，稍候存储。")
 				return false
 			end
 			stmt:reset()
-			D("store_logs_line ok")
-			D("client_stat:"..tostring( time_key ).. tostring( client_stat_fields ))
-			D("spider_stat:"..tostring( time_key ).. tostring( spider_stat_fields ))
+			-- D("store_logs_line ok")
 			update_stat(store_server_name, db, "client_stat", time_key, client_stat_fields)
 			update_stat(store_server_name, db, "spider_stat", time_key, spider_stat_fields)
-			D("stat ok")
+			-- D("stat ok")
 		end
 
-		
 		update_stat(input_server_name, db, "request_stat", time_key, request_stat_fields)
 		return true
  	end
@@ -864,12 +851,10 @@ log_by_lua_block {
 		end
 
 		local worker_id = ngx.worker.id()
-		D("worker_id:"..worker_id)
-
 		if is_working(input_server_name) then
-			D("其他worker正在存储中，稍候存储。")
+			-- D("其他worker正在存储中，稍候存储。")
 			-- cache:delete(flush_data_key)
-			return
+			return true
 		end
 		lock_working(input_server_name)
 
@@ -893,7 +878,7 @@ log_by_lua_block {
 		end
 
 		if db == nil or stmt2 == nil then
-			D("web data db error")
+			-- D("web data db error")
 			-- cache:set(storing_key, false)
 			if db and db:isopen() then
 				db:close()
@@ -904,7 +889,7 @@ log_by_lua_block {
 		status, errorString = db:exec([[BEGIN TRANSACTION]])
 		if store_end >= store_start then
 			for i=store_start, store_end, 1 do
-				D("store_start:"..store_start..":store_end:".. store_end)
+				-- D("store_start:"..store_start..":store_end:".. store_end)
 				if store_logs_line(db, stmt2, input_server_name, i) then
 					cache_clear(input_server_name, i, "log_kv")
 					cache_clear(input_server_name, i, "stat_fields")
@@ -915,11 +900,17 @@ log_by_lua_block {
 		local res, err = stmt2:finalize()
 
 		if tostring(res) == "5" then
-			D("Finalize res:"..tostring(res))
-			D("Finalize err:"..tostring(err))
-			return true
+			-- D("Finalize res:"..tostring(res))
+			-- D("Finalize err:"..tostring(err))
 		end
 
+		local now_date = os.date("*t")
+		local save_day = config['global']["save_day"]
+		local save_date_timestamp = os.time{year=now_date.year,
+			month=now_date.month, day=now_date.day-save_day, hour=0}
+		-- delete expire data
+		db:exec("DELETE FROM site_logs WHERE time<"..tostring(save_date_timestamp))
+
 		local res, err = db:execute([[COMMIT]])
 		if db and db:isopen() then
 			db:close()
@@ -930,13 +921,13 @@ log_by_lua_block {
 	end
 
 	local function run_app()
-		D("------------ debug start ------------")
+		-- D("------------ debug start ------------")
 		init_var()
 
 		local c_name = ngx.var.server_name
 		server_name = string.gsub(get_server_name(c_name),'_','.')
 
-		D("server_name:"..server_name)
+		-- D("server_name:"..server_name)
 
 		load_global_exclude_ip()
 		load_exclude_ip(server_name)
@@ -944,7 +935,7 @@ log_by_lua_block {
 		cache_logs()
 		store_logs(server_name)
 
-		D("------------ debug end -------------")
+		-- D("------------ debug end -------------")
 	end