up

plugins/op_waf/conf/init.sql

@@ -0,0 +1,15 @@
+PRAGMA synchronous = 0;
+PRAGMA page_size = 4096;
+PRAGMA journal_mode = wal;
+PRAGMA journal_size_limit = 1073741824;
+
+CREATE TABLE IF NOT EXISTS `waf_history` (
+    `time` INTEGER,
+    `ip` TEXT,
+    `domain` TEXT,
+    `server_name` TEXT,
+    `method` TEXT,
+    `status_code` INTEGER,
+    `uri` TEXT,
+    `reason` TEXT
+);

plugins/op_waf/index.py

@@ -72,6 +72,30 @@ def getConf():
     return path
 
 
+def pSqliteDb(dbname='waf_history'):
+    name = "waf"
+    db_dir = getServerDir() + '/logs/'
+
+    if not os.path.exists(db_dir):
+        mw.execShell('mkdir -p ' + db_dir)
+
+    file = db_dir + name + '.db'
+    if not os.path.exists(file):
+        conn = mw.M(dbname).dbPos(db_dir, name)
+        sql = mw.readFile(getPluginDir() + '/conf/init.sql')
+        sql_list = sql.split(';')
+        for index in range(len(sql_list)):
+            conn.execute(sql_list[index])
+    else:
+        conn = mw.M(dbname).dbPos(db_dir, name)
+
+    conn.execute("PRAGMA synchronous = 0")
+    conn.execute("PRAGMA page_size = 4096")
+    conn.execute("PRAGMA journal_mode = wal")
+    conn.execute("PRAGMA journal_size_limit = 1073741824")
+    return conn
+
+
 def initDomainInfo():
     data = []
     path_domains = getJsonPath('domains')
@@ -317,6 +341,8 @@ def initDreplace():
     initTotalInfo()
     autoMakeLuaConf()
 
+    pSqliteDb()
+
     if not mw.isAppleSystem():
         mw.execShell("chown -R www:www " + path)
 

plugins/op_waf/t/test.sh

@@ -11,66 +11,3 @@ export PATH
 # ab -c 1000 -n 1000000 http://dev156.cachecha.com/
 
 python3 index.py
-
-
-# 安装 火焰图
-# sudo apt install elfutils
-# sudo apt-get build-dep systemtap
-# systemtap
-# stap -e 'probe kernel.function("sys_open") {log("hello world") exit()}'
-# stap -v -e 'probe vfs.read {printf("read performed\n"); exit()}'
-
-
-# yum -y kernel-devel kernel-headers gcc elfutils
-# stap -ve 'probe begin { log("hello systemtap!") exit() }'
-# stap -e 'probe vfs.add_to_page_cache {printf("dev=%d, devname=%s, ino=%d, index=%d, nrpages=%d\n", dev, devname, ino, index, nrpages )}'
-# git clone https://github.com/openresty/openresty-systemtap-toolkit
-# http://openresty.org/en/build-systemtap.html
-
-# ./configure --prefix=/opt/stap --disable-docs --disable-publican --disable-refdocs CFLAGS="-g -O2"
-
-# apt-get install systemtap linux-image-`uname -r`-dbg linux-headers-`uname -r`
-# /usr/share/doc/systemtap/README.Debian
-# ./ngx-active-reqs -p 383774
-
-
-# wget -O kernel-debuginfo-$(uname -r).rpm http://debuginfo.centos.org/8/x86_64/kernel-debuginfo-$(uname -r).rpm
-
-# wget -O kernel-debuginfo-4.18.0-348.el8.x86_64.rpm http://debuginfo.centos.org/8/x86_64/Packages/kernel-debuginfo-4.18.0-348.el8.x86_64.rpm
-# wget -O kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm http://debuginfo.centos.org/8/x86_64/Packages/kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm
-
-# rpm -ivh kernel-debuginfo-4.18.0-348.el8.x86_64.rpm
-# rpm -ivh kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm
-# uname -r 
-
-# yum install kernel-devel-4.18.0-358.el8.x86_64
-# yum install kernel-debuginfo-4.18.0-358.el8.x86_64
-
-# yum search kernel-debuginfo
-# kernel-devel
-
-# rpm -ivh kernel-debuginfo-4.18.0-358.el8.x86_64
-# yum install systemtap -y
-# yum install perf -y
-
-
-# perf record -F 99  -p 4452 -g -o test.data -- sleep 100
-
-
-# perf record -F 99  -p 153145 -g -o test.data -- sleep 100
-
-# perf script -i test.data &> perf.unfold
-# ./FlameGraph/stackcollapse-perf.pl perf.unfold &> perf.folded
-# ./FlameGraph/flamegraph.pl perf.folded > perf.svg
-
-#
-# git clone https://github.com/brendangregg/FlameGraph.git
-
-# git clone https://github.com/openresty/openresty-systemtap-toolkit
-# ps -ef|grep openresty | grep -v grep | awk '{print $2}'
-# ./ngx-active-reqs -p 153145
-
-
-
-
-

plugins/op_waf/waf/lua/common.lua

@@ -25,21 +25,44 @@ function _M.new(self)
         server_name = '',
         global_tatal = nil,
         params = nil,
+        db = nil,
     }
     return setmetatable(self, mt)
 end
 
 
-
-
 function _M.getInstance(self)
     if rawget(self, "instance") == nil then
         rawset(self, "instance", self.new())
+        self.initDB()
     end
     assert(self.instance ~= nil)
     return self.instance
 end
 
+function _M.initDB(self)
+    if self.db then
+        return self.db
+    end
+
+    local path = log_dir .. "/waf.db"
+    db, err = sqlite3.open(path)
+
+    if err then
+        self:D("initDB err:"..tostring(err))
+        return nil
+    end
+
+    db:exec([[PRAGMA synchronous = 0]])
+    db:exec([[PRAGMA cache_size = 8000]])
+    db:exec([[PRAGMA page_size = 32768]])
+    db:exec([[PRAGMA journal_mode = wal]])
+    db:exec([[PRAGMA journal_size_limit = 1073741824]])
+
+    self.db = db
+    return db
+end
+
 function _M.setDebug(self, mode)
     debug_mode = mode
 end

plugins/webstats/lua/webstats_log.lua

@@ -262,7 +262,6 @@ log_by_lua_block {
 				request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1"
 			end
 
-			-- D("method:"..method)
 			local lower_method = string.lower(method)
 			if ngx.re.find("get,post,put,patch,delete", lower_method, "ijo") then
 				local field = "http_"..lower_method
@@ -304,10 +303,8 @@ log_by_lua_block {
 
 		local stat_fields = request_stat_fields..";"..client_stat_fields..";"..spider_stat_fields
 
-
+		cache_set(server_name, new_id, "stat_fields", stat_fields)
-		C:D("stat_fields:"..stat_fields)
+		cache_set(server_name, new_id, "log_kv", json.encode(kv))
-		-- cache_set(server_name, new_id, "stat_fields", stat_fields)
-		-- cache_set(server_name, new_id, "log_kv", json.encode(kv))
  	end
 
 	local function cache_logs(input_sn)
@@ -370,10 +367,7 @@ log_by_lua_block {
 		}
 
 		-- C:D(json.encode(kv))
-		local request_stat_fields = {
+		local request_stat_fields = {req=1,length=body_length}
-			req=1,
-			length=body_length,
-		}
 
 		local spider_stat_fields = {}
 		local client_stat_fields = {}
@@ -442,7 +436,7 @@ log_by_lua_block {
 		}
 
 		local push_data = json.encode(data)
-		-- C:D("push_data:"..push_data)
+
 		local key = C:getTotalKey()
 		ngx.shared.mw_total:rpush(key, push_data)		
  	end