From f5d63f7a4f99f25baeb9bb946b6d3ffe3d190f33 Mon Sep 17 00:00:00 2001 From: Mr Chen Date: Fri, 28 Dec 2018 18:11:01 +0800 Subject: [PATCH] update --- plugins/gae/ico.png | Bin 2160 -> 1034 bytes plugins/memcached/ico.png | Bin 801 -> 653 bytes plugins/simdht/workers/index_worker.py | 21 +++++---- plugins/simdht/workers/simdht_worker.py | 58 ++++++++++++++---------- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/plugins/gae/ico.png b/plugins/gae/ico.png index b1aa6620b3368ca0362ceb49a52146767874bcc8..c49edb4a15ea9d5b2c7db95f899cba4ea37f0908 100644 GIT binary patch delta 1023 zcmVVp?LV9fqQ525O3q+r{2ax-kAdi_s`;&e}UE=y5Y)zNFbMg z?HykAuaMhMcz>P;ol#It{1QI|N1%!~MV+IQ=k*7EwGF<_=jCZpt7+hm{4dbSSH}ro zwvcz|?T^aG_UOQIyk1M_C|ekx_h0L+)x8H~=?J)?Wi*l@u?OZ_4c{D0`E3eHiAeFT zja~n9rK-JjT4Z+XRJMyi!1FpA*FwUc#3d=r_*$4FIG18B~t=RCV>Zk5l}hFF*hDpp8>2HbJq3JA;_ZLF5=C>WI@MgnED+G=3CMAtJVQGY98f!ujx`}x7kfZUwvH#X)s{$5 z0smy0`-(K+8@ak=qBY(!QI7M`Nk#@Ce5WG#hy*n@SH359ztSo^LAx@~*92cs@knc2 z;K0m`1kP+iydLYiq4i}uW;DTdBxq6QrP@VxxPQ7>6E2qH&61!iC)pMjEx39=ioiak z&(TGX6~P%za2!|bDTX1?M7{T zH0H;yBwc5N4lv|QZmR2=VL=Pm2cu#Zs^!ah*s?51k0+lU*1Ez_X3ja&mb#2$gPD$z zuH(K))`KJQeQG4Z9b4Xg=9}|ND9q!}YNEq&e=4>bp}cI; tAXnJn2xR)^`{o8R!9o7F|LtFDKLB^CZa`lNnFjy>002ovPDHLkV1jQv0IUE2 delta 2158 zcmV-!2$A=S2=EY)8Gi-<00374`G)`i2scSYK~!i%?O6?M99I#Z**)8BeGLw+LkhG> z5lC$aqTw&3(8viXQ2r7~o61cb-^L=SLP1V$x27#rTNQ}y_v}QZLRH$}bsQ_SZdw`& zO+|HQnb1<23xzfkG)?8`(W8qMMLBO-)?a7J&WsVzH0=if@DMvrM2jrTdUYm9%p`$h05H!Z z@Q7tucV9FF>gwtiI*#*N&S4_zD=jUZvvcRp;prwZBN@;%?EwII(C@6{IJdXAx9_`H z2ynfBJmBp9wtu#^mG622b#-;~9LM4MA0B*raODJs-vNLw$8lO(T7O#JD8$L}bf!e0p`qcUj^q5u ztNEgQAtC~R{X}%%RESKPfUfH+5b+rRxS}Y}$0`pJ(Ml=h3zNZ_6am99?jWMQh^St` z=$in*<#CLNSm!!Kyb%ESGe&reMZ{_$L|3sSiY8zf#@C4GmwB9j2N546qD~>iseFzY zhB23jYJU;&{@_~iNURbNi3}FO`&(tlYYFEAKQDimvN( z5b;+4@R@*e#incv~PC zZ%8Q@au;3*iGm44qtS0EiV|$b{#IUI{t44G2Y(AO7Su6K^NLg|l?VpKah%mHEiF$K zf>|&DUDppG;wN*UM3lB|d*QZi+x`(Wv|+=B4-pZ!6rU%eEC9T0+xD{^9UcFys;Zi+ zD9Y~s{{B^oL}D-+joyHWe?i0&AI@PR#E7jap92jI4c9r2b36dlD1^AzKeln>#*%b8 zeSZ%kK7fd$)(HSG;g)1FxdstG3;@4~#bVVwU)Obkh`Jw3Rn_Yo8ym+og+VeG%s_Q@ z_0@a#?mfq~sO!3ch|Rv}+p4Nw)zs9~?+1)TA`c_tcgKs^z`y{UKY!j;;Tq9_Nwu`Cho27p}Wc`G8W2Y?VCz^6`~f=njE1YVEF;~(>E^Z>vu zKFk9`2zP;Nnl=CchluD&+qR$V=;%0|)5G=a*MC4&)ip%48USwgT3>#CxUH?tt$#yZ z*Iz?K-WhU{dq_&jx2_S-^9cMH z0PgeIcwI{AlG8Me55?Re=m5ZrM6^#)lqWC}iS#1koM3v0=zwKeZUhX&;PkoKU_r$1 zG&eWz3^@3AMEnSkWipv-J3Bl7<$r&^cJ0~^mX?+#d2D!i7*3oRxh~9)#bU<|!{E|; z*hk=PDdkljfdc^WrJ#=h(2JU;<=t=oM)prE~T9B5$Hig|N0n#<`T;LkcbkNWqpb@4a0Z|06yz;XtxleHbCHK0N}p+ zA)-G5z@3(5x!YhQ5@AOkM}I`Ob;#{RMD*u)JbsfGqz*(}=fnJi5Ml}cuW8y509cSG z0S5~@$aGzQ3=!*c07P_JO8GuNARk_iD1CLD{! z9`S<2I}6|9a`JN_#E8v2&bvc}68JhI?hk-06+(=Z$&w{Y-dj;oaeuh7vT~lP=I^qz z*({trdlu5^G-v7Mcs%ZARoC?@MC2xMM2P4cQc7<4#+8Byn5J2pN~QV`@p4}lo|jVQ z&IVk*e0c%@7V!&+*|TRud3iY~iW_t{+Jl3G(AU@JE?{miY};PY)6?T-$S@4P%zVLz z{hx3+Tw$7KW?Be%Mt^<^0Gk4Uw+kWM25;H2WqfAvZgFi+Clm^?v$=A~4Zr0$&RyNz z-G@BfJGre0;O-DYd~YJ>3oap6tgo-Xrlh2V7p1?)$q>G0iHcc~>N~NAh z#8G{a_nh19yPKPv-O{*_=(@fb5&6V*x9C8Yy2Ih{t)^*m3to^EMSu}?T`vcKI3n`P zt1&`Ed>wpDRe#k(wrw9(Rh7$w+wsLjq#>ev*yVBlm2fy*RqW-cXaa25yGcEVh)W9! z(*#OH^g=isURkX33_B?TJdrOivu4eD6cI0>}hRn z{a-seMVWF2^4aHw9twpj9mko=J&~d)r?c7Y87bwzr|9pP=`%uLx}6*k?-GIkXSDO# kx|D%S8JJ=OF8UJsFWnB3(>poPe*gdg07*qoM6N<$f{7vx?f?J) diff --git a/plugins/memcached/ico.png b/plugins/memcached/ico.png index 2183430fdead232dc1dd81e5e151b1783e2eb531..704baa7ce6c7d565820d45f5ac5f2ebda7f52045 100755 GIT binary patch delta 639 zcmV-_0)YLY28{)f8Gi-<006j$d2s*$0#r#vK~#7F?U~W7oG=hY-2yGp0xi%2Ew}|r z$gWtF?`?q=wuSwFvW#1B3&dCQCTLCnP=_*r!8Gzot@{+9x?3#7`vgEE1Gw`5d_C}a&+1+ffXK&=ma}zmSVLC znYm2+F8~?>&>c&qS<{wXwv8evJ+=22w8Pmz=Bmr?K*H*x?@Rp>0Ls?D=-QOO-|1X+ z!ljn3fw@ckh?=W|_`IfL(N0Sju}+TRz5YAxyKLb$`hPGRmf77zR(r60X)`x%>_EHM zX43{x8o;H;d{vn`@OZwl1FpE4L(y*Bd#8=neA1GTyw@K%E`SRl ZuV2~NQK$!8;-MT+O zL75WQh?3y^w370~qErUQl>DSr1-Hzi)bjkI>|zDyV6zu{HuW$tFi!DwaSXBOy*v4K z*dYS}*X3eb;iqL)ZfI$6Nhor4Uj6@Hzc_dDfm5GCKTj!6E3-D9%k*dYYw5Ap=%D@x)X74=;RThPYJB&sRy9EkPuYVu*&yeZG>eo+{EN&IJY=f;aUGT#|g|{;6S` zvFGm@_E)p5%u9&o2J@uIZ~(4jpybrD1jY|KaEC YZ#b4~Uc4L`2#iMtPgg&ebxsLQ086=EvH$=8 diff --git a/plugins/simdht/workers/index_worker.py b/plugins/simdht/workers/index_worker.py index a4eb661ee..c136dfc48 100755 --- a/plugins/simdht/workers/index_worker.py +++ b/plugins/simdht/workers/index_worker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -#coding: utf8 +# coding: utf8 """ 从MySQL数据库中读取未索引的资源，更新到Sphinx的实时索引中。 xiaoxia@xiaoxia.org @@ -12,29 +12,33 @@ import MySQLdb.cursors SRC_HOST = '127.0.0.1' SRC_USER = 'root' -SRC_PASS = '' +SRC_PASS = 'root' DST_HOST = '127.0.0.1' DST_USER = 'root' -DST_PASS = '' +DST_PASS = 'root' -src_conn = mdb.connect(SRC_HOST, SRC_USER, SRC_PASS, 'ssbc', charset='utf8', cursorclass=MySQLdb.cursors.DictCursor) +src_conn = mdb.connect(SRC_HOST, SRC_USER, SRC_PASS, 'ssbc', + charset='utf8', cursorclass=MySQLdb.cursors.DictCursor) src_curr = src_conn.cursor() src_curr.execute('SET NAMES utf8') -dst_conn = mdb.connect(DST_HOST, DST_USER, DST_PASS, 'rt_main', port=9306, charset='utf8') +dst_conn = mdb.connect(DST_HOST, DST_USER, DST_PASS, + 'rt_main', port=9306, charset='utf8') dst_curr = dst_conn.cursor() dst_curr.execute('SET NAMES utf8') + def work(): src_curr.execute('SELECT id, name, CRC32(category) AS category, length, UNIX_TIMESTAMP(create_time) AS create_time, ' + - 'UNIX_TIMESTAMP(last_seen) AS last_seen FROM search_hash WHERE tagged=false LIMIT 10000') + 'UNIX_TIMESTAMP(last_seen) AS last_seen FROM search_hash WHERE tagged=false LIMIT 10000') total = src_curr.rowcount print 'fetched', total for one in src_curr: ret = dst_curr.execute('insert into rt_main(id,name,category,length,create_time,last_seen) values(%s,%s,%s,%s,%s,%s)', - (one['id'], one['name'], one['category'], one['length'], one['create_time'], one['last_seen'])) + (one['id'], one['name'], one['category'], one['length'], one['create_time'], one['last_seen'])) if ret: - src_curr.execute('UPDATE search_hash SET tagged=True WHERE id=%s', (one['id'],)) + src_curr.execute( + 'UPDATE search_hash SET tagged=True WHERE id=%s', (one['id'],)) print 'Indexed', one['name'].encode('utf8') print 'Done!' return total @@ -46,4 +50,3 @@ if __name__ == '__main__': continue print 'Wait 10mins...' time.sleep(600) - diff --git a/plugins/simdht/workers/simdht_worker.py b/plugins/simdht/workers/simdht_worker.py index 1d6a7a1cf..17019769d 100755 --- a/plugins/simdht/workers/simdht_worker.py +++ b/plugins/simdht/workers/simdht_worker.py @@ -44,7 +44,7 @@ from metadata import save_metadata DB_HOST = '127.0.0.1' DB_USER = 'root' DB_PORT = 3306 -DB_PASS = '' +DB_PASS = 'root' DB_NAME = 'ssbc' BLACK_FILE = 'black_list.txt' @@ -62,6 +62,7 @@ MAX_QUEUE_PT = 200 geoip = pygeoip.GeoIP('GeoIP.dat') + def load_res_blacklist(black_list_path): black_list = [] file_path = os.path.join(os.path.dirname(__file__), black_list_path) @@ -74,8 +75,10 @@ def load_res_blacklist(black_list_path): f.close() return black_list + def is_ip_allowed(ip): - return geoip.country_code_by_addr(ip) not in ('CN','TW','HK') + return geoip.country_code_by_addr(ip) not in ('CN', 'TW', 'HK') + def entropy(length): return "".join(chr(randint(0, 255)) for _ in xrange(length)) @@ -94,9 +97,9 @@ def decode_nodes(nodes): return n for i in range(0, length, 26): - nid = nodes[i:i+20] - ip = inet_ntoa(nodes[i+20:i+24]) - port = unpack("!H", nodes[i+24:i+26])[0] + nid = nodes[i:i + 20] + ip = inet_ntoa(nodes[i + 20:i + 24]) + port = unpack("!H", nodes[i + 24:i + 26])[0] n.append((nid, ip, port)) return n @@ -107,7 +110,7 @@ def timer(t, f): def get_neighbor(target, nid, end=10): - return target[:end]+nid[end:] + return target[:end] + nid[end:] class KNode(object): @@ -173,8 +176,10 @@ class DHTClient(Thread): nodes = decode_nodes(msg["r"]["nodes"]) for node in nodes: (nid, ip, port) = node - if len(nid) != 20: continue - if ip == self.bind_ip: continue + if len(nid) != 20: + continue + if ip == self.bind_ip: + continue n = KNode(nid, ip, port) self.nodes.append(n) @@ -193,12 +198,12 @@ class DHTServer(DHTClient): "announce_peer": self.on_announce_peer_request, } - self.ufd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + self.ufd = socket.socket( + socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) self.ufd.bind((self.bind_ip, self.bind_port)) timer(RE_JOIN_DHT_INTERVAL, self.re_join_DHT) - def run(self): self.re_join_DHT() while True: @@ -290,12 +295,14 @@ class DHTServer(DHTClient): class Master(Thread): + def __init__(self): Thread.__init__(self) self.setDaemon(True) self.queue = Queue() self.metadata_queue = Queue() - self.dbconn = mdb.connect(DB_HOST, DB_USER, DB_PASS, DB_NAME, port=DB_PORT, charset='utf8') + self.dbconn = mdb.connect( + DB_HOST, DB_USER, DB_PASS, DB_NAME, port=DB_PORT, charset='utf8') self.dbconn.autocommit(False) self.dbcurr = self.dbconn.cursor() self.dbcurr.execute('SET NAMES utf8') @@ -314,10 +321,10 @@ class Master(Thread): return self.n_valid += 1 - save_metadata(self.dbcurr, binhash, address, start_time, data,self.black_list) + save_metadata(self.dbcurr, binhash, address, + start_time, data, self.black_list) self.n_new += 1 - def run(self): self.name = threading.currentThread().getName() print self.name, 'started' @@ -339,31 +346,35 @@ class Master(Thread): date = datetime.datetime(date.year, date.month, date.day) # Check if we have this info_hash - self.dbcurr.execute('SELECT id FROM search_hash WHERE info_hash=%s', (info_hash,)) + self.dbcurr.execute( + 'SELECT id FROM search_hash WHERE info_hash=%s', (info_hash,)) y = self.dbcurr.fetchone() if y: self.n_valid += 1 # 更新最近发现时间，请求数 - self.dbcurr.execute('UPDATE search_hash SET last_seen=%s, requests=requests+1 WHERE info_hash=%s', (utcnow, info_hash)) + self.dbcurr.execute( + 'UPDATE search_hash SET last_seen=%s, requests=requests+1 WHERE info_hash=%s', (utcnow, info_hash)) else: if dtype == 'pt': - t = threading.Thread(target=simMetadata.download_metadata, args=(address, binhash, self.metadata_queue)) + t = threading.Thread(target=simMetadata.download_metadata, args=( + address, binhash, self.metadata_queue)) t.setDaemon(True) t.start() self.n_downloading_pt += 1 elif dtype == 'lt' and self.n_downloading_lt < MAX_QUEUE_LT: - t = threading.Thread(target=ltMetadata.download_metadata, args=(address, binhash, self.metadata_queue)) + t = threading.Thread(target=ltMetadata.download_metadata, args=( + address, binhash, self.metadata_queue)) t.setDaemon(True) t.start() self.n_downloading_lt += 1 if self.n_reqs >= 1000: self.dbcurr.execute('INSERT INTO search_statusreport(date,new_hashes,total_requests, valid_requests) VALUES(%s,%s,%s,%s) ON DUPLICATE KEY UPDATE ' + - 'total_requests=total_requests+%s, valid_requests=valid_requests+%s, new_hashes=new_hashes+%s', - (date, self.n_new, self.n_reqs, self.n_valid, self.n_reqs, self.n_valid, self.n_new)) + 'total_requests=total_requests+%s, valid_requests=valid_requests+%s, new_hashes=new_hashes+%s', + (date, self.n_new, self.n_reqs, self.n_valid, self.n_reqs, self.n_valid, self.n_new)) self.dbconn.commit() - print '\n', time.ctime(), 'n_reqs', self.n_reqs, 'n_valid', self.n_valid, 'n_new', self.n_new, 'n_queue', self.queue.qsize(), - print 'n_d_pt', self.n_downloading_pt, 'n_d_lt', self.n_downloading_lt, + print '\n', time.ctime(), 'n_reqs', self.n_reqs, 'n_valid', self.n_valid, 'n_new', self.n_new, 'n_queue', self.queue.qsize(), + print 'n_d_pt', self.n_downloading_pt, 'n_d_lt', self.n_downloading_lt, self.n_reqs = self.n_valid = self.n_new = 0 def log_announce(self, binhash, address=None): @@ -385,7 +396,8 @@ def announce(info_hash, address): def rpc_server(): - rpcserver = SimpleXMLRPCServer.SimpleXMLRPCServer(('localhost', 8004), logRequests=False) + rpcserver = SimpleXMLRPCServer.SimpleXMLRPCServer( + ('localhost', 8004), logRequests=False) rpcserver.register_function(announce, 'announce') print 'Starting xml rpc server...' rpcserver.serve_forever() @@ -402,5 +414,3 @@ if __name__ == "__main__": dht = DHTServer(master, "0.0.0.0", 6881, max_node_qsize=200) dht.start() dht.auto_send_find_node() - -