Skip to content

K
knowledge_graph

文档服务地址：http://47.92.0.57:3000/ 周报索引地址：http://47.92.0.57:3000/s/NruNXRYmV

Commit 6d020fec by lyh3024
Options

done 

Change-Id: I8b5696a0ec5197a55a3e27e3b4ba5b9471335814
parent 3dd41db2
Showing with 227 additions and 47 deletions
block/models.py
......@@ -29,15 +29,58 @@ class Miner(StructuredNode):
class Transaction(StructuredNode):
hash = StringProperty(required=True, index=True)
amount = FloatProperty(required=True)
trade = RelationshipTo('Block', 'Trade')
transfer_in = RelationshipTo('Account', 'TRANSFER_IN')
invoke = RelationshipTo('Contract', 'INVOKE')
trade = RelationshipTo('Block', 'Trade', model=BlockRel)
transfer_in = RelationshipTo('Account', 'TRANSFER_IN', model=BlockRel)
invoke = RelationshipTo('Contract', 'INVOKE', model=BlockRel)
class Account(StructuredNode):
address = StringProperty(required=True, index=True)
transfer_out = RelationshipTo('Transaction', 'TRANSFER_OUT')
transfer_out = RelationshipTo('Transaction', 'TRANSFER_OUT', model=BlockRel)
class Contract(StructuredNode):
address = StringProperty(required=True, index=True)
def gen_node_json(node):
if isinstance(node, Block):
category, label = 1, "区块"
elif isinstance(node, Transaction):
category, label = 2, "交易"
elif isinstance(node, Miner):
category, label = 3, "矿工"
elif isinstance(node, Account):
category, label = 4, "账户"
else:
category, label = 5, "合约"
return {
"id": node.id,
"category": category,
"label": label,
"properties": {}
}
def gen_rel_json(r):
s = r.start_node()
e = r.end_node()
if isinstance(s, Account):
label = "转出"
elif isinstance(s, Transaction):
if isinstance(e, Account):
label = '转入'
elif isinstance(e, Contract):
label = '调用'
else:
label = "交易区块"
elif isinstance(s, Block):
label = '下一区块'
else:
label = '打包'
return {
"id": r.id,
"source": s.id,
"target": e.id,
"label": label
}
block/tests.py
from django.test import TestCase
# charset = utf-8
from datetime import datetime
from openpyxl import load_workbook
from datetime import datetime
from neomodel import db
from block.models import Account, Transaction, Contract, Miner, Block
from block import models
db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
def add_test_data():
pass
def test():
time1 = datetime(year=2022, month=1, day=1)
hash1 = '0xdf4af22891e6831a621c8641b3bb01d880117a7e01aab04a8b262385220c1b06'
hash2 = '0x8c46fe20c6d8d56f02e126f5d13231577ee6d56b5ec6a108fd6f222291de3263'
hash3 = '0xed77bac6775469ab6ef272bc918f0c9f7c019659f67fc3727cd97ea4de99954b'
hash4 = '0x12f5421f4e68c47d27068089ffe2bd02af0a4c9d34438153b1f872482f289292'
hash5 = '0xaea2c08b5f2b66e8460e332e364a6d478d9e7f5acc04db3b9f4a047f6a08fc01'
hash6 = '0xabe02f108f350877493fa49fb3017788333970ea2e78962b7f2b79707cda7b77'
b1 = Block(hash=hash1)
b2 = Block(hash=hash2)
b3 = Block(hash=hash3)
b4 = Block(hash=hash4)
t1 = Transaction(hash=hash5, amount=10000.0)
t2 = Transaction(hash=hash6, amount=20000.0)
address1 = '0x65f716f794026d4ff4723141100f0e3c3080e97d'
address2 = '0x687b6e85b9b08a33991b6ceeeb1d12df278a9a5b'
address3 = '0x17a84ba96a7374de9586a770b4529f57c6b991d6'
address4 = '0x422bc251a0582aba25e2dd3b27e80435bc13d402'
address5 = '0x2daa35962a6d43eb54c48367b33d0b379c930e5e'
address6 = '0x829bd824b016326a401d083b33d092293333a830'
a1 = Account(address=address1)
a2 = Account(address=address2)
c1 = Contract(address=address3)
c2 = Contract(address=address4)
m1 = Miner(address=address5)
m2 = Miner(address=address6)
for n in [b1, b2, b3, b4, t1, t2, a1, a2, c1, c2, m1, m2]:
n.save()
r1 = b1.next_block.connect(b2, {"index": "123", "date": time1})
r2 = b3.next_block.connect(b4, {"index": "124", "date": time1})
r3 = m1.pack.connect(b1, {"index": "125", "date": time1})
r4 = m1.pack.connect(b2, {"index": "126", "date": time1})
r5 = m1.pack.connect(b3, {"index": "127", "date": time1})
r6 = m2.pack.connect(b4, {"index": "128", "date": time1})
r77 = t1.trade.connect(b1, {"index": "129", "date": time1})
r7 = t2.trade.connect(b3, {"index": "130", "date": time1})
r8 = a1.transfer_out.connect(t1, {"index": "131", "date": time1})
r9 = a2.transfer_out.connect(t2, {"index": "132", "date": time1})
r10 = t1.transfer_in.connect(a1, {"index": "133", "date": time1})
r11 = t1.invoke.connect(c1, {"index": "134", "date": time1})
r12 = t2.invoke.connect(c2, {"index": "135", "date": time1})
for r in [r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r77]:
r.save()
if __name__ == '__main__':
test()
domain/models.py
......@@ -82,5 +82,5 @@ def gen_relation_json2(relation):
"id": relation.id,
"source": relation.nodes[0].id,
"target": relation.nodes[1].id,
"label": label,
"label": label
}
domain/tests.py
......@@ -88,7 +88,18 @@ def test3():
r2.save()
def test4():
param = {"ips": ['127.0.0.1', '127.0.0.2', '127.0.0.10', '127.0.0.12']}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
'return l.label_name, count(distinct lab), count(req) order by count(distinct lab) desc'
items = db.cypher_query(query, param)[0]
for item in items:
a, b, c = item
print("a=%s, b=%s, c=%s" % (a, b, c))
if __name__ == '__main__':
test()
test2()
test3()
# test4()
domain/views.py
......@@ -12,10 +12,13 @@ def get_label_list(request):
try:
items = Label.nodes.all()
labels = []
i = 1
for item in items:
labels.append({"name": item.label_name})
labels.append({"id": i, "name": item.label_name})
i += 1
resp = tools.dec_success_resp({"labels": labels})
resp = tools.dec_success_resp({"count": len(items), "labels": labels})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
......@@ -27,26 +30,26 @@ def get_request_ips(label, from_time, end_time):
from_time = tools.from_str_to_time(from_time)
end_time = tools.from_str_to_time(end_time)
ips = []
if label:
if from_time:
param = {"label": label, "from_time": from_time.timestamp(), "end_time": end_time.timestamp()}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) where d.label_name = $label and req.date >= $from_time and req.date <= $end_time return r'
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
'where l.label_name = $label and req.date >= $from_time and req.date <= $end_time return r.request_ip'
else:
param = {"label": label}
query = 'match (r:RequestIp)-[]->(d:DomainName)-[]->(l:Label) where d.label_name = $label return r'
query = 'match (r:RequestIp)-[]->(d:DomainName)-[]->(l:Label) where l.label_name = $label return r'
nodes = db.cypher_query(query, param)[0]
for node in nodes:
ips.append(getattr(node, '_properties').get("request_ip"))
ips = [node[0] for node in nodes]
else:
if from_time:
query = RequestIp.nodes.request_domain.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all()
param = {"from_time": from_time.timestamp(), "end_time": end_time.timestamp()}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
'where req.date >= $from_time and req.date <= $end_time return r.request_ip'
nodes = db.cypher_query(query, param)[0]
ips = [node[0] for node in nodes]
else:
nodes = RequestIp.nodes.all()
for node in nodes:
ips.append(node.request_ip)
ips = [node.request_ip for node in nodes]
return ips
except Exception as e:
raise e
......@@ -85,6 +88,9 @@ def get_domain_list(request):
if r.id in r_ids:
continue
resp_relations.append(gen_relation_json2(r))
r_ids.append(r.id)
resp_nodes.sort(key=lambda n: n.get("id"))
resp_relations.sort(key=lambda r: r.get("id"))
resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
return JsonResponse(resp, safe=False)
......@@ -108,12 +114,24 @@ def get_label_rank(request):
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
rank = []
ips = get_request_ips(label, from_time, end_time)
param = {"ips": ips}
if sort_type == 1:
pass
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
'return l.label_name, count(distinct lab), count(req) order by count(distinct lab) desc'
else:
pass
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
'return l.label_name, count(distinct lab), count(req) order by count(req) desc'
items = db.cypher_query(query, param)[0]
i = 1
for item in items:
name, domain_num, req_count = item
rank.append({"id": i, "name": name, "domainNum": domain_num, "requestNum": req_count})
i += 1
resp = tools.dec_success_resp({"count": len(items), "rank": rank})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
......@@ -133,6 +151,21 @@ def get_domain_rank(request):
if not from_time and end_time:
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
rank = []
ips = get_request_ips(label, from_time, end_time)
param = {"ips": ips}
query = 'match (r:RequestIp)-[]->(d:DomainName), (dns:DnsIp)-[p:PARSE]->(d) where r.request_ip in $ips ' \
'return d.domain_name, count(p) order by count(p) desc'
items = db.cypher_query(query, param)[0]
i = 1
for item in items:
name, parse_num = item
rank.append({"id": i, "name": name, "parseNum": parse_num})
i += 1
resp = tools.dec_success_resp({"count": len(items), "rank": rank})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
......@@ -151,6 +184,21 @@ def get_request_ip_rank(request):
if not from_time and end_time:
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
rank = []
ips = get_request_ips(label, from_time, end_time)
param = {"ips": ips}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName) where r.request_ip in $ips ' \
'return r.province, r.isp, count(req)'
items = db.cypher_query(query, param)[0]
i = 1
for item in items:
province, isp, request_num = item
rank.append({"id": i, "province": province, "isp": isp, "requestNum": request_num})
i += 1
resp = tools.dec_success_resp({"count": len(items), "rank": rank})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
......@@ -169,6 +217,21 @@ def get_dns_ip_rank(request):
if not from_time and end_time:
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
rank = []
ips = get_request_ips(label, from_time, end_time)
param = {"ips": ips}
query = 'match (r:RequestIp)-[]->(d:DomainName), (dns:DnsIp)-[p:PARSE]->(d) where r.request_ip in $ips ' \
'return r.province,r.isp, count(distinct p) order by count(distinct p) desc'
items = db.cypher_query(query, param)[0]
i = 1
for item in items:
province, isp, parse_num = item
rank.append({"id": i, "province": province, "isp": isp, "parseNum": parse_num})
i += 1
resp = tools.dec_success_resp({"count": len(items), "rank": rank})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
......
route/models.py
......@@ -15,7 +15,7 @@ class NextIpRel(StructuredRel):
class RouteIp(StructuredNode):
route_ip = StringProperty(required=True, unique_index=True)
ip = StringProperty(required=True, unique_index=True)
country = StringProperty(required=True)
province = StringProperty(required=True, index=True)
city = StringProperty(required=True, index=True)
......@@ -45,7 +45,7 @@ def gen_node_json(node):
"category": category,
"label": label,
"properties": {
"ip": node.route_ip,
"ip": node.ip,
"province": node.province,
"city": node.city,
"isp": node.isp,
......
route/tests.py
......@@ -7,9 +7,9 @@ db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
def test():
t1 = datetime(year=2022, month=1, day=1)
n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n1 = FirstIp(ip='127.0.0.1', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n2 = RouteIp(ip='127.0.0.2', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n3 = EndIp(ip='127.0.0.3', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n1.save()
n2.save()
n3.save()
......
route/views.py
......@@ -38,26 +38,43 @@ def get_route_param(request):
return JsonResponse(resp, safe=False)
def get_first_nodes(province, city, isp, root_ip, from_time, end_time):
def get_first_ips(province, city, isp, root_ip, from_time, end_time):
try:
from_time = tools.from_str_to_time(from_time)
end_time = tools.from_str_to_time(end_time)
query = FirstIp.nodes
params = {}
query = 'match (f:FirstIp)-[n:NEXT_IP]->() '
query_extend = []
if province:
query = query.filter(province=province)
params['province'] = province
query_extend.append('f.province = $province ')
if city:
query = query.filter(city=city)
params['city'] = city
query_extend.append('f.city = $city ')
if isp:
query = query.filter(isp=isp)
params['isp'] = isp
query_extend.append('f.isp = $isp ')
if root_ip:
query = query.filter(root_ip=root_ip)
params['root_ip'] = root_ip
query_extend.append('f.root_ip = $root_ip ')
if from_time and end_time:
query = query.next_ip.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all()
return nodes
params['from_time'] = from_time.timestamp()
params['ent_time'] = end_time.timestamp()
query_extend.append('n.date >= $from_time ')
query_extend.append('n.date <= $end_time ')
if len(query_extend) != 0:
query += "where "
for i in range(len(query_extend)):
if i == 0:
query += query_extend[i]
else:
query += 'and ' + query_extend[i]
query += 'return f.ip'
items = db.cypher_query(query, params)[0]
ips = [item[0] for item in items]
return ips
except Exception as e:
raise e
......@@ -70,7 +87,7 @@ def get_route_list(request):
isp = request.GET.get('isp')
root_ip = request.GET.get('root_ip')
from_time = request.GET.get('from_time')
end_time = request.GET.get('to_time')
end_time = request.GET.get('end_time')
if from_time and not end_time:
resp = tools.dec_error_resp("end_time is null")
......@@ -79,8 +96,9 @@ def get_route_list(request):
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
current_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time) # 当前遍历到的节点集合
ips = get_first_ips(province, city, isp, root_ip, from_time, end_time) # 当前遍历到的节点集合
current_nodes = FirstIp.nodes.filter(ip__in=ips).all()
resp_nodes, resp_relations = [], []
while current_nodes:
tmp_nodes = []
......@@ -111,7 +129,7 @@ def get_route_rank(request):
isp = request.GET.get('isp')
root_ip = request.GET.get('root_ip')
from_time = request.GET.get('from_time')
end_time = request.GET.get('to_time')
end_time = request.GET.get('end_time')
if from_time and not end_time:
resp = tools.dec_error_resp("end_time is null")
......@@ -120,10 +138,9 @@ def get_route_rank(request):
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
first_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time)
first_node_ips = [node.route_ip for node in first_nodes]
first_node_ips = get_first_ips(province, city, isp, root_ip, from_time, end_time)
param = {'start_ips': first_node_ips}
query = 'match (n:RouteIp) where n.start_ip in $start_ips with n, size((n)-[]-()) as size order by size desc limit 100 return n.route_ip, n.province ,n.isp, size'
query = 'match (n:RouteIp) where n.start_ip in $start_ips with n, size((n)-[]-()) as size order by size desc limit 100 return n.ip, n.province ,n.isp, size'
items = db.cypher_query(query, param)[0]
rank = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment