Skip to content

K
knowledge_graph

文档服务地址：http://47.92.0.57:3000/ 周报索引地址：http://47.92.0.57:3000/s/NruNXRYmV

Commit fadccd9f by lyh3024
Options

add domain 

Change-Id: Iccb00f354dca2e1049979b9451a20a6a25a0c254
parent 236d10f9
Showing with 209 additions and 77 deletions
block/views.py
from django.shortcuts import render # charset = utf-8
# Create your views here. from django.http import JsonResponse
from utils import tools
def get_block_list(request): def get_block_list(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_block_trade_rank(request): def get_block_trade_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_block_miner_rank(request): def get_block_miner_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
domain/models.py
# charset = utf-8 # charset = utf-8
from neomodel import db
from neomodel import ( from neomodel import (
StructuredNode, StructuredNode,
StructuredRel, StructuredRel,
StringProperty, StringProperty,
DateTimeProperty, DateTimeProperty,
RelationshipTo, RelationshipTo
UniqueIdProperty
) )
db.set_connection('bolt://neo4j:neo4j@localhost:7687/domain')
class DomainRel(StructuredRel): class DomainRel(StructuredRel):
index = StringProperty(required=True)
date = DateTimeProperty(required=True) # 请求时间 date = DateTimeProperty(required=True) # 请求时间
class RequestIP(StructuredNode): class RequestIp(StructuredNode):
node_id = UniqueIdProperty() request_ip = StringProperty(required=True, unique_index=True)
request_ip = StringProperty(required=True) province = StringProperty(required=True, index=True)
province = StringProperty(required=True) isp = StringProperty(required=True, index=True)
isp = StringProperty(required=True)
request_domain = RelationshipTo('DomainName', 'REQUEST', model=DomainRel) request_domain = RelationshipTo('DomainName', 'REQUEST', model=DomainRel)
class DnsIP(StructuredNode): class DnsIp(StructuredNode):
node_id = UniqueIdProperty() dns_ip = StringProperty(required=True, unique_index=True)
dns_ip = StringProperty(required=True, index=True)
parse_domain = RelationshipTo('DomainName', 'PARSE', model=DomainRel) parse_domain = RelationshipTo('DomainName', 'PARSE', model=DomainRel)
class DomainName(StructuredNode): class DomainName(StructuredNode):
node_id = UniqueIdProperty() domain_name = StringProperty(required=True, unique_index=True)
domain_name = StringProperty(required=True, index=True) label = RelationshipTo('Label', 'LABEL', model=DomainRel)
ipv4_address = StringProperty(required=True)
label = RelationshipTo('Label', 'Label', model=DomainRel)
class Label(StructuredNode): class Label(StructuredNode):
node_id = UniqueIdProperty() label_name = StringProperty(required=True, unique_index=True)
label_name = StringProperty(index=True)
def gen_node_json(node):
if isinstance(node, RequestIp):
category, label = 1, "请求IP"
elif isinstance(node, DomainName):
category, label = 2, "请求域名"
elif isinstance(node, DnsIp):
category, label = 3, "DNS服务器IP"
else:
category, label = 4, "标签"
return {
"id": node.id,
"category": category,
"label": label,
"properties": {}
}
domain/tests.py
from django.test import TestCase # charset = utf-8
# Create your tests here. from datetime import datetime
from neomodel import db
from domain.models import Label, RequestIp, DnsIp, DomainName
db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
def test():
t1 = datetime(year=2022, month=1, day=1)
l1 = Label(label_name="恶意域名")
l2 = Label(label_name="境外域名")
l1.save()
l2.save()
req1 = RequestIp(request_ip='127.0.0.1', province='四川', isp='电信')
req2 = RequestIp(request_ip='127.0.0.2', province='四川', isp='电信')
req1.save()
req2.save()
d1 = DnsIp(dns_ip='127.0.0.3')
d2 = DnsIp(dns_ip='127.0.0.4')
d1.save()
d2.save()
do1 = DomainName(domain_name='www.baidu.com')
do2 = DomainName(domain_name='google.cn')
do1.save()
do2.save()
r1 = req1.request_domain.connect(do1, {"rel_id": "123", "date": t1})
r2 = req2.request_domain.connect(do2, {"rel_id": "124", "date": t1})
r3 = d1.parse_domain.connect(do1, {"rel_id": "125", "date": t1})
r4 = d2.parse_domain.connect(do2, {"rel_id": "126", "date": t1})
r5 = do1.label.connect(l1, {"rel_id": "127", "date": t1})
r6 = do2.label.connect(l2, {"rel_id": "128", "date": t1})
for r in [r1, r2, r3, r4, r5, r6]:
r.save()
if __name__ == '__main__':
test()
domain/views.py
from django.shortcuts import render # charset = utf-8
# Create your views here. from django.http import JsonResponse
from neomodel import db
from domain.models import Label, RequestIp, gen_node_json
from utils import tools
def get_label_list(request): def get_label_list(request):
pass if request.method == "GET":
try:
items = Label.nodes.all()
labels = []
for item in items:
labels.append({"name": item.label_name})
resp = tools.dec_success_resp({"labels": labels})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_request_ip_nodes(label, from_time, end_time):
try:
from_time = tools.from_str_to_time(from_time)
end_time = tools.from_str_to_time(end_time)
if label:
if from_time:
param = {"label": label, "from_time": from_time, "end_time": end_time}
query = 'match (r:RequestIp)-[]->(d:DomainName)-[]->(l:Label) where d.label_name = $label return r'
else:
param = {"label": label}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) where d.label_name = $label and req.date >= $from_time and req.date <= $end_time return r'
nodes = db.cypher_query(query, param)[0]
else:
if from_time:
query = RequestIp.nodes.request_domain.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all()
else:
nodes = RequestIp.nodes.all()
return nodes
except Exception as e:
raise e
def get_domain_list(request): def get_domain_list(request):
pass if request.method == "GET":
try:
label = request.GET.get("label")
from_time = request.GET.get('from_time')
end_time = request.GET.get('to_time')
if from_time and not end_time:
resp = tools.dec_error_resp("end_time is null")
return JsonResponse(resp)
if not from_time and end_time:
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
resp_nodes, resp_relations = [], []
request_ip_nodes = get_request_ip_nodes(label, from_time, end_time)
request_ip_node_ids = [r.id for r in request_ip_nodes]
param = {"ids": request_ip_node_ids}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
'where d.label_name = $label and req.date >= $from_time and req.date <= $end_time return r'
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_label_rank(request): def get_label_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_domain_rank(request): def get_domain_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_request_ip_rank(request): def get_request_ip_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_dns_ip_rank(request): def get_dns_ip_rank(request):
if request.method == "GET":
try:
pass pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
knowledge_graph/settings.py
...@@ -79,14 +79,14 @@ WSGI_APPLICATION = 'knowledge_graph.wsgi.application' ...@@ -79,14 +79,14 @@ WSGI_APPLICATION = 'knowledge_graph.wsgi.application'
# Database # Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
# DATABASES = { DATABASES = {
# 'default': { 'default': {
# 'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.sqlite3',
# 'NAME': BASE_DIR / 'db.sqlite3', 'NAME': BASE_DIR / 'db.sqlite3',
# } }
# } }
config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687' config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687/neo4j'
# Password validation # Password validation
......
route/models.py
# charset = utf-8 # charset = utf-8
from neomodel import db
from neomodel import ( from neomodel import (
StructuredNode, StructuredNode,
StructuredRel, StructuredRel,
StringProperty, StringProperty,
UniqueIdProperty,
DateTimeProperty, DateTimeProperty,
RelationshipTo RelationshipTo
) )
db.set_connection('bolt://neo4j:neo4j@localhost:7687/route')
# Create your models here.
class NextIpRel(StructuredRel): class NextIpRel(StructuredRel):
rel_id = StringProperty(required=True) index = StringProperty(required=True)
date = DateTimeProperty(required=True) date = DateTimeProperty(required=True)
class RouteIp(StructuredNode): class RouteIp(StructuredNode):
node_id = UniqueIdProperty() route_ip = StringProperty(required=True, unique_index=True)
route_ip = StringProperty(required=True, index=True)
country = StringProperty(required=True) country = StringProperty(required=True)
province = StringProperty(required=True, index=True) province = StringProperty(required=True, index=True)
city = StringProperty(required=True, index=True) city = StringProperty(required=True, index=True)
...@@ -48,7 +41,7 @@ def gen_node_json(node): ...@@ -48,7 +41,7 @@ def gen_node_json(node):
else: else:
category, label = 2, "中间路由" category, label = 2, "中间路由"
return { return {
"id": node.node_id, "id": node.id,
"category": category, "category": category,
"label": label, "label": label,
"properties": { "properties": {
...@@ -70,9 +63,9 @@ def gen_rel_json(r): ...@@ -70,9 +63,9 @@ def gen_rel_json(r):
else: else:
label = '中间跳' label = '中间跳'
return { return {
"id": r.rel_id, "id": r.id,
"source": s.node_id, "source": s.id,
"target": e.node_id, "target": e.id,
"label": label, "label": label,
"date": r.date "date": r.date
} }
route/tests.py
...@@ -2,22 +2,21 @@ from neomodel import db ...@@ -2,22 +2,21 @@ from neomodel import db
from datetime import datetime from datetime import datetime
from route.models import RouteIp, FirstIp, EndIp from route.models import RouteIp, FirstIp, EndIp
db.set_connection('bolt://neo4j:neo4j@localhost:7687/route') db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
# Create your tests here.
def test(): def test():
t1 = datetime(year=2022, month=1, day=1) t1 = datetime(year=2022, month=1, day=1)
n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3') n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3') n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3') n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n1.save() n1.save()
n2.save() n2.save()
n3.save() n3.save()
r1 = n1.next_ip.connect(n2, {'rel_id': '123', 'date': t1}) r1 = n1.next_ip.connect(n2, {"index": "123", 'date': t1})
r2 = n1.next_ip.connect(n2, {'rel_id': '124', 'date': t1}) r2 = n1.next_ip.connect(n2, {"index": "124", 'date': t1})
r3 = n2.next_ip.connect(n3, {'rel_id': '125', 'date': t1}) r3 = n2.next_ip.connect(n3, {"index": "125", 'date': t1})
r4 = n2.next_ip.connect(n3, {'rel_id': '126', 'date': t1}) r4 = n2.next_ip.connect(n3, {"index": "126", 'date': t1})
r1.save() r1.save()
r2.save() r2.save()
r3.save() r3.save()
......
route/views.py
# charset = utf-8
from django.http import JsonResponse from django.http import JsonResponse
from neomodel import Q from neomodel import Q, db
from route.models import FirstIp, EndIp, RouteIp, gen_node_json, gen_rel_json from route.models import FirstIp, EndIp, gen_node_json, gen_rel_json
from utils import tools from utils import tools
...@@ -51,7 +53,8 @@ def get_first_nodes(province, city, isp, root_ip, from_time, end_time): ...@@ -51,7 +53,8 @@ def get_first_nodes(province, city, isp, root_ip, from_time, end_time):
if root_ip: if root_ip:
query = query.filter(root_ip=root_ip) query = query.filter(root_ip=root_ip)
if from_time and end_time: if from_time and end_time:
query = query.next_ip.match(Q(date__gte=from_time) | Q(date__lte=end_time)) query = query.next_ip.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all() nodes = query.all()
return nodes return nodes
...@@ -119,24 +122,16 @@ def get_route_rank(request): ...@@ -119,24 +122,16 @@ def get_route_rank(request):
first_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time) first_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time)
first_node_ips = [node.route_ip for node in first_nodes] first_node_ips = [node.route_ip for node in first_nodes]
all_nodes = RouteIp.nodes.filter() param = {'start_ips': first_node_ips}
query = 'match (n:RouteIp) where n.start_ip in $start_ips with n, size((n)-[]-()) as size order by size desc limit 100 return n.route_ip, n.province ,n.isp, size'
items = db.cypher_query(query, param)[0]
resp_nodes, resp_relations = [], [] rank = []
while current_nodes: for item in items:
tmp_nodes = [] if len(item) != 0:
tmp_node_ids = [] # 手动去掉重复的node rank.append({"ip": item[0], "province": item[1], "isp": item[2], "degree": item[3]})
for node in current_nodes:
resp_nodes.append(gen_node_json(node))
next_nodes = node.next_ip.all()
for next_node in next_nodes:
if next_node.id not in tmp_node_ids:
tmp_nodes.append(next_node)
for r in node.next_ip.all_relationships(next_node):
resp_relations.append(gen_rel_json(r))
tmp_node_ids.append(next_node.id)
current_nodes = tmp_nodes
resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations}) resp = tools.dec_success_resp({"rank": rank})
return JsonResponse(resp, safe=False) return JsonResponse(resp, safe=False)
except Exception as e: except Exception as e:
resp = tools.dec_error_resp(e) resp = tools.dec_error_resp(e)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment