Skip to content

K
knowledge_graph

文档服务地址：http://47.92.0.57:3000/ 周报索引地址：http://47.92.0.57:3000/s/NruNXRYmV

Commit fadccd9f by lyh3024
Options

add domain 

Change-Id: Iccb00f354dca2e1049979b9451a20a6a25a0c254
parent 236d10f9
Showing with 209 additions and 77 deletions
block/views.py
from django.shortcuts import render
# charset = utf-8
# Create your views here.
from django.http import JsonResponse
from utils import tools
def get_block_list(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_block_trade_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_block_miner_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
domain/models.py
# charset = utf-8
from neomodel import db
from neomodel import (
StructuredNode,
StructuredRel,
StringProperty,
DateTimeProperty,
RelationshipTo,
UniqueIdProperty
RelationshipTo
)
db.set_connection('bolt://neo4j:neo4j@localhost:7687/domain')
class DomainRel(StructuredRel):
index = StringProperty(required=True)
date = DateTimeProperty(required=True) # 请求时间
class RequestIP(StructuredNode):
node_id = UniqueIdProperty()
request_ip = StringProperty(required=True)
province = StringProperty(required=True)
isp = StringProperty(required=True)
class RequestIp(StructuredNode):
request_ip = StringProperty(required=True, unique_index=True)
province = StringProperty(required=True, index=True)
isp = StringProperty(required=True, index=True)
request_domain = RelationshipTo('DomainName', 'REQUEST', model=DomainRel)
class DnsIP(StructuredNode):
node_id = UniqueIdProperty()
dns_ip = StringProperty(required=True, index=True)
class DnsIp(StructuredNode):
dns_ip = StringProperty(required=True, unique_index=True)
parse_domain = RelationshipTo('DomainName', 'PARSE', model=DomainRel)
class DomainName(StructuredNode):
node_id = UniqueIdProperty()
domain_name = StringProperty(required=True, index=True)
ipv4_address = StringProperty(required=True)
label = RelationshipTo('Label', 'Label', model=DomainRel)
domain_name = StringProperty(required=True, unique_index=True)
label = RelationshipTo('Label', 'LABEL', model=DomainRel)
class Label(StructuredNode):
node_id = UniqueIdProperty()
label_name = StringProperty(index=True)
label_name = StringProperty(required=True, unique_index=True)
def gen_node_json(node):
if isinstance(node, RequestIp):
category, label = 1, "请求IP"
elif isinstance(node, DomainName):
category, label = 2, "请求域名"
elif isinstance(node, DnsIp):
category, label = 3, "DNS服务器IP"
else:
category, label = 4, "标签"
return {
"id": node.id,
"category": category,
"label": label,
"properties": {}
}
domain/tests.py
from django.test import TestCase
# charset = utf-8
# Create your tests here.
from datetime import datetime
from neomodel import db
from domain.models import Label, RequestIp, DnsIp, DomainName
db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
def test():
t1 = datetime(year=2022, month=1, day=1)
l1 = Label(label_name="恶意域名")
l2 = Label(label_name="境外域名")
l1.save()
l2.save()
req1 = RequestIp(request_ip='127.0.0.1', province='四川', isp='电信')
req2 = RequestIp(request_ip='127.0.0.2', province='四川', isp='电信')
req1.save()
req2.save()
d1 = DnsIp(dns_ip='127.0.0.3')
d2 = DnsIp(dns_ip='127.0.0.4')
d1.save()
d2.save()
do1 = DomainName(domain_name='www.baidu.com')
do2 = DomainName(domain_name='google.cn')
do1.save()
do2.save()
r1 = req1.request_domain.connect(do1, {"rel_id": "123", "date": t1})
r2 = req2.request_domain.connect(do2, {"rel_id": "124", "date": t1})
r3 = d1.parse_domain.connect(do1, {"rel_id": "125", "date": t1})
r4 = d2.parse_domain.connect(do2, {"rel_id": "126", "date": t1})
r5 = do1.label.connect(l1, {"rel_id": "127", "date": t1})
r6 = do2.label.connect(l2, {"rel_id": "128", "date": t1})
for r in [r1, r2, r3, r4, r5, r6]:
r.save()
if __name__ == '__main__':
test()
domain/views.py
from django.shortcuts import render
# charset = utf-8
# Create your views here.
from django.http import JsonResponse
from neomodel import db
from domain.models import Label, RequestIp, gen_node_json
from utils import tools
def get_label_list(request):
pass
if request.method == "GET":
try:
items = Label.nodes.all()
labels = []
for item in items:
labels.append({"name": item.label_name})
resp = tools.dec_success_resp({"labels": labels})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_request_ip_nodes(label, from_time, end_time):
try:
from_time = tools.from_str_to_time(from_time)
end_time = tools.from_str_to_time(end_time)
if label:
if from_time:
param = {"label": label, "from_time": from_time, "end_time": end_time}
query = 'match (r:RequestIp)-[]->(d:DomainName)-[]->(l:Label) where d.label_name = $label return r'
else:
param = {"label": label}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) where d.label_name = $label and req.date >= $from_time and req.date <= $end_time return r'
nodes = db.cypher_query(query, param)[0]
else:
if from_time:
query = RequestIp.nodes.request_domain.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all()
else:
nodes = RequestIp.nodes.all()
return nodes
except Exception as e:
raise e
def get_domain_list(request):
pass
if request.method == "GET":
try:
label = request.GET.get("label")
from_time = request.GET.get('from_time')
end_time = request.GET.get('to_time')
if from_time and not end_time:
resp = tools.dec_error_resp("end_time is null")
return JsonResponse(resp)
if not from_time and end_time:
resp = tools.dec_error_resp("from_time is null")
return JsonResponse(resp)
resp_nodes, resp_relations = [], []
request_ip_nodes = get_request_ip_nodes(label, from_time, end_time)
request_ip_node_ids = [r.id for r in request_ip_nodes]
param = {"ids": request_ip_node_ids}
query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
'where d.label_name = $label and req.date >= $from_time and req.date <= $end_time return r'
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_label_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_domain_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_request_ip_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
def get_dns_ip_rank(request):
if request.method == "GET":
try:
pass
except Exception as e:
resp = tools.dec_error_resp(e)
return JsonResponse(resp, safe=False)
knowledge_graph/settings.py
......@@ -79,14 +79,14 @@ WSGI_APPLICATION = 'knowledge_graph.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
# DATABASES = {
# 'default': {
# 'ENGINE': 'django.db.backends.sqlite3',
# 'NAME': BASE_DIR / 'db.sqlite3',
# }
# }
config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687/neo4j'
# Password validation
......
route/models.py
# charset = utf-8
from neomodel import db
from neomodel import (
StructuredNode,
StructuredRel,
StringProperty,
UniqueIdProperty,
DateTimeProperty,
RelationshipTo
)
db.set_connection('bolt://neo4j:neo4j@localhost:7687/route')
# Create your models here.
class NextIpRel(StructuredRel):
rel_id = StringProperty(required=True)
index = StringProperty(required=True)
date = DateTimeProperty(required=True)
class RouteIp(StructuredNode):
node_id = UniqueIdProperty()
route_ip = StringProperty(required=True, index=True)
route_ip = StringProperty(required=True, unique_index=True)
country = StringProperty(required=True)
province = StringProperty(required=True, index=True)
city = StringProperty(required=True, index=True)
......@@ -48,7 +41,7 @@ def gen_node_json(node):
else:
category, label = 2, "中间路由"
return {
"id": node.node_id,
"id": node.id,
"category": category,
"label": label,
"properties": {
......@@ -70,9 +63,9 @@ def gen_rel_json(r):
else:
label = '中间跳'
return {
"id": r.rel_id,
"source": s.node_id,
"target": e.node_id,
"id": r.id,
"source": s.id,
"target": e.id,
"label": label,
"date": r.date
}
route/tests.py
......@@ -2,22 +2,21 @@ from neomodel import db
from datetime import datetime
from route.models import RouteIp, FirstIp, EndIp
db.set_connection('bolt://neo4j:neo4j@localhost:7687/route')
# Create your tests here.
db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
def test():
t1 = datetime(year=2022, month=1, day=1)
n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
n1.save()
n2.save()
n3.save()
r1 = n1.next_ip.connect(n2, {'rel_id': '123', 'date': t1})
r2 = n1.next_ip.connect(n2, {'rel_id': '124', 'date': t1})
r3 = n2.next_ip.connect(n3, {'rel_id': '125', 'date': t1})
r4 = n2.next_ip.connect(n3, {'rel_id': '126', 'date': t1})
r1 = n1.next_ip.connect(n2, {"index": "123", 'date': t1})
r2 = n1.next_ip.connect(n2, {"index": "124", 'date': t1})
r3 = n2.next_ip.connect(n3, {"index": "125", 'date': t1})
r4 = n2.next_ip.connect(n3, {"index": "126", 'date': t1})
r1.save()
r2.save()
r3.save()
......
route/views.py
# charset = utf-8
from django.http import JsonResponse
from neomodel import Q
from neomodel import Q, db
from route.models import FirstIp, EndIp, RouteIp, gen_node_json, gen_rel_json
from route.models import FirstIp, EndIp, gen_node_json, gen_rel_json
from utils import tools
......@@ -51,7 +53,8 @@ def get_first_nodes(province, city, isp, root_ip, from_time, end_time):
if root_ip:
query = query.filter(root_ip=root_ip)
if from_time and end_time:
query = query.next_ip.match(Q(date__gte=from_time) | Q(date__lte=end_time))
query = query.next_ip.match(date__gte=from_time)
query = query.match(date__lte=end_time)
nodes = query.all()
return nodes
......@@ -119,24 +122,16 @@ def get_route_rank(request):
first_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time)
first_node_ips = [node.route_ip for node in first_nodes]
all_nodes = RouteIp.nodes.filter()
param = {'start_ips': first_node_ips}
query = 'match (n:RouteIp) where n.start_ip in $start_ips with n, size((n)-[]-()) as size order by size desc limit 100 return n.route_ip, n.province ,n.isp, size'
items = db.cypher_query(query, param)[0]
resp_nodes, resp_relations = [], []
while current_nodes:
tmp_nodes = []
tmp_node_ids = [] # 手动去掉重复的node
for node in current_nodes:
resp_nodes.append(gen_node_json(node))
next_nodes = node.next_ip.all()
for next_node in next_nodes:
if next_node.id not in tmp_node_ids:
tmp_nodes.append(next_node)
for r in node.next_ip.all_relationships(next_node):
resp_relations.append(gen_rel_json(r))
tmp_node_ids.append(next_node.id)
current_nodes = tmp_nodes
rank = []
for item in items:
if len(item) != 0:
rank.append({"ip": item[0], "province": item[1], "isp": item[2], "degree": item[3]})
resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
resp = tools.dec_success_resp({"rank": rank})
return JsonResponse(resp, safe=False)
except Exception as e:
resp = tools.dec_error_resp(e)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment