Merge branch 'develop' into 'master'

Develop

See merge request lyh3024/knowledge_graph!1

.gitignore

+.venv
\ No newline at end of file

README.md

@@ -13,4 +13,9 @@
 
 项目采用python3+django+neo4j的架构  
 
-项目采用poetry作为包管理工具,项目具体依赖见poetry.toml
\ No newline at end of file
+项目采用poetry作为包管理工具,项目具体依赖见poetry.toml
+
+
+## 说明
+
+本项目采用neomodel作为python连接neo4j的driver，neomodel具体使用方法见https://neomodel.readthedocs.io/en/latest
\ No newline at end of file

block/models.py

-from django.db import models
+# charset = utf-8
 
-# Create your models here.
+from neomodel import (
+    StructuredNode,
+    StructuredRel,
+    IntegerProperty,
+    StringProperty,
+    DateTimeProperty,
+    RelationshipTo,
+    FloatProperty
+)
+
+
+class BlockRel(StructuredRel):
+    index = StringProperty(required=True)
+    date = DateTimeProperty(required=True)
+
+
+class Block(StructuredNode):
+    hash = StringProperty(required=True, index=True)
+    next_block = RelationshipTo('Block', 'NEXT_BLOCK', model=BlockRel)
+
+
+class Miner(StructuredNode):
+    address = StringProperty(required=True)
+    pack = RelationshipTo('Block', 'PACK', model=BlockRel)
+
+
+class Transaction(StructuredNode):
+    hash = StringProperty(required=True, index=True)
+    amount = FloatProperty(required=True)
+    trade = RelationshipTo('Block', 'Trade', model=BlockRel)
+    transfer_in = RelationshipTo('Account', 'TRANSFER_IN', model=BlockRel)
+    invoke = RelationshipTo('Contract', 'INVOKE', model=BlockRel)
+
+
+class Account(StructuredNode):
+    address = StringProperty(required=True, index=True)
+    transfer_out = RelationshipTo('Transaction', 'TRANSFER_OUT', model=BlockRel)
+
+
+class Contract(StructuredNode):
+    address = StringProperty(required=True, index=True)
+
+
+def gen_node_json(node):
+    if isinstance(node, Block):
+        category, label = 1, "区块"
+    elif isinstance(node, Transaction):
+        category, label = 2, "交易"
+    elif isinstance(node, Miner):
+        category, label = 3, "矿工"
+    elif isinstance(node, Account):
+        category, label = 4, "账户"
+    else:
+        category, label = 5, "合约"
+    return {
+        "id": node.id,
+        "category": category,
+        "label": label,
+        "properties": {}
+    }
+
+
+def gen_rel_json(r):
+    s = r.start_node()
+    e = r.end_node()
+    if isinstance(s, Account):
+        label = "转出"
+    elif isinstance(s, Transaction):
+        if isinstance(e, Account):
+            label = '转入'
+        elif isinstance(e, Contract):
+            label = '调用'
+        else:
+            label = "交易区块"
+    elif isinstance(s, Block):
+        label = '下一区块'
+    else:
+        label = '打包'
+    return {
+        "id": r.id,
+        "source": s.id,
+        "target": e.id,
+        "label": label
+    }

block/tests.py

-from django.test import TestCase
+# charset = utf-8
 
-# Create your tests here.
+from datetime import datetime
+from neomodel import db
+from block.models import Account, Transaction, Contract, Miner, Block
+
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
+
+
+def test():
+    time1 = datetime(year=2022, month=1, day=1)
+    hash1 = '0xdf4af22891e6831a621c8641b3bb01d880117a7e01aab04a8b262385220c1b06'
+    hash2 = '0x8c46fe20c6d8d56f02e126f5d13231577ee6d56b5ec6a108fd6f222291de3263'
+    hash3 = '0xed77bac6775469ab6ef272bc918f0c9f7c019659f67fc3727cd97ea4de99954b'
+    hash4 = '0x12f5421f4e68c47d27068089ffe2bd02af0a4c9d34438153b1f872482f289292'
+    hash5 = '0xaea2c08b5f2b66e8460e332e364a6d478d9e7f5acc04db3b9f4a047f6a08fc01'
+    hash6 = '0xabe02f108f350877493fa49fb3017788333970ea2e78962b7f2b79707cda7b77'
+    b1 = Block(hash=hash1)
+    b2 = Block(hash=hash2)
+    b3 = Block(hash=hash3)
+    b4 = Block(hash=hash4)
+    t1 = Transaction(hash=hash5, amount=10000.0)
+    t2 = Transaction(hash=hash6, amount=20000.0)
+    address1 = '0x65f716f794026d4ff4723141100f0e3c3080e97d'
+    address2 = '0x687b6e85b9b08a33991b6ceeeb1d12df278a9a5b'
+    address3 = '0x17a84ba96a7374de9586a770b4529f57c6b991d6'
+    address4 = '0x422bc251a0582aba25e2dd3b27e80435bc13d402'
+    address5 = '0x2daa35962a6d43eb54c48367b33d0b379c930e5e'
+    address6 = '0x829bd824b016326a401d083b33d092293333a830'
+    a1 = Account(address=address1)
+    a2 = Account(address=address2)
+    c1 = Contract(address=address3)
+    c2 = Contract(address=address4)
+    m1 = Miner(address=address5)
+    m2 = Miner(address=address6)
+    for n in [b1, b2, b3, b4, t1, t2, a1, a2, c1, c2, m1, m2]:
+        n.save()
+    r1 = b1.next_block.connect(b2, {"index": "123", "date": time1})
+    r2 = b3.next_block.connect(b4, {"index": "124", "date": time1})
+    r3 = m1.pack.connect(b1, {"index": "125", "date": time1})
+    r4 = m1.pack.connect(b2, {"index": "126", "date": time1})
+    r5 = m1.pack.connect(b3, {"index": "127", "date": time1})
+    r6 = m2.pack.connect(b4, {"index": "128", "date": time1})
+    r77 = t1.trade.connect(b1, {"index": "129", "date": time1})
+    r7 = t2.trade.connect(b3, {"index": "130", "date": time1})
+    r8 = a1.transfer_out.connect(t1, {"index": "131", "date": time1})
+    r9 = a2.transfer_out.connect(t2, {"index": "132", "date": time1})
+    r10 = t1.transfer_in.connect(a1, {"index": "133", "date": time1})
+    r11 = t1.invoke.connect(c1, {"index": "134", "date": time1})
+    r12 = t2.invoke.connect(c2, {"index": "135", "date": time1})
+    for r in [r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r77]:
+        r.save()
+
+
+if __name__ == '__main__':
+    test()

block/urls.py

+from django.conf.urls import url
+from block.views import *
+from django.urls import path
+
+
+urlpatterns = [
+    path('getBlockList', get_block_list),
+    path('getBlockTradeRank', get_block_trade_rank),
+    path('getBlockMinerRank', get_block_miner_rank)
+]

domain/models.py

-from django.db import models
-from neomodel import StructuredNode, StringProperty
+# charset = utf-8
 
-# Create your models here.
+from neomodel import (
+    StructuredNode,
+    StructuredRel,
+    StringProperty,
+    DateTimeProperty,
+    RelationshipTo
+)
 
 
-class RequestIP(StructuredNode):
-    request_ip = StringProperty(required=True)
+class DomainRel(StructuredRel):
+    index = StringProperty(required=True)
+    date = DateTimeProperty(required=True)  # 请求时间
 
 
-class DnsIP(StructuredNode):
-    dns_ip = StringProperty(required=True)
+class RequestIp(StructuredNode):
+    request_ip = StringProperty(required=True, unique_index=True)
+    province = StringProperty(required=True, index=True)
+    isp = StringProperty(required=True, index=True)
+    request_domain = RelationshipTo('DomainName', 'REQUEST', model=DomainRel)
+
+
+class DnsIp(StructuredNode):
+    dns_ip = StringProperty(required=True, unique_index=True)
+    parse_domain = RelationshipTo('DomainName', 'PARSE', model=DomainRel)
 
 
 class DomainName(StructuredNode):
-    domain_name = StringProperty(required=True)
+    domain_name = StringProperty(required=True, unique_index=True)
+    label = RelationshipTo('Label', 'LABEL', model=DomainRel)
+
+
+class Label(StructuredNode):
+    label_name = StringProperty(required=True, unique_index=True)
+
+
+def gen_node_json(node):
+    if isinstance(node, RequestIp):
+        category, label = 1, "请求IP"
+    elif isinstance(node, DomainName):
+        category, label = 2, "请求域名"
+    elif isinstance(node, DnsIp):
+        category, label = 3, "DNS服务器IP"
+    else:
+        category, label = 4, "标签"
+    return {
+        "id": node.id,
+        "category": category,
+        "label": label,
+        "properties": {}
+    }
+
+
+def gen_node_json2(node):
+    node_type = list(node.labels)[0]
+    if node_type == "RequestIp":
+        category, label = 1, "请求IP"
+    elif node_type == "DomainName":
+        category, label = 2, "请求域名"
+    elif node_type == "DnsIp":
+        category, label = 3, "DNS服务器IP"
+    else:
+        category, label = 4, "标签"
+    return {
+        "id": node.id,
+        "category": category,
+        "label": label,
+        "properties": {}
+    }
 
 
-class Ipv4Address(StructuredNode):
-    ipv4_address = StringProperty(required=True)
+def gen_relation_json2(relation):
+    r_type = relation.type
+    if r_type == "REQUEST":
+        label = "请求"
+    elif r_type == "PARSE":
+        label = "解析"
+    else:
+        label = "标签"
+    return {
+        "id": relation.id,
+        "source": relation.nodes[0].id,
+        "target": relation.nodes[1].id,
+        "label": label
+    }

domain/tests.py

-from django.test import TestCase
+# charset = utf-8
 
-# Create your tests here.
+from datetime import datetime
+from neomodel import db
+from domain.models import Label, RequestIp, DnsIp, DomainName
+
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
+
+
+def test():
+    t1 = datetime(year=2022, month=1, day=1)
+    l1 = Label(label_name="恶意域名")
+    l2 = Label(label_name="境外域名")
+    l1.save()
+    l2.save()
+    req1 = RequestIp(request_ip='127.0.0.1', province='四川', isp='电信')
+    req2 = RequestIp(request_ip='127.0.0.2', province='四川', isp='电信')
+    req1.save()
+    req2.save()
+    d1 = DnsIp(dns_ip='127.0.0.3')
+    d2 = DnsIp(dns_ip='127.0.0.4')
+    d1.save()
+    d2.save()
+    do1 = DomainName(domain_name='www.baidu.com')
+    do2 = DomainName(domain_name='google.cn')
+    do1.save()
+    do2.save()
+    r1 = req1.request_domain.connect(do1, {"index": "123", "date": t1})
+    r2 = req2.request_domain.connect(do2, {"index": "124", "date": t1})
+    r3 = d1.parse_domain.connect(do1, {"index": "125", "date": t1})
+    r4 = d2.parse_domain.connect(do2, {"index": "126", "date": t1})
+    r5 = do1.label.connect(l1, {"index": "127", "date": t1})
+    r6 = do2.label.connect(l2, {"index": "128", "date": t1})
+    for r in [r1, r2, r3, r4, r5, r6]:
+        r.save()
+
+
+def test1():
+    t1 = datetime(year=2022, month=1, day=1).timestamp()
+    t2 = datetime(year=2022, month=1, day=2).timestamp()
+    param = {"ips": ['127.0.0.1', '127.0.0.2']}
+    query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label), (dns:DnsIp)-[par:PARSE]->(d)' \
+            'where r.request_ip in $ips return r, d, l, dns, req, lab, par'
+    items = db.cypher_query(query, param)[0]
+    print(query)
+    print(type(items))
+    for item in items:
+        nodes = item[:4]
+        node = nodes[0]
+        relations = item[4:]
+        r = relations[0]
+        # print(node.labels)
+        # print(node)
+        # node_type = list(getattr(node, "labels"))[0]
+        # print(type(node_type))
+        # print(r)
+        # print(r.id)
+        print((r.type))
+        print(r.nodes[0].id)
+
+
+def test2():
+    t1 = datetime(year=2022, month=1, day=2)
+    req1 = RequestIp(request_ip='127.0.0.10', province='北京', isp='联通')
+    req1.save()
+    d1 = DnsIp(dns_ip='127.0.0.11')
+    d1.save()
+    do1 = DomainName(domain_name='tencent.com')
+    do1.save()
+    l1 = Label.nodes.filter(label_name='恶意域名').first()
+    r1 = req1.request_domain.connect(do1, {"index": "221", "date": t1})
+    r2 = d1.parse_domain.connect(do1, {"index": "299", "date": t1})
+    r3 = do1.label.connect(l1, {"index": "199", "date": t1})
+    r1.save()
+    r2.save()
+    r3.save()
+
+
+def test3():
+    t1 = datetime(year=2022, month=1, day=2)
+    req1 = RequestIp(request_ip='127.0.0.12', province='北京', isp='联通')
+    d1 = DnsIp.nodes.filter(dns_ip='127.0.0.11').first()
+    req1.save()
+    do1 = DomainName.nodes.filter(domain_name='tencent.com').first()
+    r1 = req1.request_domain.connect(do1, {"index": "222", "date": t1})
+    r2 = d1.parse_domain.connect(do1, {"index": "999", "date": t1})
+    r1.save()
+    r2.save()
+
+
+def test4():
+    param = {"ips": ['127.0.0.1', '127.0.0.2', '127.0.0.10', '127.0.0.12']}
+    query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
+            'return l.label_name, count(distinct lab), count(req) order by count(distinct lab) desc'
+    items = db.cypher_query(query, param)[0]
+    for item in items:
+        a, b, c = item
+        print("a=%s, b=%s, c=%s" % (a, b, c))
+
+
+if __name__ == '__main__':
+    test()
+    test2()
+    test3()
+    # test4()

domain/urls.py

+from django.conf.urls import url
+from domain.views import *
+from django.urls import path
+
+
+urlpatterns = [
+    path('getDomainList', get_domain_list),
+    path('getLabelRank', get_label_rank),
+    path('getDomainRank', get_domain_rank),
+    path('getRequestIpRank', get_request_ip_rank),
+    path('getDnsIpRank', get_dns_ip_rank),
+    path('getLabelList', get_label_list)
+]

domain/views.py

-from django.shortcuts import render
+# charset = utf-8
+
+from django.http import JsonResponse
+from neomodel import db
+
+from domain.models import Label, RequestIp, gen_node_json, gen_node_json2, gen_relation_json2
+from utils import tools
+
+
+def get_label_list(request):
+    if request.method == "GET":
+        try:
+            items = Label.nodes.all()
+            labels = []
+
+            i = 1
+            for item in items:
+                labels.append({"id": i, "name": item.label_name})
+                i += 1
+
+            resp = tools.dec_success_resp({"count": len(items), "labels": labels})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_request_ips(label, from_time, end_time):
+    try:
+        from_time = tools.from_str_to_time(from_time)
+        end_time = tools.from_str_to_time(end_time)
+
+        if label:
+            if from_time:
+                param = {"label": label, "from_time": from_time.timestamp(), "end_time": end_time.timestamp()}
+                query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
+                        'where l.label_name = $label and req.date >= $from_time and req.date <= $end_time return r.request_ip'
+            else:
+                param = {"label": label}
+                query = 'match (r:RequestIp)-[]->(d:DomainName)-[]->(l:Label) where l.label_name = $label return r'
+            nodes = db.cypher_query(query, param)[0]
+            ips = [node[0] for node in nodes]
+        else:
+            if from_time:
+                param = {"from_time": from_time.timestamp(), "end_time": end_time.timestamp()}
+                query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[]->(l:Label) ' \
+                        'where req.date >= $from_time and req.date <= $end_time return r.request_ip'
+                nodes = db.cypher_query(query, param)[0]
+                ips = [node[0] for node in nodes]
+            else:
+                nodes = RequestIp.nodes.all()
+                ips = [node.request_ip for node in nodes]
+        return ips
+    except Exception as e:
+        raise e
+
+
+def get_domain_list(request):
+    if request.method == "GET":
+        try:
+            label = request.GET.get("label")
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('end_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            resp_nodes, resp_relations = [], []
+            n_ids, r_ids = [], []
+            ips = get_request_ips(label, from_time, end_time)
+            param = {"ips": ips}
+            query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label), (dns:DnsIp)-[par:PARSE]->(d)' \
+                    'where r.request_ip in $ips return r, d, l, dns, req, lab, par'
+            items = db.cypher_query(query, param)[0]
+            for item in items:
+                nodes = item[:4]
+                relations = item[4:]
+                for n in nodes:
+                    if n.id in n_ids:
+                        continue
+                    resp_nodes.append(gen_node_json2(n))
+                    n_ids.append(n.id)
+                for r in relations:
+                    if r.id in r_ids:
+                        continue
+                    resp_relations.append(gen_relation_json2(r))
+                    r_ids.append(r.id)
+            resp_nodes.sort(key=lambda n: n.get("id"))
+            resp_relations.sort(key=lambda r: r.get("id"))
+
+            resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_label_rank(request):
+    if request.method == "GET":
+        try:
+            label = request.GET.get("label")
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('end_time')
+            sort_type = request.GET.get("sort_type")
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            rank = []
+            ips = get_request_ips(label, from_time, end_time)
+            param = {"ips": ips}
+            if sort_type == 1:
+                query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
+                        'return l.label_name, count(distinct lab), count(req) order by count(distinct lab) desc'
+            else:
+                query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName)-[lab:LABEL]->(l:Label) where r.request_ip in $ips ' \
+                        'return l.label_name, count(distinct lab), count(req) order by count(req) desc'
+            items = db.cypher_query(query, param)[0]
+            i = 1
+            for item in items:
+                name, domain_num, req_count = item
+                rank.append({"id": i, "name": name, "domainNum": domain_num, "requestNum": req_count})
+                i += 1
+
+            resp = tools.dec_success_resp({"count": len(items), "rank": rank})
+            return JsonResponse(resp, safe=False)
+
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_domain_rank(request):
+    if request.method == "GET":
+        try:
+            label = request.GET.get("label")
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('to_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            rank = []
+            ips = get_request_ips(label, from_time, end_time)
+            param = {"ips": ips}
+            query = 'match (r:RequestIp)-[]->(d:DomainName), (dns:DnsIp)-[p:PARSE]->(d) where r.request_ip in $ips ' \
+                    'return d.domain_name, count(p) order by count(p) desc'
+            items = db.cypher_query(query, param)[0]
+            i = 1
+            for item in items:
+                name, parse_num = item
+                rank.append({"id": i, "name": name, "parseNum": parse_num})
+                i += 1
+
+            resp = tools.dec_success_resp({"count": len(items), "rank": rank})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_request_ip_rank(request):
+    if request.method == "GET":
+        try:
+            label = request.GET.get("label")
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('to_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            rank = []
+            ips = get_request_ips(label, from_time, end_time)
+            param = {"ips": ips}
+            query = 'match (r:RequestIp)-[req:REQUEST]->(d:DomainName) where r.request_ip in $ips ' \
+                    'return r.province, r.isp, count(req)'
+            items = db.cypher_query(query, param)[0]
+            i = 1
+            for item in items:
+                province, isp, request_num = item
+                rank.append({"id": i, "province": province, "isp": isp, "requestNum": request_num})
+                i += 1
+
+            resp = tools.dec_success_resp({"count": len(items), "rank": rank})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_dns_ip_rank(request):
+    if request.method == "GET":
+        try:
+            label = request.GET.get("label")
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('to_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            rank = []
+            ips = get_request_ips(label, from_time, end_time)
+            param = {"ips": ips}
+            query = 'match (r:RequestIp)-[]->(d:DomainName), (dns:DnsIp)-[p:PARSE]->(d) where r.request_ip in $ips ' \
+                    'return r.province,r.isp, count(distinct p) order by count(distinct p) desc'
+            items = db.cypher_query(query, param)[0]
+            i = 1
+            for item in items:
+                province, isp, parse_num = item
+                rank.append({"id": i, "province": province, "isp": isp, "parseNum": parse_num})
+                i += 1
+
+            resp = tools.dec_success_resp({"count": len(items), "rank": rank})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
 
-# Create your views here.

knowledge_graph/settings.py

@@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
 from pathlib import Path
+from neomodel import config
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -85,6 +86,8 @@ DATABASES = {
     }
 }
 
+config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687/neo4j'
+
 
 # Password validation
 # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators

knowledge_graph/urls.py

@@ -15,7 +15,12 @@ Including another URLconf
 """
 from django.contrib import admin
 from django.urls import path
+from django.conf.urls import include
+
 
 urlpatterns = [
     path('admin/', admin.site.urls),
+    path('', include('domain.urls')),
+    path('', include('route.urls')),
+    path('', include('block.urls'))
 ]

poetry.lock

@@ -146,6 +146,19 @@ url = "https://mirrors.aliyun.com/pypi/simple"
 reference = "aliyun"
 
 [[package]]
+name = "rest-framework"
+version = "99.99.99"
+description = "This is a PoC for HackerOne"
+category = "main"
+optional = false
+python-versions = ">=2.5"
+
+[package.source]
+type = "legacy"
+url = "https://mirrors.aliyun.com/pypi/simple"
+reference = "aliyun"
+
+[[package]]
 name = "shapely"
 version = "1.7.1"
 description = "Geometric objects, predicates, and operations"
@@ -192,7 +205,7 @@ reference = "aliyun"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "7e3c0debe01a3cb59d1db4be08f5aee72e1e18e42e53a657503474b09cd26142"
+content-hash = "cdb9a355a3f8c79c0138b5da278797f414265479b76c459f0fe7f2e293cb071b"
 
 [metadata.files]
 asgiref = [
@@ -229,6 +242,9 @@ pytz = [
     {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"},
     {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"},
 ]
+rest-framework = [
+    {file = "rest_framework-99.99.99.tar.gz", hash = "sha256:9633e081bcce42c48f898f7134d6666306767547c69b2980f1aef7ede9969dbf"},
+]
 shapely = [
     {file = "Shapely-1.7.1-1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:46da0ea527da9cf9503e66c18bab6981c5556859e518fe71578b47126e54ca93"},
     {file = "Shapely-1.7.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4c10f317e379cc404f8fc510cd9982d5d3e7ba13a9cfd39aa251d894c6366798"},

pyproject.toml

@@ -13,6 +13,7 @@ python = "^3.9"
 django = "^3.2.9"
 django-neomodel = "^0.0.7"
 openpyxl = "^3.0.9"
+rest-framework = "^99.99.99"
 
 [tool.poetry.dev-dependencies]
 

route/models.py

-from django.db import models
-from neomodel import StructuredNode, StringProperty, IntegerProperty
+# charset = utf-8
 
-# Create your models here.
+from neomodel import (
+    StructuredNode,
+    StructuredRel,
+    StringProperty,
+    DateTimeProperty,
+    RelationshipTo
+)
+
+
+class NextIpRel(StructuredRel):
+    index = StringProperty(required=True)
+    date = DateTimeProperty(required=True)
 
 
 class RouteIp(StructuredNode):
-    ip = StringProperty()
-    country = StringProperty()
-    region = StringProperty()
-    city = StringProperty()
-    isp = StringProperty()
-    in_degree = IntegerProperty()
-    out_degree = IntegerProperty()
+    ip = StringProperty(required=True, unique_index=True)
+    country = StringProperty(required=True)
+    province = StringProperty(required=True, index=True)
+    city = StringProperty(required=True, index=True)
+    isp = StringProperty(required=True, index=True)
+    start_ip = StringProperty(required=True, index=True)
+    root_ip = StringProperty(required=True, index=True)
+    next_ip = RelationshipTo('RouteIp', 'NEXT_IP', model=NextIpRel)
 
 
 class FirstIp(RouteIp):
@@ -20,3 +31,40 @@ class FirstIp(RouteIp):
 
 class EndIp(RouteIp):
     pass
+
+
+def gen_node_json(node):
+    if isinstance(node, FirstIp):
+        category, label = 1, "起始路由"
+    elif isinstance(node, EndIp):
+        category, label = 3, "终止路由"
+    else:
+        category, label = 2, "中间路由"
+    return {
+        "id": node.id,
+        "category": category,
+        "label": label,
+        "properties": {
+            "ip": node.ip,
+            "province": node.province,
+            "city": node.city,
+            "isp": node.isp,
+        }
+    }
+
+
+def gen_rel_json(r):
+    s = r.start_node()
+    e = r.end_node()
+    if isinstance(s, FirstIp):
+        label = "首跳"
+    elif isinstance(e, EndIp):
+        label = '尾跳'
+    else:
+        label = '中间跳'
+    return {
+        "id": r.id,
+        "source": s.id,
+        "target": e.id,
+        "label": label,
+    }

route/tests.py

-from django.test import TestCase
+from neomodel import db
+from datetime import datetime
+from route.models import RouteIp, FirstIp, EndIp
 
-# Create your tests here.
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/neo4j')
+
+
+def test():
+    t1 = datetime(year=2022, month=1, day=1)
+    n1 = FirstIp(ip='127.0.0.1', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
+    n2 = RouteIp(ip='127.0.0.2', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
+    n3 = EndIp(ip='127.0.0.3', country='中国', province='四川', city='成都', isp='电信', start_ip='127.0.0.1', root_ip='127.0.0.3')
+    n1.save()
+    n2.save()
+    n3.save()
+    r1 = n1.next_ip.connect(n2, {"index": "123", 'date': t1})
+    r2 = n1.next_ip.connect(n2, {"index": "124", 'date': t1})
+    r3 = n2.next_ip.connect(n3, {"index": "125", 'date': t1})
+    r4 = n2.next_ip.connect(n3, {"index": "126", 'date': t1})
+    r1.save()
+    r2.save()
+    r3.save()
+    r4.save()
+
+
+if __name__ == "__main__":
+    test()

route/urls.py

+from django.conf.urls import url
+from route.views import *
+from django.urls import path
+
+
+urlpatterns = [
+    path('getRouteParam', get_route_param),
+    path('getRouteList', get_route_list),
+    path('getRouteRank', get_route_rank)
+]

route/views.py

-from django.shortcuts import render
+# charset = utf-8
+
+from django.http import JsonResponse
+from neomodel import Q, db
+
+from route.models import FirstIp, EndIp, gen_node_json, gen_rel_json
+from utils import tools
+
+
+def get_route_param(request):
+    if request.method == 'GET':
+        try:
+            province_city_dict = {}
+            nodes = FirstIp.nodes.all()
+            for node in nodes:
+                if not province_city_dict.get(node.province):
+                    province_city_dict[node.province] = []
+                if not province_city_dict.get(node.city):
+                    province_city_dict[node.province].append(node.city)
+
+            provinces = []
+            for province, _cities in province_city_dict.items():
+                cities = []
+                for c in _cities:
+                    cities.append({"cityName": c})
+                provinces.append({"provinceName": province, "cities": cities})
+
+            root_ips = []
+            nodes = EndIp.nodes.all()
+            for node in nodes:
+                root_ips.append(node.root_ip)
+            root_ips = list(set(root_ips))
+
+            resp = tools.dec_success_resp({"provinces": provinces, "rootIp": root_ips})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_first_ips(province, city, isp, root_ip, from_time, end_time):
+    try:
+        from_time = tools.from_str_to_time(from_time)
+        end_time = tools.from_str_to_time(end_time)
+
+        params = {}
+        query = 'match (f:FirstIp)-[n:NEXT_IP]->() '
+        query_extend = []
+        if province:
+            params['province'] = province
+            query_extend.append('f.province = $province ')
+        if city:
+            params['city'] = city
+            query_extend.append('f.city = $city ')
+        if isp:
+            params['isp'] = isp
+            query_extend.append('f.isp = $isp ')
+        if root_ip:
+            params['root_ip'] = root_ip
+            query_extend.append('f.root_ip = $root_ip ')
+        if from_time and end_time:
+            params['from_time'] = from_time.timestamp()
+            params['ent_time'] = end_time.timestamp()
+            query_extend.append('n.date >= $from_time ')
+            query_extend.append('n.date <= $end_time ')
+        if len(query_extend) != 0:
+            query += "where "
+            for i in range(len(query_extend)):
+                if i == 0:
+                    query += query_extend[i]
+                else:
+                    query += 'and ' + query_extend[i]
+        query += 'return f.ip'
+
+        items = db.cypher_query(query, params)[0]
+        ips = [item[0] for item in items]
+        return ips
+    except Exception as e:
+        raise e
+
+
+def get_route_list(request):
+    if request.method == 'GET':
+        try:
+            province = request.GET.get('province')
+            city = request.GET.get('city')
+            isp = request.GET.get('isp')
+            root_ip = request.GET.get('root_ip')
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('end_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            ips = get_first_ips(province, city, isp, root_ip, from_time, end_time)  # 当前遍历到的节点集合
+
+            current_nodes = FirstIp.nodes.filter(ip__in=ips).all()
+            resp_nodes, resp_relations = [], []
+            while current_nodes:
+                tmp_nodes = []
+                tmp_node_ids = []  # 手动去掉重复的node
+                for node in current_nodes:
+                    resp_nodes.append(gen_node_json(node))
+                    next_nodes = node.next_ip.all()
+                    for next_node in next_nodes:
+                        if next_node.id not in tmp_node_ids:
+                            tmp_nodes.append(next_node)
+                            for r in node.next_ip.all_relationships(next_node):
+                                resp_relations.append(gen_rel_json(r))
+                        tmp_node_ids.append(next_node.id)
+                current_nodes = tmp_nodes
+
+            resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_route_rank(request):
+    if request.method == 'GET':
+        try:
+            province = request.GET.get('province')
+            city = request.GET.get('city')
+            isp = request.GET.get('isp')
+            root_ip = request.GET.get('root_ip')
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('end_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            first_node_ips = get_first_ips(province, city, isp, root_ip, from_time, end_time)
+            param = {'start_ips': first_node_ips}
+            query = 'match (n:RouteIp) where n.start_ip in $start_ips with n, size((n)-[]-()) as size order by size desc limit 100 return n.ip, n.province ,n.isp, size'
+            items = db.cypher_query(query, param)[0]
+
+            rank = []
+            for item in items:
+                if len(item) != 0:
+                    rank.append({"ip": item[0], "province": item[1], "isp": item[2], "degree": item[3]})
+
+            resp = tools.dec_success_resp({"rank": rank})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
 
-# Create your views here.

script/admin.py

+from django.contrib import admin
+
+# Register your models here.

script/apps.py

+from django.apps import AppConfig
+
+
+class ScriptConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'script'

script/models.py

+from django.db import models
+
+# Create your models here.

script/tests.py

+from django.test import TestCase
+
+# Create your tests here.

script/views.py

+from django.shortcuts import render
+
+# Create your views here.

utils/tools.py

+from datetime import datetime
+
+
+def from_str_to_time(t: str):
+    if not t:
+        return None
+    return datetime.strptime(t, "%Y-%m-%d")
+
+
+def dec_success_resp(data):
+    return {
+        "resultCode": 200,
+        "message": "success",
+        "data": data
+    }
+
+
+def dec_error_resp(msg):
+    return {
+        "resultCode": 500,
+        "message": msg,
+        "data": None
+    }
+
+