add route

Change-Id: I1131a7b3809edeef1d0ded80a951d6bf5a2e5a94

README.md

@@ -13,4 +13,9 @@
 
 项目采用python3+django+neo4j的架构  
 
-项目采用poetry作为包管理工具,项目具体依赖见poetry.toml
\ No newline at end of file
+项目采用poetry作为包管理工具,项目具体依赖见poetry.toml
+
+
+## 说明
+
+本项目采用neomodel作为python连接neo4j的driver，neomodel具体使用方法见https://neomodel.readthedocs.io/en/latest
\ No newline at end of file

block/models.py

-from neomodel import StructuredNode, IntegerProperty, StringProperty, DateTimeFormatProperty, RelationshipTo, FloatProperty
-
-# Create your models here.
+from neomodel import db
+
+from neomodel import (
+    StructuredNode,
+    IntegerProperty,
+    StringProperty,
+    DateTimeFormatProperty,
+    RelationshipTo,
+    FloatProperty
+)
+
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/block')
 
 
 class Block(StructuredNode):

domain/models.py

-from neomodel import StructuredNode, StringProperty, DateTimeFormatProperty, IntegerProperty, RelationshipTo
+# charset = utf-8
+from neomodel import db
 
-# Create your models here.
+from neomodel import (
+    StructuredNode,
+    StructuredRel,
+    StringProperty,
+    DateTimeProperty,
+    RelationshipTo,
+    UniqueIdProperty
+)
+
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/domain')
+
+
+class DomainRel(StructuredRel):
+    date = DateTimeProperty(required=True)  # 请求时间
 
 
 class RequestIP(StructuredNode):
+    node_id = UniqueIdProperty()
     request_ip = StringProperty(required=True)
-    region = StringProperty(required=True)
+    province = StringProperty(required=True)
     isp = StringProperty(required=True)
-    request_date = DateTimeFormatProperty(required=True, index=True, format='%Y-%m-%d')  # 请求时间
-    request_num = IntegerProperty()  # 请求恶意域名次数
-    request_domain = RelationshipTo('DomainName', 'REQUEST')
+    request_domain = RelationshipTo('DomainName', 'REQUEST', model=DomainRel)
 
 
 class DnsIP(StructuredNode):
+    node_id = UniqueIdProperty()
     dns_ip = StringProperty(required=True, index=True)
-    parse_date = DateTimeFormatProperty(required=True, index=True, format='%Y-%m-%d')  # 解析时间
-    parse_num = IntegerProperty()  # 解析恶意域名次数
-    parse_domain = RelationshipTo('DomainName', 'PARSE')
+    parse_domain = RelationshipTo('DomainName', 'PARSE', model=DomainRel)
 
 
 class DomainName(StructuredNode):
+    node_id = UniqueIdProperty()
     domain_name = StringProperty(required=True, index=True)
     ipv4_address = StringProperty(required=True)
+    label = RelationshipTo('Label', 'Label', model=DomainRel)
 
 
 class Label(StructuredNode):
-    label = StringProperty(index=True)
+    node_id = UniqueIdProperty()
+    label_name = StringProperty(index=True)
+
 

knowledge_graph/settings.py

@@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
 from pathlib import Path
+from neomodel import config
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -78,12 +79,14 @@ WSGI_APPLICATION = 'knowledge_graph.wsgi.application'
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': BASE_DIR / 'db.sqlite3',
-    }
-}
+# DATABASES = {
+#     'default': {
+#         'ENGINE': 'django.db.backends.sqlite3',
+#         'NAME': BASE_DIR / 'db.sqlite3',
+#     }
+# }
+
+config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687'
 
 
 # Password validation

route/models.py

-from neomodel import StructuredNode, StringProperty, IntegerProperty, DateTimeFormatProperty, RelationshipTo
+# charset = utf-8
+from neomodel import db
+
+from neomodel import (
+    StructuredNode,
+    StructuredRel,
+    StringProperty,
+    UniqueIdProperty,
+    DateTimeProperty,
+    RelationshipTo
+)
+
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/route')
+
 
 # Create your models here.
 
+class NextIpRel(StructuredRel):
+    rel_id = StringProperty(required=True)
+    date = DateTimeProperty(required=True)
+
 
 class RouteIp(StructuredNode):
-    ip = StringProperty(required=True, index=True)
+    node_id = UniqueIdProperty()
+    route_ip = StringProperty(required=True, index=True)
     country = StringProperty(required=True)
     province = StringProperty(required=True, index=True)
-    city = StringProperty(required=True)
+    city = StringProperty(required=True, index=True)
     isp = StringProperty(required=True, index=True)
-    route_date = DateTimeFormatProperty(required=True, index=True, format='%Y-%m-%d')
-    degree = IntegerProperty()
-    next_ip = RelationshipTo('RouteIp', 'NEXT_IP')
+    start_ip = StringProperty(required=True, index=True)
+    root_ip = StringProperty(required=True, index=True)
+    next_ip = RelationshipTo('RouteIp', 'NEXT_IP', model=NextIpRel)
 
 
 class FirstIp(RouteIp):
@@ -20,3 +38,41 @@ class FirstIp(RouteIp):
 
 class EndIp(RouteIp):
     pass
+
+
+def gen_node_json(node):
+    if isinstance(node, FirstIp):
+        category, label = 1, "起始路由"
+    elif isinstance(node, EndIp):
+        category, label = 3, "终止路由"
+    else:
+        category, label = 2, "中间路由"
+    return {
+        "id": node.node_id,
+        "category": category,
+        "label": label,
+        "properties": {
+            "ip": node.route_ip,
+            "province": node.province,
+            "city": node.city,
+            "isp": node.isp,
+        }
+    }
+
+
+def gen_rel_json(r):
+    s = r.start_node()
+    e = r.end_node()
+    if isinstance(s, FirstIp):
+        label = "首跳"
+    elif isinstance(e, EndIp):
+        label = '尾跳'
+    else:
+        label = '中间跳'
+    return {
+        "id": r.rel_id,
+        "source": s.node_id,
+        "target": e.node_id,
+        "label": label,
+        "date": r.date
+    }

route/tests.py

-from django.test import TestCase
+from neomodel import db
+from datetime import datetime
+from route.models import RouteIp, FirstIp, EndIp
 
+db.set_connection('bolt://neo4j:neo4j@localhost:7687/route')
 # Create your tests here.
+
+
+def test():
+    t1 = datetime(year=2022, month=1, day=1)
+    n1 = FirstIp(route_ip='127.0.0.1', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
+    n2 = RouteIp(route_ip='127.0.0.2', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
+    n3 = EndIp(route_ip='127.0.0.3', country='中国', province='四川', city='城市', isp='电信', root_ip='127.0.0.3')
+    n1.save()
+    n2.save()
+    n3.save()
+    r1 = n1.next_ip.connect(n2, {'rel_id': '123', 'date': t1})
+    r2 = n1.next_ip.connect(n2, {'rel_id': '124', 'date': t1})
+    r3 = n2.next_ip.connect(n3, {'rel_id': '125', 'date': t1})
+    r4 = n2.next_ip.connect(n3, {'rel_id': '126', 'date': t1})
+    r1.save()
+    r2.save()
+    r3.save()
+    r4.save()
+
+
+if __name__ == "__main__":
+    test()

route/views.py

 from django.http import JsonResponse
-from datetime import datetime, timedelta
 from neomodel import Q
 
-from route.models import RouteIp, FirstIp, EndIp
+from route.models import FirstIp, EndIp, RouteIp, gen_node_json, gen_rel_json
 from utils import tools
 
-# Create your views here.
-
 
 def get_route_param(request):
-    pass
+    if request.method == 'GET':
+        try:
+            province_city_dict = {}
+            nodes = FirstIp.nodes.all()
+            for node in nodes:
+                if not province_city_dict.get(node.province):
+                    province_city_dict[node.province] = []
+                if not province_city_dict.get(node.city):
+                    province_city_dict[node.province].append(node.city)
+
+            provinces = []
+            for province, _cities in province_city_dict.items():
+                cities = []
+                for c in _cities:
+                    cities.append({"cityName": c})
+                provinces.append({"provinceName": province, "cities": cities})
+
+            root_ips = []
+            nodes = EndIp.nodes.all()
+            for node in nodes:
+                root_ips.append(node.root_ip)
+            root_ips = list(set(root_ips))
+
+            resp = tools.dec_success_resp({"provinces": provinces, "rootIp": root_ips})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
+
+
+def get_first_nodes(province, city, isp, root_ip, from_time, end_time):
+    try:
+        from_time = tools.from_str_to_time(from_time)
+        end_time = tools.from_str_to_time(end_time)
+
+        query = FirstIp.nodes
+        if province:
+            query = query.filter(province=province)
+        if city:
+            query = query.filter(city=city)
+        if isp:
+            query = query.filter(isp=isp)
+        if root_ip:
+            query = query.filter(root_ip=root_ip)
+        if from_time and end_time:
+            query = query.next_ip.match(Q(date__gte=from_time) | Q(date__lte=end_time))
+
+        nodes = query.all()
+        return nodes
+    except Exception as e:
+        raise e
 
 
 def get_route_list(request):
     if request.method == 'GET':
         try:
-            first_route = request.GET.get('first_route')
-            end_route = request.GET.get('end_route')
+            province = request.GET.get('province')
+            city = request.GET.get('city')
+            isp = request.GET.get('isp')
+            root_ip = request.GET.get('root_ip')
             from_time = request.GET.get('from_time')
-            to_time = request.GET.get('to_time')
-
-            if from_time and not to_time:
-                return JsonResponse({"Error": "to_time is null"})
-            if not from_time and to_time:
-                return JsonResponse({"Error": "from_time is null"})
-
-            from_time = tools.from_str_to_time(from_time)
-            to_time = tools.from_str_to_time(to_time)
-
-            if first_route:
-                if from_time:
-                    resp = []
-                    first_nodes = FirstIp.nodes.filter(Q(ip=first_route) | Q(route_date__gt=from_time) | Q(route_date__lt=to_time))
-                    for fn in first_nodes:
-                        pass
-            else:
-                pass
+            end_time = request.GET.get('to_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            current_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time)  # 当前遍历到的节点集合
+
+            resp_nodes, resp_relations = [], []
+            while current_nodes:
+                tmp_nodes = []
+                tmp_node_ids = []  # 手动去掉重复的node
+                for node in current_nodes:
+                    resp_nodes.append(gen_node_json(node))
+                    next_nodes = node.next_ip.all()
+                    for next_node in next_nodes:
+                        if next_node.id not in tmp_node_ids:
+                            tmp_nodes.append(next_node)
+                            for r in node.next_ip.all_relationships(next_node):
+                                resp_relations.append(gen_rel_json(r))
+                        tmp_node_ids.append(next_node.id)
+                current_nodes = tmp_nodes
+
+            resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
+            return JsonResponse(resp, safe=False)
         except Exception as e:
-            return JsonResponse({"Error": e})
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
 
 
 def get_route_rank(request):
-    pass
+    if request.method == 'GET':
+        try:
+            province = request.GET.get('province')
+            city = request.GET.get('city')
+            isp = request.GET.get('isp')
+            root_ip = request.GET.get('root_ip')
+            from_time = request.GET.get('from_time')
+            end_time = request.GET.get('to_time')
+
+            if from_time and not end_time:
+                resp = tools.dec_error_resp("end_time is null")
+                return JsonResponse(resp)
+            if not from_time and end_time:
+                resp = tools.dec_error_resp("from_time is null")
+                return JsonResponse(resp)
+
+            first_nodes = get_first_nodes(province, city, isp, root_ip, from_time, end_time)
+            first_node_ips = [node.route_ip for node in first_nodes]
+            all_nodes = RouteIp.nodes.filter()
+
+            resp_nodes, resp_relations = [], []
+            while current_nodes:
+                tmp_nodes = []
+                tmp_node_ids = []  # 手动去掉重复的node
+                for node in current_nodes:
+                    resp_nodes.append(gen_node_json(node))
+                    next_nodes = node.next_ip.all()
+                    for next_node in next_nodes:
+                        if next_node.id not in tmp_node_ids:
+                            tmp_nodes.append(next_node)
+                            for r in node.next_ip.all_relationships(next_node):
+                                resp_relations.append(gen_rel_json(r))
+                        tmp_node_ids.append(next_node.id)
+                current_nodes = tmp_nodes
+
+            resp = tools.dec_success_resp({"nodes": resp_nodes, "relations": resp_relations})
+            return JsonResponse(resp, safe=False)
+        except Exception as e:
+            resp = tools.dec_error_resp(e)
+            return JsonResponse(resp, safe=False)
 

utils/tools.py

@@ -6,3 +6,20 @@ def from_str_to_time(t: str):
         return None
     return datetime.strptime(t, "%Y-%m-%d")
 
+
+def dec_success_resp(data):
+    return {
+        "resultCode": 200,
+        "message": "success",
+        "data": data
+    }
+
+
+def dec_error_resp(msg):
+    return {
+        "resultCode": 500,
+        "message": msg,
+        "data": None
+    }
+
+