new importer, for "IPsum" aggregated list

main/management/commands/import_from_ipsum.py

+#
+# -*- coding: utf-8
+# vim: set fileencoding=utf-8
+
+# system
+from ipaddress import ip_address
+import requests
+
+# django
+from django.core.management.base import BaseCommand, CommandError
+
+# projet
+from main.models import Reason, Server, IPWhiteList, ReportBadIP
+
+feed_url = "https://raw.githubusercontent.com/stamparm/ipsum/master/ipsum.txt"
+# We only import those with such a score, at least
+MINIMUM_NUMBER = 2
+
+class Command(BaseCommand):
+    help = """
+    Aggregated feed on github.
+    See https://github.com/stamparm/ipsum
+    """
+    def handle(self, *args, **options):
+
+        self.reason = Reason.objects.get(name="Imported")
+        self.server, _ = Server.objects.get_or_create(name="IPsum", user=None)
+
+        # Fetch data
+        # don't use stream=True: the file is not that big, but we are very slow to process it
+        # and the connection will break/timeout
+        r = requests.get(feed_url)
+        for line in r.iter_lines():
+            line = line.decode('ascii')
+            # ignore comments, blank lines
+            if line.startswith("#") or len(line)==0: continue
+            # Format, the ip appears in <number> different sources/feeds
+            ip, number = line.split()
+            number=int(number)
+            if number<MINIMUM_NUMBER: break
+            self.addoneip(ip)
+
+    def addoneip(self, ip):
+        if IPWhiteList.check_against(ip_address(ip)):
+            self.stdout.write(self.style.WARNING('%s: IP whitelisted, no report created' % ip))
+            return
+        # Report ips
+        if ReportBadIP.create_or_update_report(self.server, ip, self.reason):
+            self.stdout.write(self.style.SUCCESS('%s: new report created' % ip))
+        else:
+            self.stdout.write(self.style.WARNING('%s: report updated' % ip))
+        
+# vim: ai ts=4 sts=4 et sw=4
+