[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [ascension] 21/57: fixed serial fetching and added serializ
From: |
gnunet |
Subject: |
[GNUnet-SVN] [ascension] 21/57: fixed serial fetching and added serialization of zone |
Date: |
Sat, 13 Apr 2019 13:32:18 +0200 |
This is an automated email from the git hooks/post-receive script.
ng0 pushed a commit to branch master
in repository ascension.
commit 666f93fac2cece64606f66d1b457ae0a009cd59a
Author: rexxnor <address@hidden>
AuthorDate: Fri Oct 12 11:42:14 2018 +0200
fixed serial fetching and added serialization of zone
---
gnsmigrator/gnsmigrator.py | 91 ++++++++++++++++++++++++++++++----------------
1 file changed, 60 insertions(+), 31 deletions(-)
diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
index f11c211..ba3819e 100644
--- a/gnsmigrator/gnsmigrator.py
+++ b/gnsmigrator/gnsmigrator.py
@@ -23,6 +23,8 @@ from ctypes import c_void_p
from enum import Enum
from dataclasses import dataclass
import multiprocessing
+import gzip
+import pickle
import queue
import re
import sys
@@ -395,12 +397,15 @@ class BaseMigrator():
"""
Extracts the current serial from a given zone
"""
+ if zonename[-1] == '.':
+ zonename = zonename[:-1]
serial = sp.check_output([GNUNET_GNS_COMMAND,
'-t', 'SOA',
'-u', '@.%s' % zonename])
- soapattern = re.compile(r'.+\s(\d+,)\d+,+\d+,\d+,\d+')
- if re.match(soapattern, serial):
- soa_serial = re.match(soapattern, serial)
+ serial = serial.decode()
+ soapattern = re.compile(r'.+\s(\d+),\d+,+\d+,\d+,\d+', re.M)
+ if re.findall(soapattern, serial):
+ soa_serial = re.findall(soapattern, serial)[0]
else:
soa_serial = 0
return soa_serial
@@ -427,11 +432,17 @@ class TLDMigrator(BaseMigrator):
cls.zonegenerator = {}
@classmethod
- def initial_zone_transfer(cls):
+ def initial_zone_transfer(cls, serial):
"""
Transfer and initialize the zone
"""
- cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld)
+ if serial:
+ cls.zonegenerator = dns.query.xfr(cls.transferns,
+ cls.tld,
+ rdtype=dns.rdatatype.IXFR,
+ serial=serial)
+ else:
+ cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld,)
@classmethod
def bootstrap_zone(cls):
@@ -455,33 +466,49 @@ class TLDMigrator(BaseMigrator):
Extract necessary information from Generator
"""
zone = None
+ # And to unzip and unpickle:
try:
- for message in cls.zonegenerator:
- origin = message.origin
- rdclass = message.answer[0].rdclass
- if zone is None:
- if relativize:
- origin = message.origin
- else:
- origin = message.answer[0].name
+ with gzip.open('%s.zone.gz' % cls.tld[:-1], 'rb') as zonefile:
+ zone = pickle.load(zonefile)
+ except Exception:
+ pass
+
+ currentserial = cls.get_current_serial(cls.tld)
+ zoneserial = None
+ if zone:
+ zoneserial = zone[0][2].split(' ')[2].split(',')[0]
+ if currentserial != zoneserial:
+ try:
+ for message in cls.zonegenerator:
+ origin = message.origin
rdclass = message.answer[0].rdclass
- zone = zone_factory(origin, rdclass, relativize=relativize)
- for rrset in message.answer:
- znode = zone.nodes.get(rrset.name)
- if not znode:
- znode = zone.node_factory()
- zone.nodes[rrset.name] = znode
- zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
- rrset.covers, True)
- zrds.update_ttl(rrset.ttl)
- for record in rrset:
- record.choose_relativity(zone.origin, relativize)
- zrds.add(record)
- if check_origin:
- zone.check_origin()
- cls.zone = zone
- except Exception as transferexception:
- print("Error occured during Zone transfer: %s" % transferexception)
+ if zone is None:
+ if relativize:
+ origin = message.origin
+ else:
+ origin = message.answer[0].name
+ rdclass = message.answer[0].rdclass
+ zone = zone_factory(origin, rdclass,
relativize=relativize)
+ for rrset in message.answer:
+ znode = zone.nodes.get(rrset.name)
+ if not znode:
+ znode = zone.node_factory()
+ zone.nodes[rrset.name] = znode
+ zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
+ rrset.covers, True)
+ zrds.update_ttl(rrset.ttl)
+ for record in rrset:
+ record.choose_relativity(zone.origin, relativize)
+ zrds.add(record)
+ if check_origin:
+ zone.check_origin()
+ cls.zone = zone
+ except Exception as transferexception:
+ print("Error occured during Zone transfer: %s" %
transferexception)
+
+ # To zip and pickle:
+ with gzip.open('%s.zone.gz' % cls.tld[:-1], 'wb') as zonefile:
+ pickle.dump(zone, zonefile)
@classmethod
def multithreaded_add_records_to_gns(cls):
@@ -539,6 +566,7 @@ class TLDMigrator(BaseMigrator):
for thread in threads:
thread.join()
+
def main():
"""
Initializes object and handles arguments
@@ -560,7 +588,8 @@ def main():
if tld and transferns:
migrator = TLDMigrator(tld, transferns)
- migrator.initial_zone_transfer()
+ serial = migrator.get_current_serial(tld)
+ migrator.initial_zone_transfer(serial)
migrator.bootstrap_zone()
migrator.mirror_zone()
migrator.multithreaded_add_records_to_gns()
--
To stop receiving notification emails like this one, please contact
address@hidden
- [GNUnet-SVN] [ascension] 12/57: bumped version to 0.1.0, (continued)
- [GNUnet-SVN] [ascension] 12/57: bumped version to 0.1.0, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 16/57: fixed special cases with GNS2DNS, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 14/57: reworked the incremental zone transfer, added notice for bind, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 11/57: added ttl of SOA to zone for later implementation of timer, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 05/57: initial prototype, bootstrapping zones and zone transfer, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 24/57: fixed zone serialisation, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 17/57: updated requirements and README, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 36/57: some experimental fixes, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 19/57: fixed some minor bugs, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 15/57: created baseclass and separated small from big zones, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 21/57: fixed serial fetching and added serialization of zone,
gnunet <=
- [GNUnet-SVN] [ascension] 13/57: added zone merging of full and incremental zones, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 29/57: added warnings to logging if records failed to be added, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 27/57: unstable version, port specification possible, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 02/57: Added LICENSE, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 18/57: refactored adding of records, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 08/57: added GNS2DNS support and rudimentary Unittests (incomplete), gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 23/57: fixed non existing file, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 42/57: added dnscurve detection and log it, fix ttl bug with hierarchy, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 07/57: fixes, added a few tests, updated README and requirements, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 10/57: added incremental zone transfer logic, gnunet, 2019/04/13