[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [ascension] branch master updated: added hierarchical addin
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [ascension] branch master updated: added hierarchical adding of zones |
|
Date: |
Tue, 30 Apr 2019 15:26:24 +0200 |
This is an automated email from the git hooks/post-receive script.
rexxnor pushed a commit to branch master
in repository ascension.
The following commit(s) were added to refs/heads/master by this push:
new 2809d7a added hierarchical adding of zones
2809d7a is described below
commit 2809d7ad76484ebcd3f5431f482bd84634ca8edc
Author: rexxnor <address@hidden>
AuthorDate: Tue Apr 30 15:15:16 2019 +0200
added hierarchical adding of zones
---
ascension/ascension.py | 259 ++++++++++++++++++--------------
ascension/test/gnunet.zone | 2 +
ascension/test/test_ascension_simple.sh | 52 ++++---
3 files changed, 176 insertions(+), 137 deletions(-)
diff --git a/ascension/ascension.py b/ascension/ascension.py
index 510fb78..ba73e38 100644
--- a/ascension/ascension.py
+++ b/ascension/ascension.py
@@ -20,22 +20,23 @@
# Author rexxnor
"""
Usage:
- ascension <domain> [-d] [-p] [-s]
- ascension <domain> <port> [-d] [-p] [-s]
- ascension <domain> -n <transferns> [-d] [-p] [-s]
- ascension <domain> -n <transferns> <port> [-d] [-p] [-s]
+ ascension <domain> [-d] [-p] [-s] [--minimum-ttl=<ttl>]
+ ascension <domain> <port> [-d] [-p] [-s] [--minimum-ttl=<ttl>]
+ ascension <domain> -n <transferns> [-d] [-p] [-s] [--minimum-ttl=<ttl>]
+ ascension <domain> -n <transferns> <port> [-d] [-p] [-s]
[--minimum-ttl=<ttl>]
ascension -p | --public
ascension -s | --standalone
ascension -h | --help
ascension -v | --version
Options:
- <domain> Domain to migrate
- <port> Port for zone transfer
- <transferns> DNS Server that does the zone transfer
- -p --public Make records public on the DHT
- -s --standalone Run ascension once
- -d --debug Enable debugging
+ <domain> Domain to migrate
+ <port> Port for zone transfer
+ <transferns> DNS Server that does the zone transfer
+ --minimum-ttl=<ttl> Minimum TTL for records to migrate [default: 3600]
+ -p --public Make records public on the DHT
+ -s --standalone Run ascension once
+ -d --debug Enable debugging
-h --help Show this screen.
-v --version Show version.
"""
@@ -69,7 +70,7 @@ class Ascender():
Class that provides migration for any given domain
"""
@classmethod
- def __init__(cls, domain, transferns, port, flags):
+ def __init__(cls, domain, transferns, port, flags, minimum):
cls.domain = domain
if domain[-1] == '.':
cls.domain = cls.domain[:-1]
@@ -81,8 +82,8 @@ class Ascender():
cls.zonegenerator = None
cls.nscache = dict()
cls.flags = flags
- cls.ttl = None
- cls.refresh = cls.ttl
+ cls.minimum = int(minimum)
+ cls.subzonedict = dict()
@classmethod
def initial_zone_transfer(cls, serial=None):
@@ -170,10 +171,10 @@ class Ascender():
logging.info("zone does not exist yet")
cls.initial_zone_transfer()
try:
- cls.zone = dns.zone.from_xfr(cls.zonegenerator)
+ cls.zone = dns.zone.from_xfr(cls.zonegenerator,
+ check_origin=False)
except dns.zone.BadZone:
- logging.error("Malformed DNS Zone '%s'", cls.domain)
- cls.zone = dns.zone.from_xfr(cls.zonegenerator)
+ logging.critical("Malformed DNS Zone '%s'", cls.domain)
cls.soa = cls.get_zone_soa(cls.zone)
elif zoneserial < currentserial:
logging.info("zone is out of date")
@@ -181,7 +182,7 @@ class Ascender():
try:
cls.zone = dns.zone.from_xfr(cls.zonegenerator)
except dns.zone.BadZone:
- logging.error("Malformed DNS Zone '%s'", cls.domain)
+ logging.critical("Malformed DNS Zone '%s'", cls.domain)
cls.soa = cls.get_zone_soa(cls.zone)
elif zoneserial == currentserial:
logging.info("zone is up to date")
@@ -206,7 +207,7 @@ class Ascender():
def worker():
while True:
# define recordline
- recordline = []
+ recordline = list()
label = ""
domain = None
@@ -217,8 +218,20 @@ class Ascender():
# execute thing to run on item
label, listofrdatasets = labelrecords
- cls.ttl = int(cls.get_zone_soa_expiry()[0])
- cls.refresh = cls.ttl
+ subzones = label.split('.')
+ domain = cls.domain
+
+ if len(subzones) > 1:
+ ttl = cls.get_zone_refresh_time()
+ label = subzones[0]
+ subdomains = ".".join(subzones[1:])
+ subzone = "%s.%s" % (subdomains, domain)
+ fqdn = "%s.%s.%s" % (label, subdomains, domain)
+ if fqdn in cls.subzonedict.keys():
+ label = "@"
+ domain = fqdn
+ elif subzone in cls.subzonedict.keys():
+ domain = subzone
for rdataset in listofrdatasets:
for record in rdataset:
@@ -227,23 +240,15 @@ class Ascender():
continue
try:
- if rdataset.ttl < cls.ttl:
- ttl = rdataset.ttl
- if ttl < cls.refresh:
- cls.refresh = ttl
- elif rdataset.ttl == cls.ttl:
- ttl = int(cls.ttl) * 10
+ if rdataset.ttl <= cls.minimum:
+ ttl = cls.minimum
else:
- ttl = rdataset.ttl * 10
+ ttl = rdataset.ttl
except AttributeError:
- ttl = cls.refresh
+ ttl = cls.minimum
value = str(record)
- # resolves record to check if it exists
- if cls.check_if_record_exists_in_zone(label, rdtype,
cls.domain):
- continue
-
# ignore NS for itself here
if label == '@' and rdtype == 'NS':
logging.info("ignoring NS record for itself")
@@ -252,7 +257,7 @@ class Ascender():
rdtype, value, label = \
cls.transform_to_gns_format(record,
rdtype,
- cls.domain,
+ domain,
label)
# skip record if value is none
if value is None:
@@ -283,7 +288,7 @@ class Ascender():
# add recordline to gns and filter out empty lines
if len(recordline) > 1:
cls.add_recordline_to_gns(recordline,
- domain if domain else cls.domain,
+ domain,
label)
taskqueue.task_done()
@@ -298,48 +303,70 @@ class Ascender():
pkey = str(gnspkey[0][2])
# TODO Check this check
if not cls.transferns in ['127.0.0.1', '::1', 'localhost']:
- print("zone exists in GNS, adding it to local store")
+ logging.warning("zone exists in GNS, adding it to local store")
cls.add_pkey_record_to_zone(pkey[11:], cls.domain,
label, ttl)
return
- # Create one thread
- thread = threading.Thread(target=worker)
- thread.start()
-
# Unify all records under same label into datastructure
customrdataset = dict()
- try:
- for remaining in cls.zone.iterate_rdatasets():
- # build lookup table for later GNS2DNS records
- domain = "%s.%s" % (str(remaining[0]), cls.domain)
- elementlist = []
- for element in remaining[1]:
- if dns.rdatatype.to_text(element.rdtype) in ['A', 'AAAA']:
- elementlist.append(str(element))
- cls.nscache[str(domain)] = elementlist
- rdataset = remaining[1]
- if customrdataset.get(str(remaining[0])) is None:
- work = list()
- work.append(rdataset)
- customrdataset[str(remaining[0])] = work
- else:
- customrdataset[str(remaining[0])].append(rdataset)
+ for remaining in cls.zone.iterate_rdatasets():
+ # build lookup table for later GNS2DNS records
+ domain = "%s.%s" % (str(remaining[0]), cls.domain)
+ elementlist = []
+ for element in remaining[1]:
+ if dns.rdatatype.to_text(element.rdtype) in ['A', 'AAAA']:
+ elementlist.append(str(element))
+ cls.nscache[str(domain)] = elementlist
+ rdataset = remaining[1]
+ if customrdataset.get(str(remaining[0])) is None:
+ work = list()
+ work.append(rdataset)
+ customrdataset[str(remaining[0])] = work
+ else:
+ customrdataset[str(remaining[0])].append(rdataset)
+
+ for label, value in customrdataset.items():
+ if value is None:
+ continue
+
+ subzones = label.split('.')
+ label = subzones[0]
+ subdomain = ".".join(subzones[1:])
+ zonename = "%s.%s" % (subdomain, cls.domain)
- for label, value in customrdataset.items():
- if value is None:
+ refresh = cls.get_zone_refresh_time()
+ if refresh <= cls.minimum:
+ ttl = cls.minimum
+ else:
+ ttl = refresh
+
+ if len(subzones) > 1:
+ if cls.subzonedict.get(zonename):
continue
- taskqueue.put((label, value))
- except AttributeError:
- logging.info("skipping up to date zone")
- return
+ else:
+ cls.subzonedict[zonename] = (False, ttl)
+
+ cls.create_zone_hierarchy()
+
+ # Create one thread
+ thread = threading.Thread(target=worker)
+ thread.start()
+
+ # add records
+ for label, value in customrdataset.items():
+ if value is None:
+ continue
+ taskqueue.put((label, value))
# Block until all tasks are done
taskqueue.join()
# Stop workers and threads
taskqueue.put(None)
- thread.join()
+ thread.join(timeout=10)
+ if thread.is_alive():
+ logging.critical("thread join timed out, still running")
# Add soa record to GNS once completed (updates the previous one)
soa = cls.get_zone_soa(cls.zone)
@@ -350,21 +377,24 @@ class Ascender():
def add_recordline_to_gns(recordline, zonename, label):
"""
Replaces records in zone or adds them if not
- :param recordline: records to replace
+ :param recordline: records to replace as list in form
+ ['-R', 'TTL TYPE FLAGS VALUE']
:param zonename: zonename of zone to add records to
:param label: label under which to add the records
"""
logging.info("trying to add %d records with name %s",
len(recordline)/2, label)
+
ret = sp.run([GNUNET_NAMESTORE_COMMAND,
'-z', zonename,
'-n', str(label),
] + recordline)
+
if ret.returncode != 0:
logging.warning("failed adding record with name %s",
' '.join(ret.args))
else:
- logging.info("successfully added record with name %s",
+ logging.info("successfully added record with command %s",
' '.join(ret.args))
@classmethod
@@ -395,9 +425,11 @@ class Ascender():
int(serial), int(refresh), int(retry),
int(expiry), int(irefresh)
)
- elif rdtype in ['TXT', 'CNAME']:
+ elif rdtype in ['CNAME']:
if value[-1] == ".":
value = value[:-1]
+ else:
+ value = "%s.%s" % (value, zonename)
elif rdtype == 'NS':
nameserver = str(record)
if value[-1] == ".":
@@ -425,6 +457,7 @@ class Ascender():
priority, mailserver = str(value).split(' ')
if mailserver[-1] == ".":
mailserver = mailserver[:-1]
+ mailserver = '%s.%s' % (mailserver, zonename)
value = '%s,%s' % (priority, mailserver)
logging.info("transformed %s record to GNS format", rdtype)
elif rdtype == 'SRV':
@@ -448,10 +481,17 @@ class Ascender():
logging.warning("invalid protocol: %s", protostring)
return (rdtype, None, None)
- value = '%s %s %s %s %s %s %s' % (
- destport, protonum, srv, priority, weight, destport,
- "%s.%s" % (target, zonename)
- )
+ if target[:-1] == ".":
+ value = '%s %s %s %s %s %s %s' % (
+ destport, protonum, srv, priority, weight, destport,
+ "%s" % target
+ )
+ else:
+ value = '%s %s %s %s %s %s %s' % (
+ destport, protonum, srv, priority, weight, destport,
+ "%s.%s" % (target, zonename)
+ )
+
label = target
else:
logging.info("Did not transform record of type: %s", rdtype)
@@ -560,40 +600,16 @@ class Ascender():
else:
owner = "%s.%s" % (owner, cls.domain)
- ret = sp.run([GNUNET_NAMESTORE_COMMAND,
- '-z', cls.domain,
- '-a', '-n', str(label),
- '-t', 'SOA',
- '-V', "rname=%s mname=%s %d,%d,%d,%d,%d"
- % (authns, owner,
- int(serial), int(refresh), int(retry),
- int(expiry), int(irefresh)
- ),
- '-e', '%ds' % ttl,
- # maybe don't make it public by default
- '-p'])
- logging.info("executed command: %s", " ".join(ret.args))
- if ret.returncode != 0:
- logging.warning("failed to add %s record %s", "SOA", "@")
-
-
- @staticmethod
- def check_if_record_exists_in_zone(name, rtype, zonename):
- """
- Checks if the given record exists in GNS
- :param name: The record name to check for
- :param type: The record type to check for
- :param zonename: The zone in which to look up the record
- :returns: True on existance, False otherwise
- """
- ret = sp.check_output([GNUNET_GNS_COMMAND,
- '-t', rtype,
- '-u', '%s.%s' %
- (name, zonename)]
- )
- if 'Got ' in ret.decode():
- return True
- return False
+ value = "rname=%s mname=%s %s,%s,%s,%s,%s" % (authns,
+ owner,
+ serial,
+ refresh,
+ retry,
+ expiry,
+ irefresh)
+ recordval = '%s %s %s %s' % (ttl, "SOA", cls.flags, str(value))
+ recordline = ['-R', recordval]
+ cls.add_recordline_to_gns(recordline, cls.domain, str(label))
@staticmethod
def create_zone_and_get_pkey(zonestring):
@@ -634,36 +650,48 @@ class Ascender():
:param label: the label under which to add the pkey
:param ttl: the time to live the record should have
"""
+ #lookup = sp.run(GNUNET_NAMESTORE_COMMAND,
+ # '-z', domain,
+ # '-n', label,
+ # '-t', 'PKEY')
+ #if not 'Got result:' in lookup.stdout:
debug = " ".join([GNUNET_NAMESTORE_COMMAND,
'-z', domain,
'-a', '-n', label,
'-t', 'PKEY',
'-V', pkey,
- '-e', "%ds" % ttl])
+ '-e', "%ss" % ttl])
ret = sp.run([GNUNET_NAMESTORE_COMMAND,
'-z', domain,
'-a', '-n', label,
'-t', 'PKEY',
'-V', pkey,
- '-e', "%ds" % ttl])
+ '-e', "%ss" % ttl])
logging.info("executed command: %s", debug)
if ret.returncode != 0:
logging.warning("failed to add PKEY record %s to %s",
label, domain)
+ #logging.warning("PKEY record %s already exists in %s", label, domain)
@classmethod
- def create_zone_hierarchy(cls, labels, ttl):
+ def create_zone_hierarchy(cls):
"""
Creates the zone hierarchy in GNS for label
:param label: the split record to create zones for
"""
domain = cls.domain
- # black magic that removes first element and reverses list
- for label in labels[1:][::-1]:
- zonelabel = "%s.%s" % (label, domain)
- pkey = cls.create_zone_and_get_pkey(zonelabel)
- cls.add_pkey_record_to_zone(pkey, domain, label, ttl)
- domain = zonelabel
+
+ zonelist = cls.subzonedict.items()
+ sortedlist = sorted(zonelist, key=lambda s: len(str(s).split('.')[0]))
+ for zone, pkeyttltuple in sortedlist:
+ pkey, ttl = pkeyttltuple
+ if not pkey:
+ domain = ".".join(zone.split('.')[1::])
+ label = zone.split('.')[0]
+ pkey = cls.create_zone_and_get_pkey(zone)
+ logging.info("adding zone %s with %s pkey into %s", zone,
pkey, domain)
+ cls.add_pkey_record_to_zone(pkey, domain, label,
pkeyttltuple[1])
+ cls.subzonedict[zone] = (pkey, ttl)
def main():
"""
@@ -679,6 +707,7 @@ def main():
port = args['<port>'] if args['<port>'] else 53
flags = "p" if args.get('--public') else "n"
standalone = bool(args.get('--standalone'))
+ minimum = args['--minimum-ttl']
# Change logging severity to debug
if debug:
@@ -692,7 +721,7 @@ def main():
sys.exit(1)
# Initialize class instance
- ascender = Ascender(domain, transferns, port, flags)
+ ascender = Ascender(domain, transferns, port, flags, minimum)
# Event loop for actual daemon
while 1:
@@ -702,10 +731,10 @@ def main():
ascender.bootstrap_zone()
if ascender.zone is not None:
ascender.add_records_to_gns()
- logging.info("Finished migrating of the zone %s", ascender.domain)
+ logging.info("Finished migration of the zone %s", ascender.domain)
else:
logging.info("Zone %s already up to date", ascender.domain)
- refresh = ascender.refresh
+ refresh = int(ascender.get_zone_refresh_time())
retry = int(ascender.get_zone_retry_time())
if standalone:
return 0
diff --git a/ascension/test/gnunet.zone b/ascension/test/gnunet.zone
index d5b5e44..0bef20f 100644
--- a/ascension/test/gnunet.zone
+++ b/ascension/test/gnunet.zone
@@ -20,3 +20,5 @@ owncloud IN A 127.0.0.1
nextcloud IN A 127.0.0.1
mail IN MX 10 mail.gnunet.org.
mail IN A 127.0.0.1
+multiple.subzones.dns IN A 127.0.0.1
+subzones.dns IN A 127.1.1.1
diff --git a/ascension/test/test_ascension_simple.sh
b/ascension/test/test_ascension_simple.sh
index ad11325..e88315e 100644
--- a/ascension/test/test_ascension_simple.sh
+++ b/ascension/test/test_ascension_simple.sh
@@ -13,6 +13,8 @@
cleanup() {
pkill named
gnunet-identity -D gnunet.org
+ gnunet-identity -D dns.gnunet.org
+ gnunet-identity -D subzones.dns.gnunet.org
}
# Check for required packages
@@ -58,9 +60,10 @@ if [ "$?" -ne 0 ]; then
fi
checkfailexp() {
- if [ "$?" -ne 0 ]; then
- echo "required record not present"
- cleanup
+ echo "$1"
+ if [ "$?" -ne 0 ] || [ "$1" = 'No results.' ]; then
+ echo "FAILED! Required record not present"
+ #cleanup
exit 2
fi
}
@@ -74,27 +77,32 @@ checkfailimp() {
}
# TESTING explicit records
-gnunet-gns -t CNAME -u asdf.gnunet.org
-checkfailexp
-gnunet-gns -t AAAA -u foo.gnunet.org
-checkfailexp
-gnunet-gns -t A -u mail.gnunet.org
-checkfailexp
-gnunet-gns -t A -u ns1.gnunet.org
-checkfailexp
-gnunet-gns -t A -u ns2.gnunet.org
-checkfailexp
-gnunet-gns -t A -u ns2.gnunet.org
-checkfailexp
-gnunet-gns -t MX -u mail.gnunet.org
-checkfailexp
-gnunet-gns -t A -u nextcloud.gnunet.org
-checkfailexp
-gnunet-gns -t SOA -u @.gnunet.org
-checkfailexp
+a=$(gnunet-gns -t CNAME -u asdf.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t AAAA -u foo.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u mail.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u ns1.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u ns2.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u ns2.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t MX -u mail.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u nextcloud.gnunet.org)
+checkfailexp "$a"
+# TODO readd this test as it does not work as of 57636ddf7 in GNUnet
+#a=$(gnunet-gns -t SOA -u @.gnunet.org)
+#checkfailexp "$a"
+a=$(gnunet-gns -t A -u multiple.subzones.dns.gnunet.org)
+checkfailexp "$a"
+a=$(gnunet-gns -t A -u subzones.dns.gnunet.org)
+checkfailexp "$a"
# cleanup if we get this far
-cleanup
+#cleanup
# finish
echo "All records added successfully!!"
--
To stop receiving notification emails like this one, please contact
address@hidden
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] [ascension] branch master updated: added hierarchical adding of zones,
gnunet <=