Merge branch 'sql_import_script' into 'master'

Sql import script

See merge request !970

scripts/dns_sql2zf.py

+#!/usr/bin/python3
+
+# Copyright (C) 2019 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+# requirements:
+# sudo apt install libmysqlclient-dev
+# pip3 install sqlobject configparser mysqlclient argparse
+
+from sqlobject import *
+import configparser
+import argparse
+
+import os
+import string
+import sys
+import time
+import re
+from subprocess import DEVNULL, PIPE, Popen
+
+# globals
+connection = None
+soa_serial = int(time.time())
+config = configparser.ConfigParser()
+fix_absolute = False
+storage = os.getcwd()
+knotc_binary = "knotc"
+knotc_socket = None
+slave_mode = False
+conf_txn_open = False
+knotc_zone_reload = []
+knotc_timeout = 10
+
+class Domains(SQLObject):
+    # id = IntCol() # implicitly there
+    name = StringCol()
+    master = StringCol()
+    last_check = IntCol()
+    type = StringCol()
+    notified_serial = IntCol()
+    account = StringCol()
+
+class Records(SQLObject):
+    # id = IntCol() # implicitly there
+    domain = ForeignKey('Domains')
+    name = StringCol()
+    type = StringCol()
+    content = StringCol()
+    ttl = IntCol()
+    prio = IntCol()
+    change_date = IntCol()
+    ordername = StringCol()
+    auth = IntCol()
+
+class Changes(SQLObject):
+    # id = IntCol() # implicitly there
+    domain = ForeignKey('Domains')
+    type = IntCol() # -1 .. zone removed; 0 .. zone modified; 1 .. zone added
+
+def remove_dot(s):
+    return s[:-1] if s[-1] == '.' else s
+
+def fix_abs(name):
+    return remove_dot(name) + '.' if fix_absolute else name
+
+def domain_get_records(domain, txn):
+    if str(domain).isdigit():
+        return Records.select(Records.q.domain == domain, connection=txn)
+    else:
+        dn = remove_dot(domain)
+        return Records.select(AND(Domains.q.id == Records.q.domain, Domains.q.name == dn), connection=txn)
+
+def domain_id2name(domain, txn):
+    return Domains.select(Domains.q.id == domain, connection=txn)[0].name
+
+def get_config(key, default_val):
+    global config
+    return int(config['DEFAULT'][key]) if key in config['DEFAULT'] and config['DEFAULT'][key] is not None else default_val
+
+def get_soa_params():
+    refresh = get_config("soa-refresh-default", 10800)
+    retry = get_config("soa-retry-default", 3600)
+    expire = get_config("soa-expire-default", 604800)
+    minttl = get_config("soa-minimum-ttl", 3600)
+    return (refresh, retry, expire, minttl)
+
+def soa_content(db_content):
+    global soa_serial
+
+    (nameserver, contact, fake_serial) = db_content.split()
+    ns = fix_abs(nameserver)
+    co = fix_abs(contact.replace("@", "."))
+
+    (refresh, retry, expire, minttl) = get_soa_params()
+    return ("%s %s %d %d %d %d %d" % (ns, co, soa_serial, refresh, retry, expire, minttl))
+
+def zone_storage(zone):
+    global storage
+    return os.path.join(storage, "%s.zone" % remove_dot(zone))
+
+def knotc_single(*args):
+    global knotc_binary
+    global knotc_socket
+    global knotc_timeout
+
+    cmd = [ knotc_binary, "-s", knotc_socket, "-t", str(knotc_timeout) ] + list(args)
+    p = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
+    (stdout, stderr) = p.communicate()
+    if p.returncode != 0:
+        raise Exception("error: knotc %s failed: '%s'" % (str(args), stderr))
+
+def zone_template(zone):
+    # this function is intended to be patched by user's bussiness logic
+    return None
+
+# param type:
+#   -1 ... remove this zone
+#    0 ... mark this zone as modified (fallback to 2 if not exists)
+#    1 ... add zone (fallback to 0 if exists already)
+#    2 ... add zone no fallback (ignore if exists already)
+def knotc_send(type, zone):
+    global slave_mode
+    global conf_txn_open
+    global knotc_zone_reload
+    if type == 0:
+        try:
+            # ensure by zone-status that the zone exists
+            knotc_single("zone-status", zone)
+
+            knotc_zone_reload.append(zone)
+        except:
+            knotc_send(2, zone)
+    else:
+        try:
+            if not conf_txn_open:
+                knotc_single("conf-begin")
+                conf_txn_open = True
+            if type > 0:
+                try:
+                    knotc_single("conf-set", "zone[%s]" % zone)
+                    knotc_single("conf-set", "zone[%s].file" % zone, zone_storage(zone))
+                    template = zone_template(remove_dot(zone))
+                    if template is not None:
+                        knotc_single("conf-set", "zone[%s].template" % zone, template)
+                except:
+                    if type > 1:
+                        pass
+                    else:
+                        knotc_send(0, zone)
+            else:
+                knotc_single("conf-unset", "zone[%s]" % zone)
+                try:
+                    knotc_zone_reload.remove(zone)
+                except:
+                    pass
+        except:
+            knotc_single("conf-abort")
+            conf_txn_open = False
+            raise
+
+def print_record(record, outfile):
+    t = record.type.upper()
+    if t == 'SOA':
+        content = soa_content(record.content)
+    elif t == 'MX' or t == 'SRV':
+        content = "%d %s" % (record.prio, record.content)
+    elif t == 'TXT' or t == 'SPF':
+        content = "\"%s\"" % record.content
+    else:
+        content = record.content
+
+    if t in ('NS', 'MX', 'CNAME', 'DNAME', 'SRV', 'PTR'):
+        content = fix_abs(content)
+
+    record = ("%s. %d %s %s\n" % (record.name, record.ttl, t, content))
+    outfile.write(record)
+
+def print_domain(domain, change_type = 0, txn = None):
+    global knotc_socket
+    global slave_mode
+    dn = domain_id2name(domain, txn) if str(domain).isdigit() else domain
+    if not slave_mode:
+        file_name = zone_storage(dn)
+        tmp_name = "%s.tmp" % file_name
+        f = open(tmp_name, "w")
+        for r in domain_get_records(domain, txn):
+            print_record(r, f)
+        f.close()
+        os.rename(tmp_name, file_name)
+    if knotc_socket is not None:
+        knotc_send(change_type, dn)
+    print("Updated zone %s" % dn)
+
+def domain_from_change(change, txn):
+    global knotc_socket
+    if change.type >= 0:
+        print_domain(change.domain.name, change.type, txn)
+    else:
+        dn = change.domain.name
+        try:
+            os.remove(zone_storage(dn))
+        except:
+            print("Warning: failed to delete zonefile for %s" % dn, file=sys.stderr)
+        if knotc_socket is not None:
+            knotc_send(change.type, dn)
+        else:
+            print("Warning: removed zone '%s', but unspecified knotc socket." % dn, file=sys.stderr)
+
+def process_changes(startwith):
+    global connection
+    processed = []
+    try:
+        txn = connection.transaction()
+        for ch in Changes.select(Changes.q.id > startwith, connection=txn):
+            domain_from_change(ch, txn)
+            processed.append(ch.id)
+        txn.commit()
+    finally:
+        if len(processed) > 0:
+            print("Processed up to change_id %d" % processed[-1])
+
+def process_all():
+    global connection
+    txn = connection.transaction()
+    for d in Domains.select(connection=txn):
+        print_domain(d.id, txn = txn)
+    txn.commit()
+
+def read_config_file(filename):
+    global config
+
+    with open(filename, 'r') as f:
+        fcontent = '[DEFAULT]\n' + f.read()
+
+    config.read_string(fcontent)
+
+def main():
+    global storage
+    global knotc_socket
+    global soa_serial
+    global fix_absolute
+    global connection
+    global slave_mode
+    global conf_txn_open
+    global knotc_zone_reload
+    global knotc_timeout
+
+    argp = argparse.ArgumentParser(prog='dns_sql2zf', description="Export DNS records from Mysql or Postgres DB into zonefile.", epilog="(C) CZ.NIC, GPLv3")
+    argp.add_argument(dest='domains', metavar='zone', nargs='*', help='Zone to be exported.')
+    argp.add_argument('--db', dest='dburi', metavar='DB_URI', nargs=1, required=True, help='URI of database to export from (example: mysql://user:password@127.0.0.1/powerdns_db).')
+    argp.add_argument('--storage', dest='storage', metavar='path', nargs=1, help='Storage for the generated zonefile (otherwise current dir).')
+    argp.add_argument('--all', dest='all', action='store_true', help="Export all zones.")
+    argp.add_argument('--confile', dest='confile', metavar='file', nargs=1, help='PowerDNS configfile to obtain SOA parameters (otherwise defaults).')
+    argp.add_argument('--serial', dest='soa_serial', type=int, metavar='uint32', nargs=1, help='SOA serial number (otherwise UNIX timestamp).')
+    argp.add_argument('--absolute-names', dest='fix_absolute', action='store_true', help="Interpret names in records' contents (e.g. CNAME, NS...) as absolute even if w/o trailing dot.")
+    argp.add_argument('--from-changes', dest='from_changes', metavar="from_id", nargs='?', const=0, help="Export zones listed in extra 'changes' table.")
+    argp.add_argument('--knotc', dest='knotc_socket', metavar='knot_socket', nargs=1, help="Notify Knot DNS about changes (requires: $PATH/knotc).")
+    argp.add_argument('--knotc-timeout', dest='knotc_timeout', type=int, metavar='uint32', nargs=1, help='Timeout for knotc commands (default 10).')
+    argp.add_argument('--slave', dest='slave', action='store_true', help="Don't generate zonefiles, use 'knotc zone-refresh' instead of zone-reload.")
+    argp.add_argument('--version', action='version', version='dns_sql2zf 0.1')
+    args = argp.parse_args()
+
+    if args.soa_serial is not None:
+        soa_serial = args.soa_serial[0]
+
+    if args.confile is not None:
+        read_config_file(args.confile[0])
+
+    if args.storage is not None:
+        storage = args.storage[0]
+
+    if args.fix_absolute:
+        fix_absolute = True
+
+    if args.knotc_socket is not None:
+        knotc_socket = args.knotc_socket[0]
+
+    if args.knotc_timeout is not None:
+        knotc_timeout = args.knotc_timeout[0]
+
+    if args.slave:
+        slave_mode = True
+
+    connection = connectionForURI(args.dburi[0])
+    sqlhub.processConnection = connection
+
+    for domain in args.domains:
+        print_domain(domain)
+
+    if args.all:
+        process_all()
+
+    if args.from_changes is not None:
+        process_changes(args.from_changes)
+
+    if conf_txn_open:
+        knotc_single("conf-commit")
+
+    for zone in knotc_zone_reload:
+        knotc_single("zone-reload" if not slave_mode else "zone-refresh", zone)
+
+if __name__ == "__main__":
+    main()
+