Refactor of zone data handlers and some datastore methods

jetconf/data.py

@@ -2,7 +2,7 @@ import json
 from threading import Lock
 from enum import Enum
 from colorlog import error, warning as warn, info
-from typing import List, Any, Dict, Callable
+from typing import List, Any, Dict, Callable, Optional
 
 from yangson.schema import SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode, SchemaError, SemanticError
 from yangson.datamodel import DataModel
@@ -87,7 +87,7 @@ class ConfHandlerResult(Enum):
 
 class BaseDataListener:
     def __init__(self, ds: "BaseDatastore", sch_pth: str):
-        self._ds = ds
+        self.ds = ds
         self.schema_path = sch_pth                          # type: str
         self.schema_node = ds.get_schema_node(sch_pth)      # type: SchemaNode
 
@@ -191,15 +191,15 @@ class UsrChangeJournal:
             # Set new data root
             ds.set_data_root(nr)
 
-            # Notify schema node observers
+            # Run schema node handlers
             for cl in self.clists:
                 for change in cl.journal:
                     ii = ds.parse_ii(change.rpc_info.path, change.rpc_info.path_format)
-                    ds.notify_edit(ii, change)
-                    # if change.change_type != ChangeType.DELETE:
-                    #     ds.notify_edit(ii)
-                    # else:
-                    #     ds.notify_edit(ii[0:-1])
+                    try:
+                        ds.run_conf_edit_handler(ii, change)
+                    except Exception as e:
+                        ds.data_root_rollback(1, False)
+                        raise e
 
             # Clear user changelists
             self.clists.clear()
@@ -210,7 +210,8 @@ class BaseDatastore:
         self.name = name
         self.nacm = None    # type: NacmConfig
         self._data = None   # type: InstanceNode
-        self._yang_lib_data = None   # type: InstanceNode
+        self._data_history = []     # type: List[InstanceNode]
+        self._yang_lib_data = None  # type: InstanceNode
         self._dm = dm       # type: DataModel
         self._data_lock = Lock()
         self._lock_username = None  # type: str
@@ -223,8 +224,11 @@ class BaseDatastore:
         self.nacm = nacm_config
 
     # Returns the root node of data tree
-    def get_data_root(self) -> InstanceNode:
-        return self._data
+    def get_data_root(self, previous_version: int=0) -> InstanceNode:
+        if previous_version > 0:
+            return self._data_history[-previous_version]
+        else:
+            return self._data
 
     def get_yl_data_root(self) -> InstanceNode:
         return self._yang_lib_data
@@ -238,10 +242,17 @@ class BaseDatastore:
         else:
             raise NoHandlerError("No active changelist for user \"{}\"".format(username))
 
-    # Set a new Instance node as data root
+    # Set a new Instance node as data root, store old root to archive
     def set_data_root(self, new_root: InstanceNode):
+        self._data_history.append(self._data)
         self._data = new_root
 
+    def data_root_rollback(self, history_steps: int, store_current: bool):
+        if store_current:
+            self._data_history.append(self._data)
+
+        self._data = self._data_history[-history_steps]
+
     # Get schema node with particular schema address
     def get_schema_node(self, sch_pth: str) -> SchemaNode:
         sn = self._dm.get_schema_node(sch_pth)
@@ -259,10 +270,10 @@ class BaseDatastore:
         return ii
 
     # Notify data observers about change in datastore
-    def notify_edit(self, ii: InstanceRoute, ch: DataChange):
+    def run_conf_edit_handler(self, ii: InstanceRoute, ch: DataChange) -> Optional[ConfHandlerResult]:
+        h_res = None
+
         try:
-            # n = self._data.goto(ii)
-            # sn = n.schema_node
             sch_pth_list = filter(lambda n: isinstance(n, MemberName), ii)
             sch_pth = "".join([str(seg) for seg in sch_pth_list])
             sn = self.get_schema_node(sch_pth)
@@ -285,6 +296,8 @@ class BaseDatastore:
         except NonexistentInstance:
             warn("Cannnot notify {}, parent container removed".format(ii))
 
+        return h_res
+
     # Just get the node, do not evaluate NACM (needed for NACM)
     def get_node(self, root: InstanceNode, ii: InstanceRoute) -> InstanceNode:
         n = root.goto(ii)

jetconf/knot_api.py

 from enum import Enum
-from typing import List, Union, Dict, Any
+from typing import List, Union, Dict, Any, Optional
 from threading import Lock
 from colorlog import info
 
@@ -39,14 +39,18 @@ class KnotConfState(Enum):
 
 
 class RRecordBase:
-    def __init__(self, owner_name: str, res_type: str, ttl: int=3600):
-        self.owner = owner_name
+    def __init__(self, owner: str, res_type: str, ttl: Optional[int]=None):
+        self.owner = owner
         self.type = res_type
         self.ttl = ttl
 
     def rrdata_format(self) -> str:
         raise NotImplementedError("Not implemented in base class")
 
+    @property
+    def ttl_str(self):
+        return str(self.ttl) if self.ttl is not None else None
+
 
 class SOARecord(RRecordBase):
     def __init__(self):
@@ -65,9 +69,27 @@ class SOARecord(RRecordBase):
         )
 
 
+class NSRecord(RRecordBase):
+    def __init__(self, owner: str, ttl: Optional[int]=None):
+        super().__init__(owner, "NS", ttl)
+        self.nsdname = None     # type: str
+
+    def rrdata_format(self) -> str:
+        return self.nsdname
+
+
 class ARecord(RRecordBase):
-    def __init__(self, owner_name: str):
-        super().__init__(owner_name, "A")
+    def __init__(self, owner: str, ttl: Optional[int]=None):
+        super().__init__(owner, "A", ttl)
+        self.address = None     # type: str
+
+    def rrdata_format(self) -> str:
+        return self.address
+
+
+class AAAARecord(RRecordBase):
+    def __init__(self, owner: str, ttl: Optional[int]=None):
+        super().__init__(owner, "AAAA", ttl)
         self.address = None     # type: str
 
     def rrdata_format(self) -> str:
@@ -75,8 +97,8 @@ class ARecord(RRecordBase):
 
 
 class MXRecord(RRecordBase):
-    def __init__(self, owner_name: str):
-        super().__init__(owner_name, "MX")
+    def __init__(self, owner: str, ttl: Optional[int]=None):
+        super().__init__(owner, "MX", ttl)
         self.preference = None  # type: str
         self.exchange = None    # type: str
 
@@ -176,15 +198,7 @@ class KnotConfig(KnotCtl):
         for data_item in data:
             self.send_block("conf-set", section=section, identifier=identifier, item=item, zone=zone, data=data_item)
 
-    def set_zone_item(self, section=None, identifier=None, item=None, zone=None, owner=None, ttl=None, rtype=None, data=None):
-        if not self.connected:
-            raise KnotApiError("Knot socket is closed")
-
-        if data is not None:
-            self.send_block("zone-set", section=section, identifier=identifier, item=item, zone=zone, owner=owner, ttl=ttl, rtype=rtype, data=data)
-        else:
-            self.send_block("zone-unset", section=section, identifier=identifier, item=item, zone=zone, owner=owner, ttl=ttl, rtype=rtype, data=data)
-
+    # Returns a status data of all or one specific DNS zone
     def zone_status(self, domain_name: str=None) -> JsonNodeT:
         if not self.connected:
             raise KnotApiError("Knot socket is closed")
@@ -196,6 +210,7 @@ class KnotConfig(KnotCtl):
             raise KnotInternalError(str(e))
         return resp
 
+    # Adds a new DNS zone
     def zone_new(self, domain_name: str) -> JsonNodeT:
         if not self.connected:
             raise KnotApiError("Knot socket is closed")
@@ -207,6 +222,7 @@ class KnotConfig(KnotCtl):
             raise KnotInternalError(str(e))
         return resp
 
+    # Removes a DNS zone
     def zone_remove(self, domain_name: str) -> JsonNodeT:
         if not self.connected:
             raise KnotApiError("Knot socket is closed")
@@ -218,27 +234,32 @@ class KnotConfig(KnotCtl):
             raise KnotInternalError(str(e))
         return resp
 
+    # Adds a resource record to DNS zone
     def zone_add_record(self, domain_name: str, rr: RRecordBase) -> JsonNodeT:
         if not self.connected:
             raise KnotApiError("Knot socket is closed")
 
         try:
             res_data = rr.rrdata_format()
-            self.set_zone_item(zone=domain_name, owner=rr.owner, ttl=str(rr.ttl), rtype=rr.type, data=res_data)
-            debug_knot("Inserting zone \"{}\" RR, type=\"{}\", owner=\"{}\", ttl=\"{}\", data=\"{}\"".format(
-                domain_name, rr.type, rr.owner, rr.ttl, res_data
+            self.send_block("zone-set", zone=domain_name, owner=rr.owner, ttl=rr.ttl_str, rtype=rr.type, data=res_data)
+
+            debug_knot("Inserting zone \"{}\" RR, type=\"{}\", owner=\"{}\", ttl={}, data=\"{}\"".format(
+                domain_name, rr.type, rr.owner, rr.ttl_str, res_data
             ))
             resp = self.receive_block()
         except Exception as e:
             raise KnotInternalError(str(e))
         return resp
 
-    def zone_del_record(self, domain_name: str, owner: str, rr_type: str) -> JsonNodeT:
+    # Removes a resource record from DNS zone
+    # If the zone contains two or more records with the same owner and type, selector parameter can specify
+    # which one to remove. Usually it is the same as record data.
+    def zone_del_record(self, domain_name: str, owner: str, rr_type: str, selector: str=None) -> JsonNodeT:
         if not self.connected:
             raise KnotApiError("Knot socket is closed")
 
         try:
-            self.set_zone_item(zone=domain_name, owner=owner, rtype=rr_type, data=None)
+            self.send_block("zone-unset", zone=domain_name, owner=owner, rtype=rr_type, data=selector)
             resp = self.receive_block()
         except Exception as e:
             raise KnotInternalError(str(e))

jetconf/usr_conf_data_handlers.py

 from colorlog import info, warning as warn, error
+from typing import List, Dict, Union, Any
 
 from yangson.instance import InstanceRoute, ObjectValue, EntryKeys, MemberName
 from . import knot_api
 from .data import BaseDataListener, SchemaNode, PathFormat, ChangeType, DataChange, ConfHandlerResult
 from .helpers import ErrorHelpers, LogHelpers
-from .knot_api import SOARecord, ARecord
+from .knot_api import RRecordBase, SOARecord, ARecord, AAAARecord, NSRecord, MXRecord
 
+JsonNodeT = Union[Dict[str, Any], List]
 epretty = ErrorHelpers.epretty
 debug_confh = LogHelpers.create_module_dbg_logger(__name__)
 
@@ -15,8 +17,8 @@ class KnotConfServerListener(BaseDataListener):
         debug_confh(self.__class__.__name__ + " triggered")
 
         base_ii_str = self.schema_path
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
-        base_nv = self._ds.get_node(self._ds.get_data_root(), base_ii).value
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_nv = self.ds.get_node(self.ds.get_data_root(), base_ii).value
 
         knot_api.KNOT.begin()
 
@@ -49,8 +51,8 @@ class KnotConfLogListener(BaseDataListener):
         debug_confh(self.__class__.__name__ + " triggered")
 
         base_ii_str = self.schema_path
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
-        base_nv = self._ds.get_node(self._ds.get_data_root(), base_ii).value
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_nv = self.ds.get_node(self.ds.get_data_root(), base_ii).value
 
         knot_api.KNOT.begin()
         knot_api.KNOT.set_item(section="log", data=None)
@@ -76,7 +78,7 @@ class KnotConfZoneListener(BaseDataListener):
 
         # ii_str = "".join([str(seg) for seg in ii])
         base_ii_str = self.schema_path
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
 
         knot_api.KNOT.begin()
 
@@ -96,7 +98,7 @@ class KnotConfZoneListener(BaseDataListener):
             debug_confh("Editing config of zone \"{}\"".format(domain))
 
             # Write whole zone config to Knot
-            zone_nv = self._ds.get_node(self._ds.get_data_root(), ii[0:(len(base_ii) + 1)]).value
+            zone_nv = self.ds.get_node(self.ds.get_data_root(), ii[0:(len(base_ii) + 1)]).value
             knot_api.KNOT.set_item(section="zone", zone=domain, item="comment", data=zone_nv.get("description"))
             knot_api.KNOT.set_item(section="zone", zone=domain, item="file", data=zone_nv.get("file"))
             knot_api.KNOT.set_item_list(section="zone", zone=domain, item="master", data=zone_nv.get("master"))
@@ -129,8 +131,8 @@ class KnotConfControlListener(BaseDataListener):
         debug_confh(self.__class__.__name__ + " triggered")
 
         base_ii_str = self.schema_path
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
-        base_nv = self._ds.get_node(self._ds.get_data_root(), base_ii).value
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_nv = self.ds.get_node(self.ds.get_data_root(), base_ii).value
 
         knot_api.KNOT.begin()
         knot_api.KNOT.set_item(section="control", item="listen", data=base_nv.get("unix"))
@@ -161,15 +163,15 @@ class KnotConfAclListener(BaseDataListener):
         debug_confh(self.__class__.__name__ + " triggered")
 
         base_ii_str = self.schema_path
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
-        base_nv = self._ds.get_node(self._ds.get_data_root(), base_ii).value
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_nv = self.ds.get_node(self.ds.get_data_root(), base_ii).value
 
         knot_api.KNOT.begin()
         knot_api.KNOT.set_item(section="acl", data=None)
 
         if (len(ii) > len(base_ii)) and isinstance(ii[len(base_ii)], EntryKeys):
             # Write only changed list item
-            acl_nv = self._ds.get_node(self._ds.get_data_root(), ii[0:(len(base_ii) + 1)]).value
+            acl_nv = self.ds.get_node(self.ds.get_data_root(), ii[0:(len(base_ii) + 1)]).value
             print("acl nv={}".format(acl_nv))
             self._process_list_item(acl_nv)
         else:
@@ -185,11 +187,38 @@ class KnotConfAclListener(BaseDataListener):
 
 
 class KnotZoneDataListener(BaseDataListener):
+    # Create RR object from "rdata" json node
+    @staticmethod
+    def _rr_from_rdata_item(domain_name: str, rr_owner: str, rr_ttl: int, rr_type: str, rdata_item: JsonNodeT) -> RRecordBase:
+        try:
+            if rr_type == "A":
+                new_rr = ARecord(domain_name, rr_ttl)
+                new_rr.owner = rr_owner
+                new_rr.address = rdata_item["A"]["address"]
+            elif rr_type == "AAAA":
+                new_rr = AAAARecord(domain_name, rr_ttl)
+                new_rr.owner = rr_owner
+                new_rr.address = rdata_item["AAAA"]["address"]
+            elif rr_type == "NS":
+                new_rr = NSRecord(domain_name, rr_ttl)
+                new_rr.owner = rr_owner
+                new_rr.nsdname = rdata_item["NS"]["nsdname"]
+            elif rr_type == "MX":
+                new_rr = MXRecord(domain_name, rr_ttl)
+                new_rr.owner = rr_owner
+                new_rr.exchange = rdata_item["MX"]["exchange"]
+            else:
+                new_rr = None
+        except KeyError:
+            new_rr = None
+
+        return new_rr
+
     def process(self, sn: SchemaNode, ii: InstanceRoute, ch: DataChange):
         debug_confh(self.__class__.__name__ + " triggered")
 
         base_ii_str = "/dns-zones:zone-data"
-        base_ii = self._ds.parse_ii(base_ii_str, PathFormat.URL)
+        base_ii = self.ds.parse_ii(base_ii_str, PathFormat.URL)
         base_ii_len = len(base_ii)
         ii_str = "".join([str(seg) for seg in ii])
 
@@ -198,12 +227,14 @@ class KnotZoneDataListener(BaseDataListener):
         # Create new zone with SOA in zone data section
         if (ii == base_ii) and (ch.change_type == ChangeType.CREATE):
             domain_name = ch.data["zone"]["name"]
+            def_ttl = ch.data["zone"]["default-ttl"]
 
             soa = ch.data.get("zone", {}).get("SOA")
             if soa is None:
                 return ConfHandlerResult.ERROR
 
             soarr = SOARecord()
+            soarr.ttl = def_ttl
             soarr.mname = soa["mname"]
             soarr.rname = soa["rname"]
             soarr.serial = soa["serial"]
@@ -222,20 +253,50 @@ class KnotZoneDataListener(BaseDataListener):
                 len(ii) == (base_ii_len + 2)) \
                 and isinstance(ii[base_ii_len], MemberName) and (ii[base_ii_len].name == "zone") \
                 and isinstance(ii[base_ii_len + 1], EntryKeys) \
-                and (ch.change_type == ChangeType.CREATE):
+                and (ch.change_type == ChangeType.CREATE) \
+                and (ch.data.get("rrset") is not None):
             domain_name = ii[base_ii_len + 1].keys["name"]
             rr = ch.data.get("rrset", {})
-            rr_type = rr.get("type")
+            rr_owner = rr["owner"]
+            rr_type = rr.get("type").split(":")[-1]
+            rr_ttl = rr.get("ttl")
+            if rr_ttl is None:
+                # Obtain default ttl from datastore if not specified explicitly
+                rr_ttl = self.ds.get_data_root().goto(ii[0:3]).value["default-ttl"]
+
+            for rdata_item in rr["rdata"]:
+                new_rr = self._rr_from_rdata_item(domain_name, rr_owner, rr_ttl, rr_type, rdata_item)
+                if new_rr is not None:
+                    debug_confh("KnotApi: adding new {} RR to zone \"{}\"".format(rr_type, domain_name))
+                    knot_api.KNOT.begin_zone()
+                    knot_api.KNOT.zone_add_record(domain_name, new_rr)
+                    knot_api.KNOT.commit_zone()
+
+        # Add resource record to particular zone (only specific "rdata" item)
+        elif (
+                len(ii) == (base_ii_len + 4)) \
+                and isinstance(ii[base_ii_len], MemberName) and (ii[base_ii_len].name == "zone") \
+                and isinstance(ii[base_ii_len + 1], EntryKeys) \
+                and isinstance(ii[base_ii_len + 2], MemberName) and (ii[base_ii_len + 2].name == "rrset") \
+                and isinstance(ii[base_ii_len + 3], EntryKeys) \
+                and (ch.change_type == ChangeType.CREATE) \
+                and (ch.data.get("rdata") is not None):
+            domain_name = ii[base_ii_len + 1].keys["name"]
+            rdata_item = ch.data.get("rdata", {})
+            keys_ii_seg = ii[len(base_ii) + 3]
+            rr_owner = keys_ii_seg.keys["owner"]
+            rr_type = keys_ii_seg.keys["type"][0].split(":")[-1]
 
-            if rr_type == "iana-dns-parameters:A":
-                new_rr = ARecord(domain_name)
-                new_rr.owner = rr["owner"]
-                new_rr.address = rr["rdata"][0]["A"]["address"]
-            else:
-                new_rr = None
+            # Try to use record-specific ttl first
+            rr_ttl = self.ds.get_data_root().goto(ii).value.get("ttl")
+            if rr_ttl is None:
+                # Obtain default ttl from datastore if not specified explicitly
+                rr_ttl = self.ds.get_data_root().goto(ii[0:3]).value["default-ttl"]
+
+            new_rr = self._rr_from_rdata_item(domain_name, rr_owner, rr_ttl, rr_type, rdata_item)
 
             if new_rr is not None:
-                debug_confh("KnotApi: adding new RR to zone \"{}\"".format(domain_name))
+                debug_confh("KnotApi: adding new {} RR to zone \"{}\"".format(rr_type, domain_name))
                 knot_api.KNOT.begin_zone()
                 knot_api.KNOT.zone_add_record(domain_name, new_rr)
                 knot_api.KNOT.commit_zone()
@@ -253,11 +314,34 @@ class KnotZoneDataListener(BaseDataListener):
             rr_owner = keys_ii_seg.keys["owner"]
             rr_type = keys_ii_seg.keys["type"][0]
 
-            debug_confh("KnotApi: deleting RR from zone \"{}\"".format(domain_name))
+            debug_confh("KnotApi: deleting {} RR from zone \"{}\"".format(rr_type, domain_name))
             knot_api.KNOT.begin_zone()
             knot_api.KNOT.zone_del_record(domain_name, rr_owner, rr_type)
             knot_api.KNOT.commit_zone()
 
+        # Delete resource record from particular zone (only specific "rdata" item)
+        elif (
+                len(ii) == (base_ii_len + 6)) \
+                and isinstance(ii[base_ii_len], MemberName) and (ii[base_ii_len].name == "zone") \
+                and isinstance(ii[base_ii_len + 1], EntryKeys) \
+                and isinstance(ii[base_ii_len + 2], MemberName) and (ii[base_ii_len + 2].name == "rrset") \
+                and isinstance(ii[base_ii_len + 3], EntryKeys) \
+                and isinstance(ii[base_ii_len + 4], MemberName) and (ii[base_ii_len + 4].name == "rdata") \
+                and isinstance(ii[base_ii_len + 5], EntryKeys) \
+                and (ch.change_type == ChangeType.DELETE):
+            domain_name = ii[base_ii_len + 1].keys["name"]
+            keys_ii_seg = ii[len(base_ii) + 3]
+            rr_owner = keys_ii_seg.keys["owner"]
+            rr_type = keys_ii_seg.keys["type"][0]
+            rdata_item = self.ds.get_data_root(previous_version=1).goto(ii).value
+
+            rr_sel = self._rr_from_rdata_item(domain_name, rr_owner, 0, rr_type, rdata_item)
+
+            debug_confh("KnotApi: deleting {} RR from zone \"{}\"".format(rr_type, domain_name))
+            knot_api.KNOT.begin_zone()
+            knot_api.KNOT.zone_del_record(domain_name, rr_owner, rr_type, selector=rr_sel.rrdata_format())
+            knot_api.KNOT.commit_zone()
+
         else:
             return ConfHandlerResult.ERROR