Unverified Commit 48fec5e3 authored by Pavel Spirek's avatar Pavel Spirek
Browse files

Adapted to new version of yangson library, code refactor

parent 40897986
......@@ -81,12 +81,9 @@ def main():
datamodel = DataHelpers.load_data_model("data/", "data/yang-library-data.json")
# Datastore init
datastore = JsonDatastore(datamodel, "jetconf/example-data.json", "DNS data")
datastore = JsonDatastore(datamodel, "jetconf/example-data.json", "DNS data", with_nacm=True)
datastore.load()
datastore.load_yl_data("data/yang-library-data.json")
nacmc = NacmConfig(datastore)
datastore.register_nacm(nacmc)
nacmc.set_ds(datastore)
datastore.get_data_root().validate(ContentType.config)
......
......@@ -23,7 +23,8 @@ CONFIG_HTTP = {
"SERVER_SSL_CERT": "server.crt",
"SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem"
"CA_CERT": "ca.pem",
"DBG_DISABLE_CERTS": False
}
CONFIG_NACM = {
......
......@@ -4,7 +4,7 @@ GLOBAL:
PIDFILE: "/tmp/jetconf.pid"
PERSISTENT_CHANGES: true
LOG_LEVEL: "debug"
LOG_DBG_MODULES: ["usr_conf_data_handlers", "knot_api"]
LOG_DBG_MODULES: ["usr_conf_data_handlers", "knot_api", "nacm"]
HTTP_SERVER:
DOC_ROOT: "jetconf/doc-root"
......@@ -15,6 +15,7 @@ HTTP_SERVER:
SERVER_SSL_CERT: "jetconf/server.crt"
SERVER_SSL_PRIVKEY: "jetconf/server.key"
CA_CERT: "jetconf/ca.pem"
DBG_DISABLE_CERTS: false
NACM:
ALLOWED_USERS: ["lojza@mail.cz"]
......
......@@ -206,7 +206,7 @@ class UsrChangeJournal:
class BaseDatastore:
def __init__(self, dm: DataModel, name: str=""):
def __init__(self, dm: DataModel, name: str="", with_nacm: bool=False):
self.name = name
self.nacm = None # type: NacmConfig
self._data = None # type: InstanceNode
......@@ -219,9 +219,8 @@ class BaseDatastore:
self.commit_begin_callback = None # type: Callable
self.commit_end_callback = None # type: Callable
# Register NACM module to datastore
def register_nacm(self, nacm_config: "NacmConfig"):
self.nacm = nacm_config
if with_nacm:
self.nacm = NacmConfig(self)
# Returns the root node of data tree
def get_data_root(self, previous_version: int=0) -> InstanceNode:
......@@ -311,23 +310,22 @@ class BaseDatastore:
else:
root = self._data
# n = root.goto(ii)
# sn = n.schema_node
n = root.goto(ii)
sch_pth_list = filter(lambda n: isinstance(n, MemberName), ii)
sch_pth = "".join([str(seg) for seg in sch_pth_list])
sn = self.get_schema_node(sch_pth)
if not yl_data:
if sn.state_roots():
self.commit_begin_callback()
for state_node_pth in sn.state_roots():
sdh = STATE_DATA_HANDLES.get_handler(state_node_pth)
if sdh is not None:
root_val = sdh.update_node(ii, root, True)
root = self._data.update(root_val, raw=True)
else:
raise NoHandlerForStateDataError()
self.commit_end_callback()
state_roots = sn.state_roots()
if not yl_data and state_roots:
self.commit_begin_callback()
for state_node_pth in state_roots:
sdh = STATE_DATA_HANDLES.get_handler(state_node_pth)
if sdh is not None:
root_val = sdh.update_node(ii, root, True)
root = self._data.update(root_val, raw=True)
else:
raise NoHandlerForStateDataError()
self.commit_end_callback()
n = root.goto(ii)
......@@ -347,7 +345,7 @@ class BaseDatastore:
raise NacmForbiddenError()
else:
# Prun subtree data
n = nrpc.check_data_read_path(root, ii)
n = nrpc.check_data_read_path(n, root, ii)
try:
max_depth = int(rpc.qs["depth"][0])
......@@ -434,8 +432,7 @@ class BaseDatastore:
# Create list if necessary
if existing_member is None:
new_n = n.put_member(input_member_name, ArrayValue([]))
existing_member = new_n.member(input_member_name)
existing_member = n.put_member(input_member_name, ArrayValue([]))
# Convert input data from List/Dict to ArrayValue/ObjectValue
new_value_data = member_sn.from_raw([input_member_value])[0]
......@@ -463,8 +460,7 @@ class BaseDatastore:
# Create leaf list if necessary
if existing_member is None:
new_n = n.put_member(input_member_name, ArrayValue([]))
existing_member = new_n.member(input_member_name)
existing_member = n.put_member(input_member_name, ArrayValue([]))
# Convert input data from List/Dict to ArrayValue/ObjectValue
new_value_data = member_sn.from_raw([input_member_value])[0]
......@@ -650,8 +646,8 @@ class BaseDatastore:
class JsonDatastore(BaseDatastore):
def __init__(self, dm: DataModel, json_file: str, name: str = ""):
super().__init__(dm, name)
def __init__(self, dm: DataModel, json_file: str, name: str = "", with_nacm: bool=False):
super().__init__(dm, name, with_nacm)
self.json_file = json_file
def load(self):
......@@ -659,6 +655,9 @@ class JsonDatastore(BaseDatastore):
with open(self.json_file, "rt") as fp:
self._data = self._dm.from_raw(json.load(fp))
if self.nacm is not None:
self.nacm.update()
def load_yl_data(self, filename: str):
self._yang_lib_data = None
with open(filename, "rt") as fp:
......
......@@ -8,9 +8,7 @@ from pytz import timezone
from yangson.instance import InstanceRoute, MemberName, EntryKeys, InstanceIdParser, ResourceIdParser
from yangson.datamodel import DataModel
from .config import CONFIG
CERT_TEST = True
from .config import CONFIG_GLOBAL, CONFIG_HTTP
class PathFormat(Enum):
......@@ -21,7 +19,7 @@ class PathFormat(Enum):
class CertHelpers:
@staticmethod
def get_field(cert: Dict[str, Any], key: str) -> str:
if CERT_TEST and (key == "emailAddress"):
if CONFIG_HTTP["DBG_DISABLE_CERTS"] and (key == "emailAddress"):
return "test-user"
try:
......@@ -99,7 +97,7 @@ class LogHelpers:
module_name_simple = module_name.split(".")[-1]
def module_dbg_logger(msg: str):
if ({module_name_simple, "*"} & set(CONFIG["GLOBAL"]["LOG_DBG_MODULES"])) and (CONFIG["GLOBAL"]["LOG_LEVEL"] == "debug"):
if ({module_name_simple, "*"} & set(CONFIG_GLOBAL["LOG_DBG_MODULES"])) and (CONFIG_GLOBAL["LOG_LEVEL"] == "debug"):
logger = getLogger()
logger.setLevel(logging.DEBUG)
debug(module_name_simple + ": " + msg)
......
......@@ -10,7 +10,7 @@ from yangson.schema import NonexistentSchemaNode
from yangson.instance import NonexistentInstance, InstanceValueError
from yangson.datatype import YangTypeError
from jetconf.knot_api import KnotError
from .knot_api import KnotError
from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers, LogHelpers
from .data import (
......@@ -82,7 +82,7 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str, yl_dat
("server", CONFIG_HTTP["SERVER_NAME"])
]
try:
lm_time = DateTimeHelpers.to_httpdate_str(n.value.last_modified, CONFIG_GLOBAL["TIMEZONE"])
lm_time = DateTimeHelpers.to_httpdate_str(n.value.timestamp, CONFIG_GLOBAL["TIMEZONE"])
response_headers.append(("Last-Modified", lm_time))
except AttributeError:
# Only arrays and objects have last_modified attribute
......@@ -220,7 +220,7 @@ def _get_staging(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str
("server", CONFIG_HTTP["SERVER_NAME"])
]
try:
lm_time = DateTimeHelpers.to_httpdate_str(n.value.last_modified, CONFIG_GLOBAL["TIMEZONE"])
lm_time = DateTimeHelpers.to_httpdate_str(n.value.timestamp, CONFIG_GLOBAL["TIMEZONE"])
response_headers.append(("Last-Modified", lm_time))
except AttributeError:
# Only arrays and objects have last_modified attribute
......@@ -279,9 +279,8 @@ def create_get_staging_api(ds: BaseDatastore):
return get_staging_api_closure
def _post(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth: str):
data_str = data.decode("utf-8")
debug_httph("HTTP data received: " + data_str)
def _post(prot: "H2Protocol", data: str, stream_id: int, ds: BaseDatastore, pth: str):
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
......@@ -297,7 +296,7 @@ def _post(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pt
rpc1.path = url_path
try:
json_data = json.loads(data_str) if len(data_str) > 0 else {}
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse POST data: " + epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
......@@ -333,7 +332,7 @@ def _post(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pt
def create_post_api(ds: BaseDatastore):
def post_api_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: bytes):
def post_api_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: str):
username = CertHelpers.get_field(prot.client_cert, "emailAddress")
info("[{}] api_post: {}".format(username, headers[":path"]))
......@@ -353,9 +352,8 @@ def create_post_api(ds: BaseDatastore):
return post_api_closure
def _put(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth: str):
data_str = data.decode("utf-8")
debug_httph("HTTP data received: " + data_str)
def _put(prot: "H2Protocol", data: str, stream_id: int, ds: BaseDatastore, pth: str):
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
......@@ -367,7 +365,7 @@ def _put(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth
rpc1.path = url_path
try:
json_data = json.loads(data_str) if len(data_str) > 0 else {}
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse PUT data: " + epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
......@@ -398,7 +396,7 @@ def _put(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth
def create_put_api(ds: BaseDatastore):
def put_api_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: bytes):
def put_api_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: str):
username = CertHelpers.get_field(prot.client_cert, "emailAddress")
info("[{}] api_put: {}".format(username, headers[":path"]))
......@@ -474,10 +472,9 @@ def create_api_delete(ds: BaseDatastore):
def create_api_op(ds: BaseDatastore):
def api_op_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: bytes):
def api_op_closure(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: str):
username = CertHelpers.get_field(prot.client_cert, "emailAddress")
info("[{}] invoke_op: {}".format(username, headers[":path"]))
data_str = data.decode("utf-8")
api_pth = headers[":path"][len(API_ROOT_ops):]
op_name_fq = api_pth[1:]
......@@ -492,7 +489,7 @@ def create_api_op(ds: BaseDatastore):
op_name = op_name_splitted[1]
try:
json_data = json.loads(data_str) if len(data_str) > 0 else {}
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse POST data: " + epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
......
import collections
import copy
from io import StringIO
from threading import Lock
from enum import Enum
from colorlog import error, warning as warn, info
from colorlog import error, info
from typing import List, Set
from yangson.instance import (
......@@ -18,7 +19,7 @@ from yangson.instance import (
EntryKeys
)
from .helpers import DataHelpers, ErrorHelpers, LogHelpers
from .helpers import DataHelpers, ErrorHelpers, LogHelpers, PathFormat
epretty = ErrorHelpers.epretty
debug_nacm = LogHelpers.create_module_dbg_logger(__name__)
......@@ -104,11 +105,9 @@ class NacmRuleList:
class DataRuleTree:
def __init__(self):
def __init__(self, rule_lists: List[NacmRuleList]):
self.root = [] # type: List[RuleTreeNode]
# Access to datastore needed for parsing IIs
def create_rule_tree(self, rule_lists: List[NacmRuleList]):
for rl in rule_lists:
for rule in filter(lambda r: r.type == NacmRuleType.NACM_RULE_DATA, rl.rules):
try:
......@@ -136,17 +135,8 @@ class DataRuleTree:
node_match_prev = node_match
nl = node_match.children
def _print_rule_tree(self, rule_node_list: List[RuleTreeNode], depth: int = 0, vbars=[]):
if depth == 0:
print("----- NACM Data Rule tree -----")
ind_str = ""
d = depth
while d:
ind_str += " "
d -= 1
ind_str += "+--"
def _print_rule_tree(self, io_str: StringIO, rule_node_list: List[RuleTreeNode], depth: int, vbars: List[int]):
ind_str = (" " * depth) + "+--"
for vb in vbars:
isl = list(ind_str)
......@@ -156,38 +146,31 @@ class DataRuleTree:
for rule_node in rule_node_list:
action = rule_node.get_action(Permission.NACM_ACCESS_READ)
action_str = str(action.name) if action is not None else ""
print(ind_str + " " + str(rule_node.isel) + " " + action_str)
io_str.write(ind_str + " " + str(rule_node.isel) + " " + action_str + "\n")
if rule_node is rule_node_list[-1]:
self._print_rule_tree(rule_node.children, depth + 1, vbars)
self._print_rule_tree(io_str, rule_node.children, depth + 1, vbars)
else:
self._print_rule_tree(rule_node.children, depth + 1, vbars + [depth])
self._print_rule_tree(io_str, rule_node.children, depth + 1, vbars + [depth])
def print_rule_tree(self):
self._print_rule_tree(self.root)
def print_rule_tree(self) -> str:
io_str = StringIO()
io_str.write("----- NACM Data Rule tree -----\n")
self._print_rule_tree(io_str, self.root, 0, [])
return io_str.getvalue()
class NacmConfig:
def __init__(self, nacm_ds: "BaseDatastore"):
self.nacm_ds = nacm_ds
self.ds = None
self.enabled = False
self.default_read = Action.PERMIT
self.default_write = Action.PERMIT
self.default_exec = Action.PERMIT
self.nacm_groups = []
self.rule_lists = []
self._user_nacm_rpc = {}
self.internal_data_lock = Lock()
self._lock_username = None
self._user_nacm_rpc = {}
try:
self.nacm_ds.get_data_root().member("ietf-netconf-acm:nacm")
except NonexistentInstance:
raise ValueError("Data does not contain \"ietf-netconf-acm:nacm\" root element")
def set_ds(self, ds: "BaseDatastore"):
self.ds = ds
self.update()
# Fills internal read-only data structures
def update(self):
......@@ -200,11 +183,14 @@ class NacmConfig:
self.rule_lists = []
self._user_nacm_rpc = {}
nacm_json = self.nacm_ds.get_data_root().member("ietf-netconf-acm:nacm").value
self.enabled = nacm_json["enable-nacm"]
try:
nacm_json = self.nacm_ds.get_data_root().member("ietf-netconf-acm:nacm").value
except NonexistentInstance:
raise ValueError("Data does not contain \"ietf-netconf-acm:nacm\" root element")
# NACM not enabled, no need to continue
self.enabled = nacm_json["enable-nacm"]
if not self.enabled:
# NACM not enabled, no need to continue
self.internal_data_lock.release()
return
......@@ -260,7 +246,6 @@ class NacmConfig:
error("Invalid rule definition (multiple cases from rule-type choice): \"{}\"".format(rule.name))
else:
rule.type = NacmRuleType.NACM_RULE_DATA
# i.e. /ietf-interfaces:interfaces/interface[name='eth0']/ietf-ip:ipv4/ip
rule.type_data.path = rule_json["path"]
rule.action = Action.PERMIT if rule_json["action"] == "permit" else Action.DENY
......@@ -282,9 +267,15 @@ class NacmConfig:
# if username not in all_users:
# raise NonexistentUserError
if not self.internal_data_lock.acquire(blocking=True, timeout=1):
error("Cannot acquire NACM config lock ")
return
info("Creating personalized rule list for user \"{}\"".format(username))
self._user_nacm_rpc[username] = UserNacm(self, username)
self.internal_data_lock.release()
def get_user_nacm(self, username: str) -> "UserNacm":
user_nacm = self._user_nacm_rpc.get(username)
if user_nacm is None:
......@@ -298,28 +289,17 @@ class NacmConfig:
class UserNacm:
def __init__(self, config: NacmConfig, username: str):
self.nacm_enabled = config.enabled
self.data = config.ds
self.default_read = config.default_read
self.default_write = config.default_write
self.default_exec = config.default_exec
self.rule_lists = []
self.rule_tree = DataRuleTree()
lock_res = config.internal_data_lock.acquire(blocking=True, timeout=1)
if not lock_res:
error("NacmRpc: cannot acquire config lock ")
return
user_groups = list(filter(lambda x: username in x.users, config.nacm_groups))
user_groups_names = list(map(lambda x: x.name, user_groups))
self.rule_lists = list(filter(lambda x: (set(user_groups_names) & set(x.groups)), config.rule_lists))
self.rule_tree.create_rule_tree(self.rule_lists)
# self.rule_tree.print_rule_tree()
# No need to hold lock anymore
# config.update() always creates new structures instead of modifying ones
config.internal_data_lock.release()
self.rule_tree = DataRuleTree(self.rule_lists)
debug_nacm("Rule tree for user \"{}\":\n{}".format(username, self.rule_tree.print_rule_tree()))
def check_data_node_path(self, root: InstanceNode, ii: InstanceRoute, access: Permission, out_matching_rule: List[NacmRule]=None) -> Action:
if not self.nacm_enabled:
......@@ -330,18 +310,13 @@ class UserNacm:
nl = self.rule_tree.root
for isel in ii:
# node_match = (list(filter(lambda x: x.isel == isel, nl)) or [None])[0]
# print("j {}".format(isel))
node_match = None
for rule_node in nl:
if (type(rule_node.isel) == type(isel)) and (rule_node.isel == isel):
node_match = rule_node
break
# print("{} {}".format(type(isel), type(rule_node.isel)))
if isinstance(isel, EntryIndex) and isinstance(rule_node.isel, EntryKeys):
# print("k")
if isel.peek_step(data_node.value) is rule_node.isel.peek_step(data_node.value):
node_match = rule_node
break
......@@ -364,14 +339,12 @@ class UserNacm:
else:
retval = self.default_write
debug_nacm("check_data_node_path, result = {}".format(retval.name))
return retval
def _check_data_read_path(self, node: InstanceNode, root: InstanceNode, ii: InstanceRoute) -> InstanceNode:
# node = self.data.get_node(ii)
if isinstance(node.value, ObjectValue):
# print("obj: {}".format(node.value))
# print(str(ii))
for child_key in sorted(node.value.keys()):
nsel = MemberName(child_key)
m = nsel.goto_step(node)
......@@ -380,9 +353,9 @@ class UserNacm:
debug_nacm("checking mii {}".format(mii))
if self.check_data_node_path(root, mii, Permission.NACM_ACCESS_READ) == Action.DENY:
# info("Pruning node {} {}".format(id(node.value[child_key]), node.value[child_key]))
# debug_nacm("Pruning node {} {}".format(id(node.value[child_key]), node.value[child_key]))
debug_nacm("Pruning node {}".format(mii))
node = node.delete_member(child_key, validate=False)
node = node.delete_member(child_key)
else:
node = self._check_data_read_path(m, root, mii).up()
elif isinstance(node.value, ArrayValue):
......@@ -397,7 +370,8 @@ class UserNacm:
debug_nacm("checking eii {}".format(eii))
if self.check_data_node_path(root, eii, Permission.NACM_ACCESS_READ) == Action.DENY:
debug_nacm("Pruning node {} {}".format(id(node.value[i]), node.value[i]))
# debug_nacm("Pruning node {} {}".format(id(node.value[i]), node.value[i]))
debug_nacm("Pruning node {}".format(eii))
node = node.delete_entry(i)
arr_len -= 1
else:
......@@ -406,12 +380,11 @@ class UserNacm:
return node
def check_data_read_path(self, root: InstanceNode, ii: InstanceRoute) -> InstanceNode:
n = self.data.get_node(root, ii)
def check_data_read_path(self, node: InstanceNode, root: InstanceNode, ii: InstanceRoute) -> InstanceNode:
if not self.nacm_enabled:
return n
return node
else:
return self._check_data_read_path(n, root, ii)
return self._check_data_read_path(node, root, ii)
def check_rpc_name(self, rpc_name: str, out_matching_rule: List[NacmRule] = None) -> Action:
if not self.nacm_enabled:
......@@ -425,11 +398,7 @@ class UserNacm:
return rpc_rule.action
return self.default_exec
# raise NacmForbiddenError("Op \"{}\" invocation denied for user \"{}\"".format(rpc.path, rpc.username))
def test():
error("Tests moved to tests/tests_jetconf.py")
from .data import JsonDatastore, PathFormat
import io
import asyncio
import ssl
from io import BytesIO
from collections import OrderedDict
from colorlog import error, warning as warn, info, debug
from typing import List, Tuple, Dict, Any, Callable
......@@ -77,7 +77,7 @@ class H2Protocol(asyncio.Protocol):
if isinstance(event, RequestReceived):
# Store request headers
headers = OrderedDict(event.headers)
request_data = RequestData(headers, io.BytesIO())
request_data = RequestData(headers, BytesIO())
self.stream_data[event.stream_id] = request_data
elif isinstance(event, DataReceived):
# Store incoming data
......@@ -150,12 +150,13 @@ class RestServer:
except AttributeError:
info("Python not compiled with ALPN support, using NPN instead.")
ssl_context.set_npn_protocols(["h2"])
# ssl_context.verify_mode = ssl.CERT_REQUIRED
if not CONFIG_HTTP["DBG_DISABLE_CERTS"]:
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.load_verify_locations(cafile=CONFIG_HTTP["CA_CERT"])
self.loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
# Each client connection will create a new H2Protocol instance
listener = self.loop.create_server(H2Protocol, "127.0.0.1", CONFIG_HTTP["PORT"], ssl=ssl_context)
self.server = self.loop.run_until_complete(listener)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment