Commit eeb7412d authored by Štěpán Balážik's avatar Štěpán Balážik
Browse files

switch to ruamel.yaml and voluptuous parser/validator

parent 1c6b668b
{
"path": "examples/iter_minim_a.yaml",
"config": "",
"server_groups": [
{
"step_range": [
0,
100
],
"addresses": [
"127.0.0.10"
],
"rtts": null,
"matchers": [
null,
null
]
},
{
"step_range": [
null,
null
],
"addresses": [
"192.5.6.30"
],
"rtts": null,
"matchers": [
null
]
},
{
"step_range": [
null,
null
],
"addresses": [
"1.2.3.4"
],
"rtts": null,
"matchers": [
null
]
}
],
"steps": [
{
"id": 1,
"generator": null,
"checker": null
},
{
"time_add": 100
}
]
}
\ No newline at end of file
# Parsing:
# --------
# In an interactive python console, run
#
# from mockdns.parser import ParsedScenario
# scenario = ParsedScenario("examples/iter_minim_a.yaml")
#
# and explore the resulting object.
config: "No configuration handling yet. :("
# defaults for Matchpart:
# -----------------------
# criteria: []
# adjust: []
# message:
# id: <random>
# opcode: QUERY
# rcode: NOERROR
# flags: QR
# eflags: [] # EDNS flags
# question: []
# answer: []
# authority: []
# additional: []
# edns:
# version: 0
# payload: 4096
server_groups:
config:
data: No configuration handling yet. :("
server_groups:
# k.root-servers.net.
- first_step: 0
last_step: 100
addresses:
......@@ -14,8 +41,8 @@ server_groups:
adjust:
- copy_id
message:
question: . IN NS
answer: . 3600 IN NS K.ROOT-SERVERS.NET.
question: . IN NS # can be both list and one value
answer: . 3600 IN NS K.ROOT-SERVERS.NET. # ttl is required for now
additional: K.ROOT-SERVERS.NET. 3600 IN A 127.0.0.10
- type: Matchpart
......@@ -27,7 +54,7 @@ server_groups:
authority: com. 3600 IN NS a.gtld-servers.net.
additional: a.gtld-servers.net. 3600 IN A 192.5.6.30
# a.gtld-servers.net.
- addresses: 192.5.6.30
matchers:
- type: Matchpart
......@@ -42,7 +69,7 @@ server_groups:
authority: example.com. 3600 IN NS ns.example.com.
additional: ns.example.com. 3600 IN A 1.2.3.4
# ns.example.com
- addresses: 1.2.3.4
matchers:
- type: Matchpart
......@@ -71,6 +98,6 @@ steps:
question: www.example.com. IN A
answer: www.example.com. 3600 IN A 10.20.30.40
- id: 3
time_add: 100
time_set: 5
[mypy]
[mypy-strictyaml.*]
[mypy-voluptuous.*]
ignore_missing_imports = True
[mypy-ruamel.*]
ignore_missing_imports = True
[mypy-ruamel.yaml]
ignore_missing_imports = False
[mypy-dns.*]
ignore_missing_imports = True
from mockdns.parser.parser import YAMLScenario, JSONScenario
from mockdns.parser.parser import ParsedScenario
......@@ -4,42 +4,28 @@ from copy import deepcopy
from typing import Any, Dict, Iterable, Iterator, Mapping, Optional
from dns.message import Message
import strictyaml
import ruamel.yaml
import voluptuous
from mockdns.scenario import DataMismatch
from mockdns.scenario.scenario import DataMismatch
class Actor(ABC):
def __init__(self, yaml: Optional[strictyaml.YAML]):
if yaml is None:
return
self.options = yaml["options"].data
self.line_number = yaml.start_line
# self._dict = yaml["options"].data
# self._dict.update({"__line_number": self.line_number})
# self.yaml = yaml
# @classmethod
# def from_dict(cls, d: Dict[str, Any]):
# actor = cls(None)
# actor._dict = deepcopy(d)
# print(cls, d)
# actor.line_number = d.pop("__line_number")
# actor.options = d
# actor.yaml = None
# return actor
def __init__(self, options: Dict[str, Any]):
self.options = options
if isinstance(options, ruamel.yaml.comments.CommentedMap):
self.line_number = options.lc.line
else:
self.line_number = None
@staticmethod
@abstractmethod
def schema() -> strictyaml.Map:
def schema() -> voluptuous.Schema:
pass
def set_up(self) -> None:
pass
# def to_dict(self) -> Dict[str, Any]:
# return self._dict
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.options}, line: {self.line_number})"
......
import glob
import os.path
from typing import Any, Dict, Type
from mockdns.parser.actor_abc import Matcher, Checker, Generator
__all__ = [
os.path.basename(file_name)[:-3]
for file_name in glob.glob(os.path.join(__path__[0], "*.py")) # type: ignore # mypy issue #1422
if os.path.basename(file_name) not in ("__init__.py", "classes.py")
if os.path.basename(file_name) not in ("__init__.py")
]
from mockdns.parser.actors import *
ACTORS: Dict[str, Dict[str, Any]] = {
"matcher": {cls.__name__: cls for cls in Matcher.__subclasses__()},
"checker": {cls.__name__: cls for cls in Checker.__subclasses__()},
"generator": {cls.__name__: cls for cls in Generator.__subclasses__()},
}
......@@ -2,16 +2,19 @@ from copy import deepcopy
from typing import Iterable, Optional
import dns.message
import voluptuous
from dns.flags import EDNSFlag, Flag
from dns.opcode import Opcode
from dns.rcode import Rcode
from dns.flags import Flag
from dns.flags import EDNSFlag
import strictyaml
import snscrape
from mockdns.parser.actor_abc import Checker, CheckerMismatch, Generator, Matcher, Actor
from mockdns.parser.schema import DNSQuestion, DNSRecord, ListOrCommaSeparated, OneOrListOf
from mockdns.contrib.matchpart import matchpart
from mockdns.parser.actor_abc import Actor, Checker, CheckerMismatch, Generator, Matcher
from mockdns.parser.utils import (
dns_question,
dns_record,
list_or_comma_separated,
nonnegative,
one_or_list_of,
)
from mockdns.scenario import MatcherNoResponse
......@@ -20,39 +23,51 @@ class _Matchpart(Actor):
@staticmethod
def schema():
return strictyaml.Map(
return voluptuous.Schema(
{
"criteria": ListOrCommaSeparated(strictyaml.Enum(matchpart.MATCH.keys())),
strictyaml.Optional("adjust", default=[]): ListOrCommaSeparated(strictyaml.Enum(_Matchpart.ADJUST_OPTIONS)),
"message": strictyaml.Map(
{
strictyaml.Optional("id"): strictyaml.Int(),
strictyaml.Optional("opcode", default="QUERY"): strictyaml.Enum(
list(Opcode.__members__.keys())
),
strictyaml.Optional("rcode", default="NOERROR"): strictyaml.Enum(
list(Rcode.__members__.keys())
),
strictyaml.Optional("flags", default=["QR"]): ListOrCommaSeparated(
strictyaml.Enum(list(Flag.__members__.keys()))
),
strictyaml.Optional("eflags", default=[]): ListOrCommaSeparated(
strictyaml.Enum(list(EDNSFlag.__members__.keys()))
),
strictyaml.Optional("question", default=[]): OneOrListOf(DNSQuestion()),
strictyaml.Optional("answer", default=[]): OneOrListOf(DNSRecord()),
strictyaml.Optional("authority", default=[]): OneOrListOf(DNSRecord()),
strictyaml.Optional("additional", default=[]): OneOrListOf(DNSRecord()),
strictyaml.Optional("edns", default={"version": 0, "payload": 4096}): strictyaml.Map(
{
strictyaml.Optional("version", default=0): strictyaml.Int(),
strictyaml.Optional("payload", default=4096): strictyaml.Int(),
}
)
}
"criteria": list_or_comma_separated(
voluptuous.In(matchpart.MATCH.keys())
),
voluptuous.Optional("adjust", default=[]): list_or_comma_separated(
voluptuous.In(_Matchpart.ADJUST_OPTIONS)
),
}
"message": {
voluptuous.Optional("id"): voluptuous.All(
int, voluptuous.Range(min=0)
),
voluptuous.Optional("opcode", default="QUERY"): voluptuous.In(
Opcode.__members__
),
voluptuous.Optional("rcode", default="NOERROR"): voluptuous.In(
Rcode.__members__
),
voluptuous.Optional(
"flags", default=["QR"]
): list_or_comma_separated(voluptuous.In(Flag.__members__)),
voluptuous.Optional("eflags", default=[]): list_or_comma_separated(
voluptuous.In(EDNSFlag.__members__)
),
voluptuous.Optional("question", default=[]): one_or_list_of(
dns_question
),
voluptuous.Optional("answer", default=[]): one_or_list_of(
dns_record
),
voluptuous.Optional("authority", default=[]): one_or_list_of(
dns_record
),
voluptuous.Optional("additional", default=[]): one_or_list_of(
dns_record
),
voluptuous.Optional(
"edns", default={"version": 0, "payload": 4096}
): {
voluptuous.Optional("version", default=0): nonnegative(int),
voluptuous.Optional("payload", default=4096): nonnegative(int),
},
},
},
required=True,
)
def set_up(self):
......@@ -63,19 +78,24 @@ class _Matchpart(Actor):
self._message.set_opcode(dns.opcode.from_text(parsed["opcode"]))
self._message.set_rcode(dns.rcode.from_text(parsed["rcode"]))
self._message.flags = sum([dns.flags.from_text(f) for f in set(parsed["flags"])])
self._message.flags = sum(
[dns.flags.from_text(f) for f in set(parsed["flags"])]
)
for section in ["question", "answer", "authority", "additional"]:
if isinstance(parsed[section], dns.rrset.RRset):
getattr(self._message, section).append(parsed[section])
else:
for record in parsed[section]:
getattr(self._message, section).append(record)
for record in parsed[section]:
getattr(self._message, section).append(record)
eflags = sum([dns.flags.edns_from_text(f) for f in set(parsed["eflags"])])
self._message.use_edns(edns=parsed["edns"]["version"], ednsflags=eflags, payload=parsed["edns"]["payload"])
self._message.use_edns(
edns=parsed["edns"]["version"],
ednsflags=eflags,
payload=parsed["edns"]["payload"],
)
def _adjust_message(self, source: dns.message.Message, destination: dns.message.Message) -> None:
def _adjust_message(
self, source: dns.message.Message, destination: dns.message.Message
) -> None:
if "do_not_answer" in self.options["adjust"]:
# FIXME: It's kind of weird to have this as adjust field but it was like this
# in the old Deckard, so let's leave it that way for now
......@@ -90,7 +110,6 @@ class _Matchpart(Actor):
if "copy_id" in self.options["adjust"]:
destination.id = source.id
def match(self, query: dns.message.Message) -> Optional[dns.message.Message]:
for criterion in self.options["criteria"]:
try:
......@@ -115,6 +134,7 @@ class _Matchpart(Actor):
# Matcher and Checker both implement __call__ so in order to reuse code,
# we subclass _Matchpart like this:
class Matchpart(_Matchpart, Matcher):
pass
......
options:
adjust: copy_id, copy_query
criteria: qname, qtype
message:
id: 12
flags: QR, AD, AD
eflags: DO
question: cz. IN NS
answer:
- cz. 1 IN NS a.ns.nic.cz
- cz. 1 IN NS b.ns.nic.cz
- cz. 1 IN NS c.ns.nic.cz
additional:
- a.ns.nic.cz 1 IN A 1.1.1.1
authority: tvoje.mama 69 IN A 69.69.69.69
edns:
payload: 6969
\ No newline at end of file
from typing import Iterable
from dns.message import make_query, make_response, Message
from strictyaml import Int, Map, Str
import voluptuous
from mockdns.parser.actor_abc import Checker, Generator, Matcher
from mockdns.parser.schema import DNSQuestion, ListOrCommaSeparated
from mockdns.parser.utils import dns_question
class EchoMatcher(Matcher):
@staticmethod
def schema() -> Map:
return Map({"a": Str(), "b": Str()})
def schema() -> voluptuous.Schema:
return voluptuous.Schema({"a": str, "b": str})
def match(self, query: Message) -> Message:
return make_response(query)
......@@ -18,8 +18,8 @@ class EchoMatcher(Matcher):
class PermissiveChecker(Checker):
@staticmethod
def schema() -> Map:
return Map({"a": Str(), "b": Str()})
def schema() -> voluptuous.Schema:
return voluptuous.Schema({"a": str, "b": str})
def check(self, query: Message, answer: Message) -> None:
return None
......@@ -27,8 +27,8 @@ class PermissiveChecker(Checker):
class Static(Generator):
@staticmethod
def schema() -> Map:
return Map({"question": DNSQuestion()})
def schema() -> voluptuous.Schema:
return voluptuous.Schema({"question": dns_question})
def generate(self) -> Iterable[Message]:
q = make_query(".", 0)
......
import ipaddress
import json
import os.path
from typing import Any, cast, Dict, List, Mapping, Optional, Type, Tuple, Union
import strictyaml
from mockdns.parser.actor_abc import Actor, Checker, Generator, Matcher
from mockdns.parser.actors import * # pylint: disable=unused-wildcard-import
from mockdns.scenario import Config, ServerGroup, Scenario, Step, Time, RTT
from mockdns.parser.schema import SCHEMA, ACTOR_STEP_SCHEMA, TIME_ADD_SCHEMA, TIME_SET_SCHEMA
MATCHERS = {cls.__name__: cls for cls in Matcher.__subclasses__()}
CHECKERS = {cls.__name__: cls for cls in Checker.__subclasses__()}
GENERATORS = {cls.__name__: cls for cls in Generator.__subclasses__()}
def _get_actor(parsed_actor: Union[strictyaml.YAML, Dict[str, Any]], superclass: str, subclasses: Dict[str, Type]) -> Actor:
try:
type_name = parsed_actor["type"]
cls = subclasses[type_name]
assert issubclass(cls, Actor)
schema = cls.schema()
except KeyError:
if isinstance(parsed_actor, strictyaml.YAML):
line = parsed_actor.start_line
from dataclasses import dataclass
from typing import Any, Dict, List, Mapping, Optional, Union
import ruamel.yaml
from voluptuous import Invalid
from mockdns.scenario import Scenario, Config, Step, ServerGroup, RTT, Time
from mockdns.parser.schema import SCHEMA, IP
from mockdns.parser.actors import ACTORS
class ValidationError(Exception):
def __init__(self, voluptuous_error: Invalid, data: Dict):
super().__init__()
self.data = data
self.voluptuous_error = voluptuous_error
def __str__(self):
error = []
error.append(str(self.voluptuous_error))
line_info = self.get_line_info()
if line_info:
error.append(f"near line {line_info.line} collumn {line_info.col}")
if isinstance(self.data, ruamel.yaml.comments.CommentedMap):
value = self.data.mlget(self.voluptuous_error.path, list_ok=True)
else:
line = parsed_actor["__line_number"]
raise RuntimeError(
f"Unknown {superclass} type {type_name} at line {line}, make sure there's a class of this name inhereting from Matcher in some .py file in parser/actors directory."
)
if isinstance(parsed_actor, strictyaml.YAML):
parsed_actor["options"].revalidate(schema)
return cls(parsed_actor)
else:
return cls.from_dict(parsed_actor)
value = None
if not isinstance(value, (ruamel.yaml.comments.CommentedBase, list, dict)):
error.append(f"in '{value}'")
return " ".join(error)
def get_line_info(self) -> Optional[ruamel.yaml.comments.LineCol]:
cur = self.data
lc = None
for part in self.voluptuous_error.path:
try:
lc = cur.lc # type: ignore
cur = cur[part]
except (AttributeError, KeyError):
return lc
return lc
class ParsedConfig(Config):
def to_dict(self) -> Dict[str, Any]:
return {} # Not implemented for now
def get_actor(actor_type: str, actor_dict: Dict[str, Any]):
return ACTORS[actor_type][actor_dict["type"]](actor_dict["options"])
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "ParsedConfig":
return cls()
class ParsedConfig(Config):
def __init__(self, d: Dict[str, Any]):
self._dict = d
super().__init__(d["data"])
class ParsedRTT(RTT):
def to_dict(self):
return self.__dict__
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "ParsedRTT":
return cls(d["mean"], d["std"], d["loss"])
class ParsedRTT(RTT):
def __init__(self, d: Dict[str, Optional[float]]):
self._dict = d
super().__init__(d.get("mean"), d.get("std"), d.get("loss"))
class ParsedServerGroup(ServerGroup):
def to_dict(self):
return {
"step_range": list(self.step_range),
"addresses": [str(a) for a in self.addresses],
"rtts": {str(a): rtt.to_dict() for a, rtt in self.rtts.items()} if self.rtts else None,
"matchers": [m.to_dict() for m in self.matchers]
}
def __init__(self, d: Dict[str, Any]):
self._dict = d
step_range = (d.get("first_step"), d.get("last_step"))
addresses = d["addresses"]
parsed_rtts = d.get("rtts")
if parsed_rtts is not None:
rtts: Optional[Mapping[IP, RTT]] = {
ip: ParsedRTT(rtt) for ip, rtt in parsed_rtts.values
}
else:
rtts = None
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "ParsedServerGroup":
return cls(
cast(Tuple[Optional[int], Optional[int]], tuple(d["step_range"])),
[ipaddress.ip_address(ip) for ip in d["addresses"]],
{ipaddress.ip_address(ip): ParsedRTT.from_dict(rtt) for ip, rtt in d["rtts"].items()} if d["rtts"] else None,
[_get_actor(m, "matcher", MATCHERS).from_dict(m) for m in d["matchers"]]
)
matchers = [
get_actor("matcher", m_dict) for m_dict in d.get("matchers", default=[])
]
super().__init__(step_range, addresses, rtts, matchers)
class ParsedStep(Step):
def to_dict(self):
return {
"id": self.step_id,
"generator": self.generator.to_dict(),
"checker": self.checker.to_dict()
}
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "ParsedStep":
return cls(
class ParsedStep(Step):
def __init__(self, d: Dict[str, Any]):
self._dict = d
super().__init__(
d["id"],
_get_actor(d, "generator", GENERATORS).from_dict(d["generator"]),
_get_actor(d, "checker", CHECKERS).from_dict(d["checker"])
get_actor("generator", d["generator"]),
get_actor("checker", d["checker"]),
)
class ParsedTime(Time):
def to_dict(self):
return {
"id": self.step_id,
"time_add" if self.action == Time.TimeAction.ADD else "time_set": self.value
}
</