Verified Commit 30c3b119 authored by Petr Špaček's avatar Petr Špaček
Browse files

domain2ipset: memory optimization

parent 9023d272
Pipeline #48080 failed with stage
in 19 seconds
......@@ -14,6 +14,7 @@ import multiprocessing
import pickle
import random
from typing import Counter, Deque, Dict, Iterable, Set, Tuple
import weakref
import dns.message
import dns.query
......@@ -120,14 +121,16 @@ def retry_candidates(retry_queue: Deque[Tuple[int,, AnIPAddress]],
logging.debug('retrying %s @%s (attempt no. %s)', domain, ip, attempt + 1)
yield (attempt + 1, domain, ip)
def process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset):
def process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset, ipsetcache: Dict[int, Set[AnIPAddress]]):
netstats.record_ip(ip, state)
if state == IP_state.notauth:
return # not authoritative, look for another domain on this NS
elif state == IP_state.timeout:
retry_queue.append((attempt, domain, ip))
elif state == IP_state.ok:
domain2ipset.setdefault(domain, set()).add(ip)
# add IP address to set, re-use existing IP address sets to prevent memory bloat
newset = frozenset((ip, )).union(domain2ipset.get(domain, ())) # type: Set[AnIPAddress]
domain2ipset[domain] = ipsetcache.setdefault(hash(newset), newset)
#if len(ip_done) % 100 == 0:
#'generated output for %s IP addresses', len(ip_done))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment