|
|
|
@ -146,6 +146,7 @@ lKNOWN_NODNS = [
|
|
|
|
|
'nx42.de',
|
|
|
|
|
'ormycloud.org',
|
|
|
|
|
'plied-privacy.net',
|
|
|
|
|
'redacted.org',
|
|
|
|
|
'rification-for-nusenu.net',
|
|
|
|
|
'sv.ch',
|
|
|
|
|
'thingtohide.nl',
|
|
|
|
@ -179,8 +180,7 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
if not yaml: return l
|
|
|
|
|
if os.path.exists(sFile):
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
o = yaml.safe_load(oFd)
|
|
|
|
|
oBAD_NODES = o
|
|
|
|
|
oBAD_NODES = yaml.safe_load(oFd)
|
|
|
|
|
|
|
|
|
|
# BROKEN
|
|
|
|
|
# root = 'ExcludeNodes'
|
|
|
|
@ -190,10 +190,10 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
root = 'ExcludeDomains'
|
|
|
|
|
if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]:
|
|
|
|
|
o[oBAD_ROOT][root] = lKNOWN_NODNS
|
|
|
|
|
if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
|
|
|
|
|
oBAD_NODES[oBAD_ROOT][root] = lKNOWN_NODNS
|
|
|
|
|
else:
|
|
|
|
|
lKNOWN_NODNS = o[oBAD_ROOT][root]
|
|
|
|
|
lKNOWN_NODNS = oBAD_NODES[oBAD_ROOT][root]
|
|
|
|
|
return l
|
|
|
|
|
|
|
|
|
|
oGOOD_NODES = {}
|
|
|
|
@ -261,9 +261,8 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
|
|
|
|
|
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
|
|
|
|
|
aCachedContact = aTRUST_DB_INDEX[fp]
|
|
|
|
|
if aCachedContact['email'] = a['email']:
|
|
|
|
|
if aCachedContact['email'] == a['email']:
|
|
|
|
|
return aCachedContact
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if 'url' not in keys:
|
|
|
|
|
if 'uri' not in keys:
|
|
|
|
@ -281,15 +280,16 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
|
|
|
|
|
# domain should be a unique key for contacts
|
|
|
|
|
domain = a['url'][8:].strip('/')
|
|
|
|
|
if domain in lKNOWN_NODNS:
|
|
|
|
|
LOG.warn(f"{domain} in lKNOWN_NODNS")
|
|
|
|
|
return {}
|
|
|
|
|
try:
|
|
|
|
|
ip = sTorResolve(domain)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
ip = ''
|
|
|
|
|
if ip == '':
|
|
|
|
|
try:
|
|
|
|
|
lpair = getaddrinfo(domain, 443)
|
|
|
|
|
except (socket.gaierror, ) as e:
|
|
|
|
|
LOG.debug("{e}")
|
|
|
|
|
lpair = None
|
|
|
|
|
lKNOWN_NODNS.append(domain)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn("{e}")
|
|
|
|
|
lpair = None
|
|
|
|
@ -479,31 +479,35 @@ def oMainArgparser(_=None):
|
|
|
|
|
help="Write the proof data of the included nodes to a YAML file")
|
|
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
def vwrite_badnodes(oArgs, oBAD_NODES):
|
|
|
|
|
def vwrite_badnodes(oArgs, oBAD_NODES, slen):
|
|
|
|
|
if oArgs.bad_nodes:
|
|
|
|
|
tmp = oArgs.bad_nodes +'.tmp'
|
|
|
|
|
bak = oArgs.bad_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {len(list(oBAD_NODES.keys()))} to {oArgs.bad_nodes}")
|
|
|
|
|
LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.bad_nodes):
|
|
|
|
|
os.rename(oArgs.bad_nodes, bak)
|
|
|
|
|
os.rename(tmp, oArgs.bad_nodes)
|
|
|
|
|
|
|
|
|
|
def vwrite_goodnodes(oArgs, oGOOD_NODES):
|
|
|
|
|
def vwrite_goodnodes(oArgs, oGOOD_NODES, slen):
|
|
|
|
|
if oArgs.good_nodes:
|
|
|
|
|
tmp = oArgs.good_nodes +'.tmp'
|
|
|
|
|
bak = oArgs.good_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {len(list(oGOOD_NODES.keys()))} good nodes to {oArgs.good_nodes}")
|
|
|
|
|
LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.good_nodes):
|
|
|
|
|
os.rename(oArgs.good_nodes, bak)
|
|
|
|
|
os.rename(tmp, oArgs.good_nodes)
|
|
|
|
|
|
|
|
|
|
def iMain(lArgs):
|
|
|
|
|
global aTRUST_DB
|
|
|
|
|
global aTRUST_DB_INDEX
|
|
|
|
|
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
parser = oMainArgparser()
|
|
|
|
|
oArgs = parser.parse_args(lArgs)
|
|
|
|
|
|
|
|
|
@ -515,9 +519,6 @@ def iMain(lArgs):
|
|
|
|
|
if sFile and os.path.exists(sFile):
|
|
|
|
|
icheck_torrc(sFile, oArgs)
|
|
|
|
|
|
|
|
|
|
global aTRUST_DB
|
|
|
|
|
global aTRUST_DB_INDEX
|
|
|
|
|
|
|
|
|
|
sFile = oArgs.proof_output
|
|
|
|
|
if sFile and os.path.exists(sFile):
|
|
|
|
|
try:
|
|
|
|
@ -525,14 +526,14 @@ def iMain(lArgs):
|
|
|
|
|
aTRUST_DB = yaml.safe_load(oFd)
|
|
|
|
|
# reverse lookup of fps to contacts
|
|
|
|
|
# but...
|
|
|
|
|
for k,v in aTRUST_DB:
|
|
|
|
|
for k,v in aTRUST_DB.items():
|
|
|
|
|
aTRUST_DB_INDEX[k] = v
|
|
|
|
|
if 'fps' in aTRUST_DB[k].keys():
|
|
|
|
|
for fp in aTRUST_DB[k]['fps']:
|
|
|
|
|
aTRUST_DB_INDEX[fp] = v
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"Error reading YAML TrustDB {sFile} {e}")
|
|
|
|
|
LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
|
|
|
|
|
|
|
|
|
|
if os.path.exists(oArgs.proxy_ctl):
|
|
|
|
|
controller = oMakeController(sSock=oArgs.proxy_ctl)
|
|
|
|
@ -576,8 +577,6 @@ def iMain(lArgs):
|
|
|
|
|
|
|
|
|
|
LOG.info(f"lYamlBadNodes {len(exit_excludelist)}")
|
|
|
|
|
|
|
|
|
|
relays = controller.get_server_descriptors()
|
|
|
|
|
|
|
|
|
|
tProofGoodFps = set()
|
|
|
|
|
iDnsContact = 0
|
|
|
|
|
lBadContactUrls = []
|
|
|
|
@ -586,6 +585,8 @@ def iMain(lArgs):
|
|
|
|
|
aProofUri = {}
|
|
|
|
|
lConds = oArgs.contact.split(',')
|
|
|
|
|
iR = 0
|
|
|
|
|
|
|
|
|
|
relays = controller.get_server_descriptors()
|
|
|
|
|
for relay in relays:
|
|
|
|
|
iR += 1
|
|
|
|
|
if not is_valid_fingerprint(relay.fingerprint):
|
|
|
|
@ -610,27 +611,36 @@ def iMain(lArgs):
|
|
|
|
|
relay.fingerprint in aTRUST_DB[relay.fingerprint]['fps']:
|
|
|
|
|
tProofGoodFps.add(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if relay.contact and b'dns-rsa' in relay.contact.lower():
|
|
|
|
|
|
|
|
|
|
if type(relay.contact) == bytes:
|
|
|
|
|
# dunno
|
|
|
|
|
relay.contact = str(relay.contact, 'UTF-8')
|
|
|
|
|
c = relay.contact.lower()
|
|
|
|
|
i = c.find('url:')
|
|
|
|
|
if i >=0:
|
|
|
|
|
c = c[i+4:]
|
|
|
|
|
i = c.find(' ')
|
|
|
|
|
if i >=0:
|
|
|
|
|
c = c[:i]
|
|
|
|
|
if c in lKNOWN_NODNS:
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}")
|
|
|
|
|
iDnsContact += 1
|
|
|
|
|
if ('Empty' in lConds and not relay.contact) or \
|
|
|
|
|
('NoEmail' in lConds and relay.contact and not '@' in relay.contact):
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if relay.contact and b'proof:uri-rsa' in relay.contact.lower():
|
|
|
|
|
relay.contact = str(relay.contact, 'UTF-8')
|
|
|
|
|
if not relay.contact:
|
|
|
|
|
# should be unreached 'Empty' should always be in lConds
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
c = relay.contact.lower()
|
|
|
|
|
i = c.find('url:')
|
|
|
|
|
if i >=0: c = c[i+4:]
|
|
|
|
|
i = c.find(' ')
|
|
|
|
|
if i >=0: c = c[:i]
|
|
|
|
|
domain = c.replace('https://', '').replace('http://', '').strip('/')
|
|
|
|
|
|
|
|
|
|
if domain in lKNOWN_NODNS:
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {domain} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
|
|
|
|
|
elif 'dns-rsa' in relay.contact.lower():
|
|
|
|
|
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{domain} {sofar}")
|
|
|
|
|
iDnsContact += 1
|
|
|
|
|
|
|
|
|
|
elif 'proof:uri-rsa' in relay.contact.lower():
|
|
|
|
|
a = aParseContact(relay.contact, relay.fingerprint)
|
|
|
|
|
if not a:
|
|
|
|
|
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
|
|
|
|
@ -657,7 +667,7 @@ def iMain(lArgs):
|
|
|
|
|
host=oArgs.proxy_host,
|
|
|
|
|
port=oArgs.proxy_port)
|
|
|
|
|
|
|
|
|
|
if not b['fps'] or not b['url']:
|
|
|
|
|
if not b or not 'fps' in b or not b['fps'] or not b['url']:
|
|
|
|
|
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
|
|
|
|
|
# If it's giving contact info that doesnt check out
|
|
|
|
|
# it could be a bad exit with fake contact info
|
|
|
|
@ -687,12 +697,7 @@ def iMain(lArgs):
|
|
|
|
|
with open(proof_output_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aProofUri, indent=2, stream=oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if ('Empty' in lConds and not relay.contact) or \
|
|
|
|
|
('NoEmail' in lConds and relay.contact and not b'@' in relay.contact):
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
|
|
|
|
|
exit_excludelist = list(set(exit_excludelist).difference(set(lGoodOverrideSet)))
|
|
|
|
|
|
|
|
|
|
if oArgs.proof_output and aProofUri:
|
|
|
|
@ -721,13 +726,12 @@ def iMain(lArgs):
|
|
|
|
|
|
|
|
|
|
global oBAD_NODES
|
|
|
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = exit_excludelist
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
|
|
|
|
|
vwrite_badnodes(oArgs, oBAD_NODES)
|
|
|
|
|
vwrite_badnodes(oArgs, oBAD_NODES, str(len(exit_excludelist)))
|
|
|
|
|
|
|
|
|
|
global oGOOD_NODES
|
|
|
|
|
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = tProofGoodFps
|
|
|
|
|
vwrite_goodnodes(oArgs, oGOOD_NODES)
|
|
|
|
|
vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(tProofGoodFps)))
|
|
|
|
|
|
|
|
|
|
retval = 0
|
|
|
|
|
try:
|
|
|
|
|