From ae22d1443774c4256346bed65bf18f80ea1c3b14 Mon Sep 17 00:00:00 2001 From: emdee Date: Wed, 9 Nov 2022 18:43:54 +0000 Subject: [PATCH] Changed to sets --- exclude_badExits.py | 100 +++++++++++++++++++++++--------------------- https_adapter.py | 3 +- support_onions.py | 4 +- 3 files changed, 56 insertions(+), 51 deletions(-) diff --git a/exclude_badExits.py b/exclude_badExits.py index 86761a8..06e64bf 100644 --- a/exclude_badExits.py +++ b/exclude_badExits.py @@ -146,6 +146,7 @@ lKNOWN_NODNS = [ 'nx42.de', 'ormycloud.org', 'plied-privacy.net', + 'redacted.org', 'rification-for-nusenu.net', 'sv.ch', 'thingtohide.nl', @@ -179,8 +180,7 @@ def lYamlBadNodes(sFile, if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - o = yaml.safe_load(oFd) - oBAD_NODES = o + oBAD_NODES = yaml.safe_load(oFd) # BROKEN # root = 'ExcludeNodes' @@ -190,10 +190,10 @@ def lYamlBadNodes(sFile, global lKNOWN_NODNS root = 'ExcludeDomains' - if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]: - o[oBAD_ROOT][root] = lKNOWN_NODNS + if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]: + oBAD_NODES[oBAD_ROOT][root] = lKNOWN_NODNS else: - lKNOWN_NODNS = o[oBAD_ROOT][root] + lKNOWN_NODNS = oBAD_NODES[oBAD_ROOT][root] return l oGOOD_NODES = {} @@ -261,9 +261,8 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050) if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys(): aCachedContact = aTRUST_DB_INDEX[fp] - if aCachedContact['email'] = a['email']: + if aCachedContact['email'] == a['email']: return aCachedContact - if 'url' not in keys: if 'uri' not in keys: @@ -281,15 +280,16 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050) # domain should be a unique key for contacts domain = a['url'][8:].strip('/') + if domain in lKNOWN_NODNS: + LOG.warn(f"{domain} in lKNOWN_NODNS") + return {} try: ip = sTorResolve(domain) except Exception as e: + ip = '' + if ip == '': try: lpair = getaddrinfo(domain, 443) - except (socket.gaierror, ) as e: - LOG.debug("{e}") - lpair = None - lKNOWN_NODNS.append(domain) except Exception as e: LOG.warn("{e}") lpair = None @@ -479,31 +479,35 @@ def oMainArgparser(_=None): help="Write the proof data of the included nodes to a YAML file") return parser -def vwrite_badnodes(oArgs, oBAD_NODES): +def vwrite_badnodes(oArgs, oBAD_NODES, slen): if oArgs.bad_nodes: tmp = oArgs.bad_nodes +'.tmp' bak = oArgs.bad_nodes +'.bak' with open(tmp, 'wt') as oFYaml: yaml.dump(oBAD_NODES, indent=2, stream=oFYaml) - LOG.info(f"Wrote {len(list(oBAD_NODES.keys()))} to {oArgs.bad_nodes}") + LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}") oFYaml.close() if os.path.exists(oArgs.bad_nodes): os.rename(oArgs.bad_nodes, bak) os.rename(tmp, oArgs.bad_nodes) -def vwrite_goodnodes(oArgs, oGOOD_NODES): +def vwrite_goodnodes(oArgs, oGOOD_NODES, slen): if oArgs.good_nodes: tmp = oArgs.good_nodes +'.tmp' bak = oArgs.good_nodes +'.bak' with open(tmp, 'wt') as oFYaml: yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml) - LOG.info(f"Wrote {len(list(oGOOD_NODES.keys()))} good nodes to {oArgs.good_nodes}") + LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}") oFYaml.close() if os.path.exists(oArgs.good_nodes): os.rename(oArgs.good_nodes, bak) os.rename(tmp, oArgs.good_nodes) def iMain(lArgs): + global aTRUST_DB + global aTRUST_DB_INDEX + + global lKNOWN_NODNS parser = oMainArgparser() oArgs = parser.parse_args(lArgs) @@ -515,9 +519,6 @@ def iMain(lArgs): if sFile and os.path.exists(sFile): icheck_torrc(sFile, oArgs) - global aTRUST_DB - global aTRUST_DB_INDEX - sFile = oArgs.proof_output if sFile and os.path.exists(sFile): try: @@ -525,14 +526,14 @@ def iMain(lArgs): aTRUST_DB = yaml.safe_load(oFd) # reverse lookup of fps to contacts # but... - for k,v in aTRUST_DB: + for k,v in aTRUST_DB.items(): aTRUST_DB_INDEX[k] = v if 'fps' in aTRUST_DB[k].keys(): for fp in aTRUST_DB[k]['fps']: aTRUST_DB_INDEX[fp] = v except Exception as e: - LOG.warn(f"Error reading YAML TrustDB {sFile} {e}") + LOG.exception(f"Error reading YAML TrustDB {sFile} {e}") if os.path.exists(oArgs.proxy_ctl): controller = oMakeController(sSock=oArgs.proxy_ctl) @@ -576,8 +577,6 @@ def iMain(lArgs): LOG.info(f"lYamlBadNodes {len(exit_excludelist)}") - relays = controller.get_server_descriptors() - tProofGoodFps = set() iDnsContact = 0 lBadContactUrls = [] @@ -586,6 +585,8 @@ def iMain(lArgs): aProofUri = {} lConds = oArgs.contact.split(',') iR = 0 + + relays = controller.get_server_descriptors() for relay in relays: iR += 1 if not is_valid_fingerprint(relay.fingerprint): @@ -610,27 +611,36 @@ def iMain(lArgs): relay.fingerprint in aTRUST_DB[relay.fingerprint]['fps']: tProofGoodFps.add(relay.fingerprint) continue - - if relay.contact and b'dns-rsa' in relay.contact.lower(): + + if type(relay.contact) == bytes: + # dunno relay.contact = str(relay.contact, 'UTF-8') - c = relay.contact.lower() - i = c.find('url:') - if i >=0: - c = c[i+4:] - i = c.find(' ') - if i >=0: - c = c[:i] - if c in lKNOWN_NODNS: - LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}") - exit_excludelist.append(relay.fingerprint) - continue - LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}") - iDnsContact += 1 + if ('Empty' in lConds and not relay.contact) or \ + ('NoEmail' in lConds and relay.contact and not '@' in relay.contact): + exit_excludelist.append(relay.fingerprint) continue - if relay.contact and b'proof:uri-rsa' in relay.contact.lower(): - relay.contact = str(relay.contact, 'UTF-8') + if not relay.contact: + # should be unreached 'Empty' should always be in lConds + continue + + c = relay.contact.lower() + i = c.find('url:') + if i >=0: c = c[i+4:] + i = c.find(' ') + if i >=0: c = c[:i] + domain = c.replace('https://', '').replace('http://', '').strip('/') + + if domain in lKNOWN_NODNS: + LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {domain} {sofar}") + exit_excludelist.append(relay.fingerprint) + + elif 'dns-rsa' in relay.contact.lower(): + LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{domain} {sofar}") + iDnsContact += 1 + + elif 'proof:uri-rsa' in relay.contact.lower(): a = aParseContact(relay.contact, relay.fingerprint) if not a: LOG.warn(f"{relay.fingerprint} did not parse {sofar}") @@ -657,7 +667,7 @@ def iMain(lArgs): host=oArgs.proxy_host, port=oArgs.proxy_port) - if not b['fps'] or not b['url']: + if not b or not 'fps' in b or not b['fps'] or not b['url']: LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}") # If it's giving contact info that doesnt check out # it could be a bad exit with fake contact info @@ -687,12 +697,7 @@ def iMain(lArgs): with open(proof_output_tmp, 'wt') as oFYaml: yaml.dump(aProofUri, indent=2, stream=oFYaml) oFYaml.close() - continue - if ('Empty' in lConds and not relay.contact) or \ - ('NoEmail' in lConds and relay.contact and not b'@' in relay.contact): - exit_excludelist.append(relay.fingerprint) - exit_excludelist = list(set(exit_excludelist).difference(set(lGoodOverrideSet))) if oArgs.proof_output and aProofUri: @@ -721,13 +726,12 @@ def iMain(lArgs): global oBAD_NODES oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = exit_excludelist - global lKNOWN_NODNS oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS - vwrite_badnodes(oArgs, oBAD_NODES) + vwrite_badnodes(oArgs, oBAD_NODES, str(len(exit_excludelist))) global oGOOD_NODES oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = tProofGoodFps - vwrite_goodnodes(oArgs, oGOOD_NODES) + vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(tProofGoodFps))) retval = 0 try: diff --git a/https_adapter.py b/https_adapter.py index 3fd4b7d..d19fc9c 100644 --- a/https_adapter.py +++ b/https_adapter.py @@ -326,5 +326,6 @@ def match_hostname(cert, hostname): raise CertificateError( "no appropriate commonName or subjectAltName fields were found" ) -from urllib3.util.ssl_match_hostname = match_hostname + +urllib3.util.ssl_match_hostname = match_hostname diff --git a/support_onions.py b/support_onions.py index 55f5d90..a806b31 100644 --- a/support_onions.py +++ b/support_onions.py @@ -153,8 +153,8 @@ def getaddrinfo(sHost, sPort): lPair = lElts[0][-1] assert len(lPair) == 2, repr(lPair) assert type(lPair[1]) == int, repr(lPair) - except Exception as e: - LOG.exception(e) + except (socket.gaierror, OSError, BaseException) as e: + LOG.error(e) return None return lPair