Changed to sets

exclude_badExits.py

@@ -146,6 +146,7 @@ lKNOWN_NODNS = [
     'nx42.de',
     'ormycloud.org',
     'plied-privacy.net',
+    'redacted.org',
     'rification-for-nusenu.net',
     'sv.ch',
     'thingtohide.nl',
@@ -179,8 +180,7 @@ def lYamlBadNodes(sFile,
     if not yaml: return l
     if os.path.exists(sFile):
         with open(sFile, 'rt') as oFd:
-            o = yaml.safe_load(oFd)
+            oBAD_NODES = yaml.safe_load(oFd)
-            oBAD_NODES = o
 
         # BROKEN
 #    root = 'ExcludeNodes'
@@ -190,10 +190,10 @@ def lYamlBadNodes(sFile,
 
     global lKNOWN_NODNS
     root = 'ExcludeDomains'
-    if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]:
+    if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
-        o[oBAD_ROOT][root] = lKNOWN_NODNS
+        oBAD_NODES[oBAD_ROOT][root] = lKNOWN_NODNS
     else:
-        lKNOWN_NODNS = o[oBAD_ROOT][root]
+        lKNOWN_NODNS = oBAD_NODES[oBAD_ROOT][root]
     return l
 
 oGOOD_NODES = {}
@@ -261,9 +261,8 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
 
     if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
         aCachedContact = aTRUST_DB_INDEX[fp]
-        if aCachedContact['email'] = a['email']:
+        if aCachedContact['email'] == a['email']:
             return aCachedContact
-        
        
     if 'url' not in keys:
         if 'uri' not in keys:
@@ -281,15 +280,16 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
         
     # domain should be a unique key for contacts
     domain = a['url'][8:].strip('/')
+    if domain in lKNOWN_NODNS:
+        LOG.warn(f"{domain} in lKNOWN_NODNS")
+        return {}
     try:
         ip = sTorResolve(domain)
     except Exception as e:
+        ip = ''
+    if ip == '':
         try:
             lpair = getaddrinfo(domain, 443)
-        except (socket.gaierror, ) as e:
-            LOG.debug("{e}")
-            lpair = None
-            lKNOWN_NODNS.append(domain)
         except Exception as e:
             LOG.warn("{e}")
             lpair = None
@@ -479,31 +479,35 @@ def oMainArgparser(_=None):
                         help="Write the proof data of the included nodes to a YAML file")
     return parser
 
-def vwrite_badnodes(oArgs, oBAD_NODES):
+def vwrite_badnodes(oArgs, oBAD_NODES, slen):
     if oArgs.bad_nodes:
         tmp = oArgs.bad_nodes +'.tmp'
         bak = oArgs.bad_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
             yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
-            LOG.info(f"Wrote {len(list(oBAD_NODES.keys()))}  to {oArgs.bad_nodes}")
+            LOG.info(f"Wrote {slen}  to {oArgs.bad_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.bad_nodes):
             os.rename(oArgs.bad_nodes, bak)
         os.rename(tmp, oArgs.bad_nodes)
         
-def vwrite_goodnodes(oArgs, oGOOD_NODES):
+def vwrite_goodnodes(oArgs, oGOOD_NODES, slen):
     if oArgs.good_nodes:
         tmp = oArgs.good_nodes +'.tmp'
         bak = oArgs.good_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
             yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
-            LOG.info(f"Wrote {len(list(oGOOD_NODES.keys()))} good nodes to {oArgs.good_nodes}")
+            LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.good_nodes):
             os.rename(oArgs.good_nodes, bak)
         os.rename(tmp, oArgs.good_nodes)
         
 def iMain(lArgs):
+    global aTRUST_DB
+    global aTRUST_DB_INDEX
+    
+    global lKNOWN_NODNS
     parser = oMainArgparser()
     oArgs = parser.parse_args(lArgs)
 
@@ -515,9 +519,6 @@ def iMain(lArgs):
     if sFile and os.path.exists(sFile):
         icheck_torrc(sFile, oArgs)
 
-    global aTRUST_DB
-    global aTRUST_DB_INDEX
-    
     sFile = oArgs.proof_output
     if sFile and os.path.exists(sFile):
         try:
@@ -525,14 +526,14 @@ def iMain(lArgs):
                 aTRUST_DB = yaml.safe_load(oFd)
             # reverse lookup of fps to contacts
             # but...
-            for k,v in aTRUST_DB:
+            for k,v in aTRUST_DB.items():
                 aTRUST_DB_INDEX[k] = v
                 if 'fps' in aTRUST_DB[k].keys():
                     for fp in aTRUST_DB[k]['fps']:
                         aTRUST_DB_INDEX[fp] = v
                         
         except Exception as e:
-            LOG.warn(f"Error reading YAML TrustDB {sFile} {e}")
+            LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
 
     if os.path.exists(oArgs.proxy_ctl):
         controller = oMakeController(sSock=oArgs.proxy_ctl)
@@ -576,8 +577,6 @@ def iMain(lArgs):
 
     LOG.info(f"lYamlBadNodes {len(exit_excludelist)}")
 
-    relays = controller.get_server_descriptors()
-
     tProofGoodFps = set()
     iDnsContact = 0
     lBadContactUrls = []
@@ -586,6 +585,8 @@ def iMain(lArgs):
     aProofUri = {}
     lConds = oArgs.contact.split(',')
     iR = 0
+    
+    relays = controller.get_server_descriptors()
     for relay in relays:
         iR += 1
         if not is_valid_fingerprint(relay.fingerprint):
@@ -610,27 +611,36 @@ def iMain(lArgs):
               relay.fingerprint in aTRUST_DB[relay.fingerprint]['fps']:
                 tProofGoodFps.add(relay.fingerprint)
             continue
-                
+
-        if relay.contact and b'dns-rsa' in relay.contact.lower():
+        if type(relay.contact) == bytes:
+            # dunno
             relay.contact = str(relay.contact, 'UTF-8')
-            c = relay.contact.lower()
-            i = c.find('url:')
-            if i >=0:
-                c = c[i+4:]
-            i = c.find(' ')
-            if i >=0:
-                c = c[:i]
-            if c in lKNOWN_NODNS:
-                LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
-                exit_excludelist.append(relay.fingerprint)
-                continue
             
-            LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}")
+        if ('Empty' in lConds and not relay.contact) or \
-            iDnsContact += 1
+           ('NoEmail' in lConds and relay.contact and not '@' in relay.contact):
+            exit_excludelist.append(relay.fingerprint)
             continue
         
-        if relay.contact and b'proof:uri-rsa' in relay.contact.lower():
+        if not relay.contact:
-            relay.contact = str(relay.contact, 'UTF-8')
+            # should be unreached 'Empty' should always be in lConds
+            continue
+        
+        c = relay.contact.lower()
+        i = c.find('url:')
+        if i >=0: c = c[i+4:]
+        i = c.find(' ')
+        if i >=0: c = c[:i]
+        domain = c.replace('https://', '').replace('http://', '').strip('/')
+        
+        if domain in lKNOWN_NODNS:
+            LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {domain} {sofar}")
+            exit_excludelist.append(relay.fingerprint)
+        
+        elif 'dns-rsa' in relay.contact.lower():            
+            LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{domain} {sofar}")
+            iDnsContact += 1
+        
+        elif 'proof:uri-rsa' in relay.contact.lower():
             a = aParseContact(relay.contact, relay.fingerprint)
             if not a:
                 LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
@@ -657,7 +667,7 @@ def iMain(lArgs):
                                host=oArgs.proxy_host,
                                port=oArgs.proxy_port)
             
-            if not b['fps'] or not b['url']:
+            if not b or not 'fps' in b or not b['fps'] or not b['url']:
                 LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
                 # If it's giving contact info that doesnt check out
                 # it could be a bad exit with fake contact info
@@ -687,12 +697,7 @@ def iMain(lArgs):
                 with open(proof_output_tmp, 'wt') as oFYaml:
                     yaml.dump(aProofUri, indent=2, stream=oFYaml)
                     oFYaml.close()
-            continue
         
-        if ('Empty' in lConds and not relay.contact) or \
-           ('NoEmail' in lConds and relay.contact and not b'@' in relay.contact):
-            exit_excludelist.append(relay.fingerprint)
-
     exit_excludelist = list(set(exit_excludelist).difference(set(lGoodOverrideSet)))
 
     if oArgs.proof_output and aProofUri:
@@ -721,13 +726,12 @@ def iMain(lArgs):
 
     global oBAD_NODES
     oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = exit_excludelist
-    global lKNOWN_NODNS
     oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
-    vwrite_badnodes(oArgs, oBAD_NODES)
+    vwrite_badnodes(oArgs, oBAD_NODES, str(len(exit_excludelist)))
     
     global oGOOD_NODES
     oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = tProofGoodFps
-    vwrite_goodnodes(oArgs, oGOOD_NODES)
+    vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(tProofGoodFps)))
         
     retval = 0
     try:

https_adapter.py

@@ -326,5 +326,6 @@ def match_hostname(cert, hostname):
         raise CertificateError(
             "no appropriate commonName or subjectAltName fields were found"
         )
-from urllib3.util.ssl_match_hostname = match_hostname
+
+urllib3.util.ssl_match_hostname = match_hostname
 

support_onions.py

@@ -153,8 +153,8 @@ def getaddrinfo(sHost, sPort):
         lPair = lElts[0][-1]
         assert len(lPair) == 2, repr(lPair)
         assert type(lPair[1]) == int, repr(lPair)
-    except Exception as e:
+    except (socket.gaierror, OSError, BaseException) as e:
-        LOG.exception(e)
+        LOG.error(e)
         return None
     return lPair