updates

Makefile

@@ -1,12 +1,24 @@
 PREFIX=/usr/local
 PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh
 PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh
+PYTHON_MINOR=`python3 --version 2>&1 | sed -e 's@^.* @@' -e 's@\.[0-9]*$$@@'`
 LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash
 DOCTEST=${LOCAL_DOCTEST}
 MOD=exclude_badExits
 
+all:: README.md exclude_badExits.hlp
+
+exclude_badExits.hlp:: src/exclude_badExits/__main__.py
+	PYTHONPATH=${PWD}/src \
+	${PYTHON_EXE_MSYS} -m exclude_badExits --help > exclude_badExits.hlp
+
+README.md:: src/exclude_badExits/__main__.py
+	PYTHONPATH=${PWD}/src \
+	${PYTHON_EXE_MSYS} -c 'from exclude_badExits.__main__ import __doc__; print(__doc__)' \
+	> README.md
+
 check::
-	sh python3.sh -c "import ${MOD}"
+	sh ${PYTHON_EXE_MSYS} -c "import ${MOD}"
 
 lint::
 	sh .pylint.sh
@@ -16,6 +28,8 @@ install::
 		--no-deps \
 		--target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \
 		--upgrade .
+	sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \
+		${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/*
 
 rsync::
 	bash .rsync.sh

README.md

@@ -42,8 +42,8 @@ Use the GoodNodes/Onions list to list onion services you want the
 Introduction Points whitelisted - these points may change daily
 Look in tor's notice.log for warnings of 'Every introduction point for service'
 
-```--hs_dir``` ```default='/var/lib/tor'``` will make the program 
+```--hs_dir``` ```default='/var/lib/tor'``` will make the program
-parse  the files named ```hostname``` below this dir to find 
+parse  the files named ```hostname``` below this dir to find
 Hidden Services to whitelist.
 
 The Introduction Points can change during the day, so you may want to
@@ -86,9 +86,12 @@ Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
 you want the Introduction Points whitelisted - these points may change daily.
 Look in tor's notice.log for 'Every introduction point for service'
 
-```notice_log``` will parse the notice log for warnings about relays and 
+```notice_log``` will parse the notice log for warnings about relays and
 services that will then  be whitelisted.
 
+```--torrc``` will read a file like /etc/tor/torrc and make some
+suggestions based on what it finds; it will not edit or change the file.
+
 ```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
 
 ```--good_contacts``` will write the contact info as a ciiss dictionary
@@ -129,15 +132,10 @@ To be "good" the ContactInfo must:
 ```--wellknown_output``` will make the program write the well-known files
 (```/.well-known/tor-relay/rsa-fingerprint.txt```) to a directory.
 
-```--torrc_output``` will write a file of the commands that it sends to
+```--relays_output write the download relays in json to a file. The relays
-the Tor controller, so you can include it in a ```/etc/toc/torrc```.
-
-```--relays_output``` write the download relays in json to a file. The relays
 are downloaded from https://onionoo.torproject.org/details
 
-For usage, do ```python3 exclude_badExits.py --help```
+For usage, do ```python3 exclude_badExits.py --help`
+See [exclude_badExits.txt](./exclude_badExits.txt)
 
-See [exclude_badExits.hlp](./exclude_badExits.hlp)
-or there's a doctest file in [exclude_badExits.txt](./exclude_badExits.txt)
 
-Up-to-date code is on https://git.plastiras.org/emdee/exclude_badExits

exclude_badExits.bash

@@ -6,16 +6,22 @@ SOCKS_PORT=9050
 SOCKS_HOST=127.0.0.1
 CAFILE=/etc/ssl/certs/ca-certificates.crt
 # you may have a special python for installed packages
-EXE=`which python3.bash`
+EXE=`which python3.sh`
 MOD=exclude_badExits
 
-[ -f exclude_badExits.hlp -a exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py] || \
+if [ ! -f exclude_badExits.hlp ] || \
+       [ exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py ] ; then
     PYTHONPATH=$PWD/src \
 	$EXE src/${MOD}/exclude_badExits.py --help > exclude_badExits.hlp
-[ -f  README.md -a README.md -nt src/exclude_badExits/exclude_badExits.py] || \
+fi
+
+if [ ! -f  README.md ] || \
+       [ README.md -nt src/exclude_badExits/exclude_badExits.py ] ; then
     PYTHONPATH=$PWD/src \
 	$EXE -c 'from exclude_badExits.exclude_badExits import __doc__; print(__doc__)' \
 	> README.md
+fi
+
 # an example of running exclude_badExits with full debugging
 # expected to 20 minutes or so
 declare -a LARGS

pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
 dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license']
 
 [project.scripts]
-exclude_badExits = "exclude_badExits.exclude_badExits:iMain"
+exclude_badExits = "exclude_badExits.__main__:iMain"
 
 [tool.setuptools.dynamic]
 version = {attr = "exclude_badExits.__version__"} 

src/exclude_badExits/__init__.py

@@ -1,2 +1,8 @@
+# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
 
 __version__ = "1.0.0"
+
+__all__ = [
+           'exclude_utils',
+           'torcontactinfo',
+           'trustor_poc']

src/exclude_badExits/__main__.py

@@ -197,30 +197,29 @@ from toxygen_wrapper.tests.support_onions import (
     bAreWeConnected, icheck_torrc, lIntroductionPoints,
     yKNOWN_NODNS, zResolveDomain)
 
-from trustor_poc import TrustorError, idns_validate
+from exclude_badExits.trustor_poc import TrustorError, idns_validate
-try:
+if False:
-    import xxxhttpx
+    import httpx
     import asyncio
-    from trustor_poc import oDownloadUrlHttpx
+    from exclude_badExits.trustor_poc import oDownloadUrlHttpx
-except:
+else:
     httpx = None
-    from trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
+    from exclude_badExits.trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
 
-try:
+from exclude_badExits.torcontactinfo import TorContactInfoParser
-    from torcontactinfo import TorContactInfoParser
-    oPARSER = TorContactInfoParser()
-except ImportError:
-    oPARSER = None
 
-from exclude_utils import (aCleanContact, sCleanEmail, aParseContact,
+from exclude_badExits.exclude_utils import (
-                           oStemController, oMainArgparser,
+    aCleanContact, sCleanEmail, aParseContact,
-                           vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
+    oStemController, oMainArgparser,
-                           vwritefinale, vsetup_logging)
+    vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
+    vwritefinale, vsetup_logging )
 
 warnings.filterwarnings('ignore')
 global LOG
 LOG = logging.getLogger()
 
+oPARSER = TorContactInfoParser()
+
 aGOOD_CONTACTS_DB = {}
 aGOOD_CONTACTS_FPS = {}
 aBAD_CONTACTS_DB = {}
@@ -555,10 +554,10 @@ def aContactFps(oargs, a, fp, o, domain):
     if oargs.wellknown_output:
         sdir = os.path.join(oargs.wellknown_output, domain,
                             '.well-known', 'tor-relay')
+        sfile = os.path.join(sdir, "rsa-fingerprint.txt")
         try:
             if not os.path.isdir(sdir):
                 os.makedirs(sdir)
-            sfile = os.path.join(sdir, "rsa-fingerprint.txt")
             with open(sfile, 'wt') as oFd:
                 oFd.write(data)
         except Exception as e:
@@ -665,6 +664,7 @@ def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0):
 
     return True
 
+lNOT_IN_RELAYS_DB = []
 def bCheckFp(relay, sofar, lConds, texclude_set):
     global aGOOD_CONTACTS_DB
     global aGOOD_CONTACTS_FPS
@@ -853,7 +853,6 @@ def iMain(lArgs):
     iFakeContact = 0
     iTotalContacts = 0
     aBadContacts = {}
-    lNOT_IN_RELAYS_DB = []
     iR = 0
     relays = controller.get_server_descriptors()
     lqueue = []

src/exclude_badExits/exclude_utils.py

@@ -11,7 +11,7 @@ import logging
 import warnings
 global LOG
 
-from toxygen_wrapper.tests.support_onions import (woGetStemController,
+from toxygen_wrapper.tests.support_onions import (oGetStemController,
                                 vwait_for_controller,)
 
 try:
@@ -20,8 +20,6 @@ try:
     yaml.indent(mapping=2, sequence=2)
     safe_load = yaml.load
 except:
-    yaml = None
-if yaml is None:
     try:
         import yaml
         safe_load = yaml.safe_load
@@ -36,6 +34,15 @@ try:
 except ImportError:
     coloredlogs = False
 
+lMORONS = ['hoster:Quintex Alliance Consulting ']
+oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
+lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
+lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
+          'sandbox', 'offlinemasterkey']
+lEMAILS = ['abuse', 'email']
+
+ETC_DIR = '/usr/local/etc/tor/yaml'
+
 def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict:
     # cleanups
     for elt in lINTS:
@@ -78,15 +85,6 @@ def sCleanEmail(s, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> str:
         s = s.replace(elt, '?')
     return s
 
-lMORONS = ['hoster:Quintex Alliance Consulting ']
-oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
-lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
-lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
-          'sandbox', 'offlinemasterkey']
-lEMAILS = ['abuse', 'email']
-
-ETC_DIR = '/usr/local/etc/tor/yaml'
-
 def oStemController(oargs, sEXCLUDE_EXIT_GROUP):
     if os.path.exists(oargs.proxy_ctl):
         controller = oGetStemController(log_level=oargs.log_level,
@@ -261,7 +259,6 @@ def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
         kwargs = dict(level=log_level,
                     force=True,
                     format='%(levelname)s %(message)s')
-
         logging.basicConfig(**kwargs)
         if add and logfile:
             oHandler = logging.StreamHandler(stream)

src/exclude_badExits/torcontactinfo.py

@@ -29,7 +29,7 @@ import textwrap
 #    HAS_RICH = True
 if True:
     def rprint(value='', *args, **kwargs):
-        if value not in [None, False, True] and
+        if value not in [None, False, True] and \
             isinstance(value, (dict, list, set, tuple)):
             value = json.dumps(value, indent=4)
         return LOG.debug(value, *args, **kwargs)

src/exclude_badExits/trustor_poc.py

@@ -9,7 +9,7 @@ import re
 import sys
 import ipaddress
 import warnings
-
+import logging
 
 import urllib3.util
 from urllib3.util import parse_url as urlparse
@@ -24,7 +24,6 @@ except:
     ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
 
 global LOG
-import logging
 warnings.filterwarnings('ignore')
 LOG = logging.getLogger()
 
@@ -36,7 +35,7 @@ logging.getLogger("urllib3").setLevel(logging.INFO)
 # https://github.com/erans/torcontactinfoparser
 # sys.path.append('/home/....')
 try:
-    from torcontactinfo import TorContactInfoParser
+    from exclude_badExits.torcontactinfo import TorContactInfoParser
 except:
     TorContactInfoParser = None
 
@@ -266,7 +265,10 @@ def oDownloadUrlRequests(uri, sCAfile: str, timeout: int = 30, host: str = '127.
     return oReqResp
 
 # There's no point in using asyncio because of duplicate urls in the tasks
-async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout;int = 30, host:str = '127.0.0.1', port:int = 9050, content_type:str = 'text/plain'):
+async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout:int = 30,
+                            host:str = '127.0.0.1', port:int = 9050,
+                            content_type:str = 'text/plain'):
+
     import httpcore
     import asyncio
     import httpx