add https_adapter.py
parent
0198994486
commit
d11d95aafe
@ -0,0 +1,263 @@
|
|||||||
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
||||||
|
|
||||||
|
from requests import adapters
|
||||||
|
from requests.utils import (
|
||||||
|
DEFAULT_CA_BUNDLE_PATH,
|
||||||
|
get_auth_from_url,
|
||||||
|
get_encoding_from_headers,
|
||||||
|
prepend_scheme_if_needed,
|
||||||
|
select_proxy,
|
||||||
|
urldefragauth,
|
||||||
|
)
|
||||||
|
from urllib3.util import parse_url
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
from urllib3.util import Timeout as TimeoutSauce
|
||||||
|
|
||||||
|
DEFAULT_POOLBLOCK = False
|
||||||
|
DEFAULT_POOLSIZE = 10
|
||||||
|
DEFAULT_RETRIES = 0
|
||||||
|
DEFAULT_POOL_TIMEOUT = None
|
||||||
|
|
||||||
|
class HTTPAdapter(adapters.HTTPAdapter):
|
||||||
|
def __init__(self,
|
||||||
|
pool_connections=DEFAULT_POOLSIZE,
|
||||||
|
pool_maxsize=DEFAULT_POOLSIZE,
|
||||||
|
max_retries=DEFAULT_RETRIES,
|
||||||
|
pool_block=DEFAULT_POOLBLOCK
|
||||||
|
):
|
||||||
|
self.config = {}
|
||||||
|
self.proxy_manager = {}
|
||||||
|
|
||||||
|
if isinstance(max_retries, Retry):
|
||||||
|
self.max_retries = max_retries
|
||||||
|
else:
|
||||||
|
max_retries = Retry.from_int(max_retries)
|
||||||
|
self.max_retries = max_retries
|
||||||
|
|
||||||
|
self._pool_connections = pool_connections
|
||||||
|
self._pool_maxsize = pool_maxsize
|
||||||
|
self._pool_block = pool_block
|
||||||
|
|
||||||
|
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPSAdapter(HTTPAdapter):
|
||||||
|
"""The built-in HTTP Adapter for urllib3.
|
||||||
|
|
||||||
|
Provides a general-case interface for Requests sessions to contact HTTP and
|
||||||
|
HTTPS urls by implementing the Transport Adapter interface. This class will
|
||||||
|
usually be created by the :class:`Session <Session>` class under the
|
||||||
|
covers.
|
||||||
|
|
||||||
|
:param pool_connections: The number of urllib3 connection pools to cache.
|
||||||
|
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||||||
|
:param max_retries: The maximum number of retries each connection
|
||||||
|
should attempt. Note, this applies only to failed DNS lookups, socket
|
||||||
|
connections and connection timeouts, never to requests where data has
|
||||||
|
made it to the server. By default, Requests does not retry failed
|
||||||
|
connections. If you need granular control over the conditions under
|
||||||
|
which we retry a request, import urllib3's ``Retry`` class and pass
|
||||||
|
that instead.
|
||||||
|
:param pool_block: Whether the connection pool should block for connections.
|
||||||
|
|
||||||
|
Usage::
|
||||||
|
|
||||||
|
>>> import requests
|
||||||
|
>>> s = requests.Session()
|
||||||
|
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
|
||||||
|
>>> s.mount('http://', a)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
pool_connections=DEFAULT_POOLSIZE,
|
||||||
|
pool_maxsize=1,
|
||||||
|
max_retries=3,
|
||||||
|
pool_block=DEFAULT_POOLBLOCK,
|
||||||
|
):
|
||||||
|
retries = Retry(connect=max_retries, read=2, redirect=0)
|
||||||
|
adapters.HTTPAdapter.__init__(self,
|
||||||
|
pool_connections=pool_connections,
|
||||||
|
pool_maxsize=pool_maxsize,
|
||||||
|
max_retries=retries,
|
||||||
|
pool_block=pool_block)
|
||||||
|
|
||||||
|
def get_connection(self, url, proxies=None, use_forwarding_for_https=True):
|
||||||
|
"""Returns a urllib3 connection for the given URL. This should not be
|
||||||
|
called from user code, and is only exposed for use when subclassing the
|
||||||
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||||
|
|
||||||
|
:param url: The URL to connect to.
|
||||||
|
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
|
||||||
|
:rtype: urllib3.ConnectionPool
|
||||||
|
"""
|
||||||
|
proxy = select_proxy(url, proxies)
|
||||||
|
|
||||||
|
if proxy:
|
||||||
|
proxy = prepend_scheme_if_needed(proxy, "http")
|
||||||
|
proxy_url = parse_url(proxy)
|
||||||
|
if not proxy_url.host:
|
||||||
|
raise InvalidProxyURL(
|
||||||
|
"Please check proxy URL. It is malformed "
|
||||||
|
"and could be missing the host."
|
||||||
|
)
|
||||||
|
proxy_manager = self.proxy_manager_for(proxy)
|
||||||
|
conn = proxy_manager.connection_from_url(url)
|
||||||
|
else:
|
||||||
|
# Only scheme should be lower case
|
||||||
|
parsed = urlparse(url)
|
||||||
|
url = parsed.geturl()
|
||||||
|
conn = self.poolmanager.connection_from_url(url, use_forwarding_for_https=True)
|
||||||
|
|
||||||
|
return conn
|
||||||
|
|
||||||
|
def send(
|
||||||
|
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||||||
|
):
|
||||||
|
"""Sends PreparedRequest object. Returns Response object.
|
||||||
|
|
||||||
|
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||||
|
:param stream: (optional) Whether to stream the request content.
|
||||||
|
:param timeout: (optional) How long to wait for the server to send
|
||||||
|
data before giving up, as a float, or a :ref:`(connect timeout,
|
||||||
|
read timeout) <timeouts>` tuple.
|
||||||
|
:type timeout: float or tuple or urllib3 Timeout object
|
||||||
|
:param verify: (optional) Either a boolean, in which case it controls whether
|
||||||
|
we verify the server's TLS certificate, or a string, in which case it
|
||||||
|
must be a path to a CA bundle to use
|
||||||
|
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||||||
|
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||||||
|
:rtype: requests.Response
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
#? _socks_options
|
||||||
|
conn = self.get_connection(request.url, proxies, use_forwarding_for_https=True)
|
||||||
|
except LocationValueError as e:
|
||||||
|
raise InvalidURL(e, request=request)
|
||||||
|
|
||||||
|
self.cert_verify(conn, request.url, verify, cert)
|
||||||
|
url = self.request_url(request, proxies)
|
||||||
|
self.add_headers(
|
||||||
|
request,
|
||||||
|
stream=stream,
|
||||||
|
timeout=timeout,
|
||||||
|
verify=verify,
|
||||||
|
cert=cert,
|
||||||
|
proxies=proxies,
|
||||||
|
)
|
||||||
|
|
||||||
|
chunked = not (request.body is None or "Content-Length" in request.headers)
|
||||||
|
|
||||||
|
if isinstance(timeout, tuple):
|
||||||
|
try:
|
||||||
|
connect, read = timeout
|
||||||
|
timeout = TimeoutSauce(connect=connect, read=read)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
|
||||||
|
f"or a single float to set both timeouts to the same value."
|
||||||
|
)
|
||||||
|
elif isinstance(timeout, TimeoutSauce):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not chunked:
|
||||||
|
resp = conn.urlopen(
|
||||||
|
method=request.method,
|
||||||
|
url=url,
|
||||||
|
body=request.body,
|
||||||
|
headers=request.headers,
|
||||||
|
redirect=False,
|
||||||
|
assert_same_host=False,
|
||||||
|
preload_content=False,
|
||||||
|
decode_content=False,
|
||||||
|
retries=self.max_retries,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Send the request.
|
||||||
|
else:
|
||||||
|
if hasattr(conn, "proxy_pool"):
|
||||||
|
conn = conn.proxy_pool
|
||||||
|
|
||||||
|
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
|
||||||
|
|
||||||
|
try:
|
||||||
|
skip_host = "Host" in request.headers
|
||||||
|
low_conn.putrequest(
|
||||||
|
request.method,
|
||||||
|
url,
|
||||||
|
skip_accept_encoding=True,
|
||||||
|
skip_host=skip_host,
|
||||||
|
)
|
||||||
|
|
||||||
|
for header, value in request.headers.items():
|
||||||
|
low_conn.putheader(header, value)
|
||||||
|
|
||||||
|
low_conn.endheaders()
|
||||||
|
|
||||||
|
for i in request.body:
|
||||||
|
low_conn.send(hex(len(i))[2:].encode("utf-8"))
|
||||||
|
low_conn.send(b"\r\n")
|
||||||
|
low_conn.send(i)
|
||||||
|
low_conn.send(b"\r\n")
|
||||||
|
low_conn.send(b"0\r\n\r\n")
|
||||||
|
|
||||||
|
# Receive the response from the server
|
||||||
|
r = low_conn.getresponse()
|
||||||
|
|
||||||
|
resp = HTTPResponse.from_httplib(
|
||||||
|
r,
|
||||||
|
pool=conn,
|
||||||
|
connection=low_conn,
|
||||||
|
preload_content=False,
|
||||||
|
decode_content=False,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# If we hit any problems here, clean up the connection.
|
||||||
|
# Then, raise so that we can handle the actual exception.
|
||||||
|
low_conn.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
except (ProtocolError, OSError) as err:
|
||||||
|
raise ConnectionError(err, request=request)
|
||||||
|
|
||||||
|
except MaxRetryError as e:
|
||||||
|
if isinstance(e.reason, ConnectTimeoutError):
|
||||||
|
# TODO: Remove this in 3.0.0: see #2811
|
||||||
|
if not isinstance(e.reason, NewConnectionError):
|
||||||
|
raise ConnectTimeout(e, request=request)
|
||||||
|
|
||||||
|
if isinstance(e.reason, ResponseError):
|
||||||
|
raise RetryError(e, request=request)
|
||||||
|
|
||||||
|
if isinstance(e.reason, _ProxyError):
|
||||||
|
raise ProxyError(e, request=request)
|
||||||
|
|
||||||
|
if isinstance(e.reason, _SSLError):
|
||||||
|
# This branch is for urllib3 v1.22 and later.
|
||||||
|
raise SSLError(e, request=request)
|
||||||
|
|
||||||
|
raise ConnectionError(e, request=request)
|
||||||
|
|
||||||
|
except ClosedPoolError as e:
|
||||||
|
raise ConnectionError(e, request=request)
|
||||||
|
|
||||||
|
except _ProxyError as e:
|
||||||
|
raise ProxyError(e)
|
||||||
|
|
||||||
|
except (_SSLError, _HTTPError) as e:
|
||||||
|
if isinstance(e, _SSLError):
|
||||||
|
# This branch is for urllib3 versions earlier than v1.22
|
||||||
|
raise SSLError(e, request=request)
|
||||||
|
elif isinstance(e, ReadTimeoutError):
|
||||||
|
raise ReadTimeout(e, request=request)
|
||||||
|
elif isinstance(e, _InvalidHeader):
|
||||||
|
raise InvalidHeader(e, request=request)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
return self.build_response(request, resp)
|
@ -0,0 +1,160 @@
|
|||||||
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
import traceback
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import select
|
||||||
|
from ctypes import *
|
||||||
|
import time, contextlib
|
||||||
|
import unittest
|
||||||
|
from random import Random
|
||||||
|
random = Random()
|
||||||
|
|
||||||
|
bHAVE_TORR = shutil.which('tor-resolve')
|
||||||
|
|
||||||
|
def bAreWeConnected():
|
||||||
|
# FixMe: Linux only
|
||||||
|
sFile = f"/proc/{os.getpid()}/net/route"
|
||||||
|
if not os.path.isfile(sFile): return None
|
||||||
|
i = 0
|
||||||
|
for elt in open(sFile, "r").readlines():
|
||||||
|
if elt.startswith('Iface'): continue
|
||||||
|
if elt.startswith('lo'): continue
|
||||||
|
i += 1
|
||||||
|
return i > 0
|
||||||
|
|
||||||
|
def sMapaddressResolv(target, iPort=9051):
|
||||||
|
if not stem:
|
||||||
|
LOG.warn('please install the stem Python package')
|
||||||
|
return ''
|
||||||
|
|
||||||
|
try:
|
||||||
|
controller = oGetStemController(log_level=10)
|
||||||
|
|
||||||
|
map_dict = {"0.0.0.0": target}
|
||||||
|
map_ret = controller.map_address(map_dict)
|
||||||
|
|
||||||
|
return map_ret
|
||||||
|
except Exception as e:
|
||||||
|
LOG.exception(e)
|
||||||
|
return ''
|
||||||
|
|
||||||
|
def lIntroductionPoints(target, iPort=9051):
|
||||||
|
if stem == False: return ''
|
||||||
|
from stem import StreamStatus
|
||||||
|
from stem.control import EventType, Controller
|
||||||
|
import getpass
|
||||||
|
l = []
|
||||||
|
try:
|
||||||
|
controller = oGetStemController(log_level=10)
|
||||||
|
desc = controller.get_hidden_service_descriptor(target)
|
||||||
|
l = desc.introduction_points()
|
||||||
|
if l:
|
||||||
|
LOG.warn(f"{elt} NO introduction points for {target}\n")
|
||||||
|
return l
|
||||||
|
LOG.debug(f"{elt} len(l) introduction points for {target}")
|
||||||
|
|
||||||
|
for introduction_point in l:
|
||||||
|
l.append('%s:%s => %s' % (introduction_point.address,
|
||||||
|
introduction_point.port,
|
||||||
|
introduction_point.identifier))
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
LOG.exception(e)
|
||||||
|
return l
|
||||||
|
|
||||||
|
def sTorResolve(target,
|
||||||
|
verbose=False,
|
||||||
|
sHost='127.0.0.1',
|
||||||
|
iPort=9050,
|
||||||
|
SOCK_TIMEOUT_SECONDS=10.0,
|
||||||
|
SOCK_TIMEOUT_TRIES=3,
|
||||||
|
):
|
||||||
|
MAX_INFO_RESPONSE_PACKET_LENGTH = 8
|
||||||
|
|
||||||
|
seb = b"\o004\o360\o000\o000\o000\o000\o000\o001\o000"
|
||||||
|
seb = b"\x04\xf0\x00\x00\x00\x00\x00\x01\x00"
|
||||||
|
seb += bytes(target, 'US-ASCII') + b"\x00"
|
||||||
|
assert len(seb) == 10+len(target), str(len(seb))+repr(seb)
|
||||||
|
|
||||||
|
# LOG.debug(f"0 Sending {len(seb)} to The TOR proxy {seb}")
|
||||||
|
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.connect((sHost, iPort))
|
||||||
|
|
||||||
|
sock.settimeout(SOCK_TIMEOUT_SECONDS)
|
||||||
|
oRet = sock.sendall(seb)
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
data = ''
|
||||||
|
while i < SOCK_TIMEOUT_TRIES:
|
||||||
|
i += 1
|
||||||
|
time.sleep(3)
|
||||||
|
lReady = select.select([sock.fileno()], [], [],
|
||||||
|
SOCK_TIMEOUT_SECONDS)
|
||||||
|
if not lReady[0]: continue
|
||||||
|
try:
|
||||||
|
flags=socket.MSG_WAITALL
|
||||||
|
data = sock.recv(MAX_INFO_RESPONSE_PACKET_LENGTH, flags)
|
||||||
|
except socket.timeout:
|
||||||
|
LOG.warn("4 The TOR proxy " \
|
||||||
|
+repr((sHost, iPort)) \
|
||||||
|
+" didnt reply in " + str(SOCK_TIMEOUT_SECONDS) + " sec."
|
||||||
|
+" #" +str(i))
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error("4 The TOR proxy " \
|
||||||
|
+repr((sHost, iPort)) \
|
||||||
|
+" errored with " + str(e)
|
||||||
|
+" #" +str(i))
|
||||||
|
sock.close()
|
||||||
|
raise SystemExit(4)
|
||||||
|
else:
|
||||||
|
if len(data) > 0: break
|
||||||
|
|
||||||
|
if len(data) == 0:
|
||||||
|
if i > SOCK_TIMEOUT_TRIES:
|
||||||
|
sLabel = "5 No reply #"
|
||||||
|
else:
|
||||||
|
sLabel = "5 No data #"
|
||||||
|
LOG.info(sLabel +f"{i} from {sHost} {iPort}" )
|
||||||
|
sock.close()
|
||||||
|
raise SystemExit(5)
|
||||||
|
|
||||||
|
assert len(data) >= 8
|
||||||
|
packet_sf = data[1]
|
||||||
|
if packet_sf == 90:
|
||||||
|
# , "%d" % packet_sf
|
||||||
|
assert f"{packet_sf}" == "90", f"packet_sf = {packet_sf}"
|
||||||
|
return f"{data[4]}.{data[5]}.{data[6]}.{data[7]}"
|
||||||
|
else:
|
||||||
|
# 91
|
||||||
|
LOG.warn(f"tor-resolve failed for {target} from {sHost} {iPort}" )
|
||||||
|
|
||||||
|
os.system(f"tor-resolve -4 {target} > /tmp/e 2>/dev/null")
|
||||||
|
# os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'")
|
||||||
|
|
||||||
|
return ''
|
||||||
|
|
||||||
|
def getaddrinfo(sHost, sPort):
|
||||||
|
# do this the explicit way = Ive seen the compact connect fail
|
||||||
|
# >>> sHost, sPort = 'l27.0.0.1', 33446
|
||||||
|
# >>> sock.connect((sHost, sPort))
|
||||||
|
# socket.gaierror: [Errno -2] Name or service not known
|
||||||
|
try:
|
||||||
|
lElts = socket.getaddrinfo(sHost, int(sPort), socket.AF_INET)
|
||||||
|
lElts = list(filter(lambda elt: elt[1] == socket.SOCK_DGRAM, lElts))
|
||||||
|
assert len(lElts) == 1, repr(lElts)
|
||||||
|
lPair = lElts[0][-1]
|
||||||
|
assert len(lPair) == 2, repr(lPair)
|
||||||
|
assert type(lPair[1]) == int, repr(lPair)
|
||||||
|
except Exception as e:
|
||||||
|
LOG.exception(e)
|
||||||
|
return None
|
||||||
|
return lPair
|
||||||
|
|
Loading…
Reference in New Issue