diff --git a/twisted/web/_newclient.py b/twisted/web/_newclient.py index 431e029..a33ce9d 100644 --- a/twisted/web/_newclient.py +++ b/twisted/web/_newclient.py @@ -28,6 +28,8 @@ Various other classes in this module support this usage: __metaclass__ = type +import re + from zope.interface import implements from twisted.python import log @@ -540,6 +542,73 @@ class HTTPClientParser(HTTPParser): del self._responseDeferred +_VALID_METHOD = re.compile( + br"\A[%s]+\Z" % ( + bytes().join( + ( + b"!", b"#", b"$", b"%", b"&", b"'", b"*", + b"+", b"-", b".", b"^", b"_", b"`", b"|", b"~", + b"\x30-\x39", + b"\x41-\x5a", + b"\x61-\x7A", + ), + ), + ), +) + + + +def _ensureValidMethod(method): + """ + An HTTP method is an HTTP token, which consists of any visible + ASCII character that is not a delimiter (i.e. one of + C{"(),/:;<=>?@[\\]{}}.) + + @param method: the method to check + @type method: L{bytes} + + @return: the method if it is valid + @rtype: L{bytes} + + @raise ValueError: if the method is not valid + + @see: U{https://tools.ietf.org/html/rfc7230#section-3.1.1}, + U{https://tools.ietf.org/html/rfc7230#section-3.2.6}, + U{https://tools.ietf.org/html/rfc5234#appendix-B.1} + """ + if _VALID_METHOD.match(method): + return method + raise ValueError("Invalid method {!r}".format(method)) + + + +_VALID_URI = re.compile(br'\A[\x21-\x7e]+\Z') + + + +def _ensureValidURI(uri): + """ + A valid URI cannot contain control characters (i.e., characters + between 0-32, inclusive and 127) or non-ASCII characters (i.e., + characters with values between 128-255, inclusive). + + @param uri: the URI to check + @type uri: L{bytes} + + @return: the URI if it is valid + @rtype: L{bytes} + + @raise ValueError: if the URI is not valid + + @see: U{https://tools.ietf.org/html/rfc3986#section-3.3}, + U{https://tools.ietf.org/html/rfc3986#appendix-A}, + U{https://tools.ietf.org/html/rfc5234#appendix-B.1} + """ + if _VALID_URI.match(uri): + return uri + raise ValueError("Invalid URI {!r}".format(uri)) + + class Request: """ @@ -566,8 +635,8 @@ class Request: @type persistent: C{bool} """ def __init__(self, method, uri, headers, bodyProducer, persistent=False): - self.method = method - self.uri = uri + self.method = _ensureValidMethod(method) + self.uri = _ensureValidURI(uri) self.headers = headers self.bodyProducer = bodyProducer self.persistent = persistent @@ -583,7 +652,14 @@ class Request: # weren't limited to issueing HTTP/1.1 requests. requestLines = [] requestLines.append( - '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) + b' '.join( + [ + _ensureValidMethod(self.method), + _ensureValidURI(self.uri), + b'HTTP/1.1\r\n', + ] + ), + ) if not self.persistent: requestLines.append('Connection: close\r\n') if TEorCL is not None: diff --git a/twisted/web/client.py b/twisted/web/client.py index 7e9a488..3c57b50 100644 --- a/twisted/web/client.py +++ b/twisted/web/client.py @@ -27,6 +27,8 @@ from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse from twisted.web.http_headers import Headers from twisted.python.compat import set +from twisted.web._newclient import _ensureValidURI, _ensureValidMethod + class PartialDownloadError(error.Error): """ @@ -58,11 +60,13 @@ class HTTPPageGetter(http.HTTPClient): _completelyDone = True - _specialHeaders = set(('host', 'user-agent', 'cookie', 'content-length')) + _specialHeaders = set( + (b'host', b'user-agent', b'cookie', b'content-length'), + ) def connectionMade(self): - method = getattr(self.factory, 'method', 'GET') - self.sendCommand(method, self.factory.path) + method = _ensureValidMethod(getattr(self.factory, 'method', b'GET')) + self.sendCommand(method, _ensureValidURI(self.factory.path)) if self.factory.scheme == 'http' and self.factory.port != 80: host = '%s:%s' % (self.factory.host, self.factory.port) elif self.factory.scheme == 'https' and self.factory.port != 443: @@ -330,7 +334,7 @@ class HTTPClientFactory(protocol.ClientFactory): # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("connection", "close") self.postdata = postdata - self.method = method + self.method = _ensureValidMethod(method) self.setURL(url) @@ -357,6 +361,7 @@ class HTTPClientFactory(protocol.ClientFactory): return "<%s: %s>" % (self.__class__.__name__, self.url) def setURL(self, url): + _ensureValidURI(url.strip()) self.url = url scheme, host, port, path = _parse(url) if scheme and host: @@ -596,7 +601,7 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, @return: The factory created by C{factoryFactory} """ - scheme, host, port, path = _parse(url) + scheme, host, port, path = _parse(_ensureValidURI(url.strip())) factory = factoryFactory(url, *args, **kwargs) if scheme == 'https': from twisted.internet import ssl @@ -1074,6 +1079,8 @@ class _AgentBase(object): Issue a new request, given the endpoint and the path sent as part of the request. """ + method = _ensureValidMethod(method) + # Create minimal headers, if necessary: if headers is None: headers = Headers() @@ -1196,6 +1203,7 @@ class Agent(_AgentBase): given URI is not supported. @rtype: L{Deferred} """ + uri = _ensureValidURI(uri.strip()) parsedURI = _parse(uri) try: endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, @@ -1228,6 +1236,8 @@ class ProxyAgent(_AgentBase): """ Issue a new request via the configured proxy. """ + uri = _ensureValidURI(uri.strip()) + # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: key = ("http-proxy", self._proxyEndpoint) diff --git a/twisted/web/newsfragments/9647.bugfix b/twisted/web/newsfragments/9647.bugfix new file mode 100644 index 0000000..b76916c --- /dev/null +++ b/twisted/web/newsfragments/9647.bugfix @@ -0,0 +1 @@ +All HTTP clients in twisted.web.client now raise a ValueError when called with a method and/or URL that contain invalid characters. This mitigates CVE-2019-12387. Thanks to Alex Brasetvik for reporting this vulnerability. \ No newline at end of file diff --git a/twisted/web/test/injectionhelpers.py b/twisted/web/test/injectionhelpers.py new file mode 100644 index 0000000..ffeb862 --- /dev/null +++ b/twisted/web/test/injectionhelpers.py @@ -0,0 +1,168 @@ +""" +Helpers for URI and method injection tests. + +@see: U{CVE-2019-12387} +""" + +import string + + +UNPRINTABLE_ASCII = ( + frozenset(range(0, 128)) - + frozenset(bytearray(string.printable, 'ascii')) +) + +NONASCII = frozenset(range(128, 256)) + + + +class MethodInjectionTestsMixin(object): + """ + A mixin that runs HTTP method injection tests. Define + L{MethodInjectionTestsMixin.attemptRequestWithMaliciousMethod} in + a L{twisted.trial.unittest.SynchronousTestCase} subclass to test + how HTTP client code behaves when presented with malicious HTTP + methods. + + @see: U{CVE-2019-12387} + """ + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt to send a request with the given method. This should + synchronously raise a L{ValueError} if either is invalid. + + @param method: the method (e.g. C{GET\x00}) + + @param uri: the URI + + @type method: + """ + raise NotImplementedError() + + + def test_methodWithCLRFRejected(self): + """ + Issuing a request with a method that contains a carriage + return and line feed fails with a L{ValueError}. + """ + with self.assertRaises(ValueError) as cm: + method = b"GET\r\nX-Injected-Header: value" + self.attemptRequestWithMaliciousMethod(method) + self.assertRegex(str(cm.exception), "^Invalid method") + + + def test_methodWithUnprintableASCIIRejected(self): + """ + Issuing a request with a method that contains unprintable + ASCII characters fails with a L{ValueError}. + """ + for c in UNPRINTABLE_ASCII: + method = b"GET%s" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousMethod(method) + self.assertRegex(str(cm.exception), "^Invalid method") + + + def test_methodWithNonASCIIRejected(self): + """ + Issuing a request with a method that contains non-ASCII + characters fails with a L{ValueError}. + """ + for c in NONASCII: + method = b"GET%s" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousMethod(method) + self.assertRegex(str(cm.exception), "^Invalid method") + + + +class URIInjectionTestsMixin(object): + """ + A mixin that runs HTTP URI injection tests. Define + L{MethodInjectionTestsMixin.attemptRequestWithMaliciousURI} in a + L{twisted.trial.unittest.SynchronousTestCase} subclass to test how + HTTP client code behaves when presented with malicious HTTP + URIs. + """ + + def attemptRequestWithMaliciousURI(self, method): + """ + Attempt to send a request with the given URI. This should + synchronously raise a L{ValueError} if either is invalid. + + @param uri: the URI. + + @type method: + """ + raise NotImplementedError() + + + def test_hostWithCRLFRejected(self): + """ + Issuing a request with a URI whose host contains a carriage + return and line feed fails with a L{ValueError}. + """ + with self.assertRaises(ValueError) as cm: + uri = b"http://twisted\r\n.invalid/path" + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") + + + def test_hostWithWithUnprintableASCIIRejected(self): + """ + Issuing a request with a URI whose host contains unprintable + ASCII characters fails with a L{ValueError}. + """ + for c in UNPRINTABLE_ASCII: + uri = b"http://twisted%s.invalid/OK" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") + + + def test_hostWithNonASCIIRejected(self): + """ + Issuing a request with a URI whose host contains non-ASCII + characters fails with a L{ValueError}. + """ + for c in NONASCII: + uri = b"http://twisted%s.invalid/OK" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") + + + def test_pathWithCRLFRejected(self): + """ + Issuing a request with a URI whose path contains a carriage + return and line feed fails with a L{ValueError}. + """ + with self.assertRaises(ValueError) as cm: + uri = b"http://twisted.invalid/\r\npath" + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") + + + def test_pathWithWithUnprintableASCIIRejected(self): + """ + Issuing a request with a URI whose path contains unprintable + ASCII characters fails with a L{ValueError}. + """ + for c in UNPRINTABLE_ASCII: + uri = b"http://twisted.invalid/OK%s" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") + + + def test_pathWithNonASCIIRejected(self): + """ + Issuing a request with a URI whose path contains non-ASCII + characters fails with a L{ValueError}. + """ + for c in NONASCII: + uri = b"http://twisted.invalid/OK%s" % (bytearray([c]),) + with self.assertRaises(ValueError) as cm: + self.attemptRequestWithMaliciousURI(uri) + self.assertRegex(str(cm.exception), "^Invalid URI") diff --git a/twisted/web/test/test_webclient.py b/twisted/web/test/test_webclient.py index 1841402..8765c82 100644 --- a/twisted/web/test/test_webclient.py +++ b/twisted/web/test/test_webclient.py @@ -6,6 +6,7 @@ Tests for L{twisted.web.client}. """ import cookielib +import io import os from errno import ENOSPC import zlib @@ -19,7 +20,8 @@ from twisted.trial import unittest from twisted.web import server, static, client, error, util, resource, http_headers from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed from twisted.web._newclient import ResponseNeverReceived, ResponseFailed -from twisted.internet import reactor, defer, interfaces, task +from twisted.internet import address, reactor, defer, interfaces, task +from twisted.internet.protocol import ClientFactory from twisted.python.failure import Failure from twisted.python.filepath import FilePath from twisted.python.log import msg @@ -39,6 +41,11 @@ from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse from twisted.web._newclient import HTTP11ClientProtocol, Response from twisted.web.error import SchemeNotSupported +from twisted.web.test.injectionhelpers import ( + MethodInjectionTestsMixin, + URIInjectionTestsMixin, +) + try: from twisted.internet import ssl except: @@ -3142,3 +3149,306 @@ if ssl is None or not hasattr(ssl, 'DefaultOpenSSLContextFactory'): if not interfaces.IReactorSSL(reactor, None): for case in [WebClientSSLTestCase, WebClientRedirectBetweenSSLandPlainText]: case.skip = "Reactor doesn't support SSL" + + + +class GetPageMethodInjectionTests( + MethodInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.getPage} against HTTP method injections. + """ + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt a request with the provided method. + + @param method: see L{MethodInjectionTestsMixin} + """ + uri = b'http://twisted.invalid' + client.getPage(uri, method=method) + + + +class GetPageURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.getPage} against URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: see L{URIInjectionTestsMixin} + """ + client.getPage(uri) + + + +class DownloadPageMethodInjectionTests( + MethodInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.getPage} against HTTP method injections. + """ + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt a request with the provided method. + + @param method: see L{MethodInjectionTestsMixin} + """ + uri = b'http://twisted.invalid' + client.downloadPage(uri, file=io.BytesIO(), method=method) + + + +class DownloadPageURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.downloadPage} against URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: see L{URIInjectionTestsMixin} + """ + client.downloadPage(uri, file=io.BytesIO()) + + + +def makeHTTPPageGetterFactory(protocolClass, method, host, path): + """ + Make a L{ClientFactory} that can be used with + L{client.HTTPPageGetter} and its subclasses. + + @param protocolClass: The protocol class + @type protocolClass: A subclass of L{client.HTTPPageGetter} + + @param method: the HTTP method + + @param host: the host + + @param path: The URI path + + @return: A L{ClientFactory}. + """ + factory = ClientFactory.forProtocol(protocolClass) + + factory.method = method + factory.host = host + factory.path = path + + factory.scheme = b"http" + factory.port = 0 + factory.headers = {} + factory.agent = b"User/Agent" + factory.cookies = {} + + return factory + + + +class HTTPPageGetterMethodInjectionTests( + MethodInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.HTTPPageGetter} against HTTP method injections. + """ + protocolClass = client.HTTPPageGetter + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt a request with the provided method. + + @param method: L{MethodInjectionTestsMixin} + """ + transport = StringTransport() + factory = makeHTTPPageGetterFactory( + self.protocolClass, + method=method, + host=b"twisted.invalid", + path=b"/", + ) + getter = factory.buildProtocol( + address.IPv4Address("TCP", "127.0.0.1", 0), + ) + getter.makeConnection(transport) + + + +class HTTPPageGetterURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Test L{client.HTTPPageGetter} against HTTP URI injections. + """ + protocolClass = client.HTTPPageGetter + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: L{URIInjectionTestsMixin} + """ + transport = StringTransport() + # Setting the host and path to the same value is imprecise but + # doesn't require parsing an invalid URI. + factory = makeHTTPPageGetterFactory( + self.protocolClass, + method=b"GET", + host=uri, + path=uri, + ) + getter = factory.buildProtocol( + address.IPv4Address("TCP", "127.0.0.1", 0), + ) + getter.makeConnection(transport) + + + +class HTTPPageDownloaderMethodInjectionTests( + HTTPPageGetterMethodInjectionTests +): + + """ + Test L{client.HTTPPageDownloader} against HTTP method injections. + """ + protocolClass = client.HTTPPageDownloader + + + +class HTTPPageDownloaderURIInjectionTests( + HTTPPageGetterURIInjectionTests +): + """ + Test L{client.HTTPPageDownloader} against HTTP URI injections. + """ + protocolClass = client.HTTPPageDownloader + + + +class HTTPClientFactoryMethodInjectionTests( + MethodInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPClientFactory} against HTTP method injections. + """ + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt a request with the provided method. + + @param method: L{MethodInjectionTestsMixin} + """ + client.HTTPClientFactory(b"https://twisted.invalid", method) + + + +class HTTPClientFactoryURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPClientFactory} against HTTP URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: L{URIInjectionTestsMixin} + """ + client.HTTPClientFactory(uri) + + + +class HTTPClientFactorySetURLURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPClientFactory.setURL} against HTTP URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: L{URIInjectionTestsMixin} + """ + client.HTTPClientFactory(b"https://twisted.invalid").setURL(uri) + + + +class HTTPDownloaderMethodInjectionTests( + MethodInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPDownloader} against HTTP method injections. + """ + + def attemptRequestWithMaliciousMethod(self, method): + """ + Attempt a request with the provided method. + + @param method: L{MethodInjectionTestsMixin} + """ + client.HTTPDownloader( + b"https://twisted.invalid", + io.BytesIO(), + method=method, + ) + + + +class HTTPDownloaderURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPDownloader} against HTTP URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: L{URIInjectionTestsMixin} + """ + client.HTTPDownloader(uri, io.BytesIO()) + + + +class HTTPDownloaderSetURLURIInjectionTests( + URIInjectionTestsMixin, + unittest.SynchronousTestCase, +): + """ + Tests L{client.HTTPDownloader.setURL} against HTTP URI injections. + """ + + def attemptRequestWithMaliciousURI(self, uri): + """ + Attempt a request with the provided URI. + + @param uri: L{URIInjectionTestsMixin} + """ + downloader = client.HTTPDownloader( + b"https://twisted.invalid", + io.BytesIO(), + ) + downloader.setURL(uri)