Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

try: 

    import http.client as httplib 

except ImportError: 

    import httplib 

import zlib 

import io 

from socket import timeout as SocketTimeout 

 

from ._collections import HTTPHeaderDict 

from .exceptions import ( 

    ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked 

) 

from .packages.six import string_types as basestring, binary_type, PY3 

from .connection import HTTPException, BaseSSLError 

from .util.response import is_fp_closed 

 

 

class DeflateDecoder(object): 

 

    def __init__(self): 

        self._first_try = True 

        self._data = binary_type() 

        self._obj = zlib.decompressobj() 

 

    def __getattr__(self, name): 

        return getattr(self._obj, name) 

 

    def decompress(self, data): 

        if not data: 

            return data 

 

        if not self._first_try: 

            return self._obj.decompress(data) 

 

        self._data += data 

        try: 

            return self._obj.decompress(data) 

        except zlib.error: 

            self._first_try = False 

            self._obj = zlib.decompressobj(-zlib.MAX_WBITS) 

            try: 

                return self.decompress(self._data) 

            finally: 

                self._data = None 

 

 

class GzipDecoder(object): 

 

    def __init__(self): 

        self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) 

 

    def __getattr__(self, name): 

        return getattr(self._obj, name) 

 

    def decompress(self, data): 

        if not data: 

            return data 

        return self._obj.decompress(data) 

 

 

def _get_decoder(mode): 

    if mode == 'gzip': 

        return GzipDecoder() 

 

    return DeflateDecoder() 

 

 

class HTTPResponse(io.IOBase): 

    """ 

    HTTP Response container. 

 

    Backwards-compatible to httplib's HTTPResponse but the response ``body`` is 

    loaded and decoded on-demand when the ``data`` property is accessed.  This 

    class is also compatible with the Python standard library's :mod:`io` 

    module, and can hence be treated as a readable object in the context of that 

    framework. 

 

    Extra parameters for behaviour not present in httplib.HTTPResponse: 

 

    :param preload_content: 

        If True, the response's body will be preloaded during construction. 

 

    :param decode_content: 

        If True, attempts to decode specific content-encoding's based on headers 

        (like 'gzip' and 'deflate') will be skipped and raw data will be used 

        instead. 

 

    :param original_response: 

        When this HTTPResponse wrapper is generated from an httplib.HTTPResponse 

        object, it's convenient to include the original for debug purposes. It's 

        otherwise unused. 

    """ 

 

    CONTENT_DECODERS = ['gzip', 'deflate'] 

    REDIRECT_STATUSES = [301, 302, 303, 307, 308] 

 

    def __init__(self, body='', headers=None, status=0, version=0, reason=None, 

                 strict=0, preload_content=True, decode_content=True, 

                 original_response=None, pool=None, connection=None): 

 

        if isinstance(headers, HTTPHeaderDict): 

            self.headers = headers 

        else: 

            self.headers = HTTPHeaderDict(headers) 

        self.status = status 

        self.version = version 

        self.reason = reason 

        self.strict = strict 

        self.decode_content = decode_content 

 

        self._decoder = None 

        self._body = None 

        self._fp = None 

        self._original_response = original_response 

        self._fp_bytes_read = 0 

 

        if body and isinstance(body, (basestring, binary_type)): 

            self._body = body 

 

        self._pool = pool 

        self._connection = connection 

 

        if hasattr(body, 'read'): 

            self._fp = body 

 

        # Are we using the chunked-style of transfer encoding? 

        self.chunked = False 

        self.chunk_left = None 

        tr_enc = self.headers.get('transfer-encoding', '') 

        if tr_enc.lower() == "chunked": 

            self.chunked = True 

 

        # We certainly don't want to preload content when the response is chunked. 

        if not self.chunked: 

            if preload_content and not self._body: 

                self._body = self.read(decode_content=decode_content) 

 

    def get_redirect_location(self): 

        """ 

        Should we redirect and where to? 

 

        :returns: Truthy redirect location string if we got a redirect status 

            code and valid location. ``None`` if redirect status and no 

            location. ``False`` if not a redirect status code. 

        """ 

        if self.status in self.REDIRECT_STATUSES: 

            return self.headers.get('location') 

 

        return False 

 

    def release_conn(self): 

        if not self._pool or not self._connection: 

            return 

 

        self._pool._put_conn(self._connection) 

        self._connection = None 

 

    @property 

    def data(self): 

        # For backwords-compat with earlier urllib3 0.4 and earlier. 

        if self._body: 

            return self._body 

 

        if self._fp: 

            return self.read(cache_content=True) 

 

    def tell(self): 

        """ 

        Obtain the number of bytes pulled over the wire so far. May differ from 

        the amount of content returned by :meth:``HTTPResponse.read`` if bytes 

        are encoded on the wire (e.g, compressed). 

        """ 

        return self._fp_bytes_read 

 

    def _init_decoder(self): 

        """ 

        Set-up the _decoder attribute if necessar. 

        """ 

        # Note: content-encoding value should be case-insensitive, per RFC 7230 

        # Section 3.2 

        content_encoding = self.headers.get('content-encoding', '').lower() 

        if self._decoder is None: 

            if content_encoding in self.CONTENT_DECODERS: 

                self._decoder = _get_decoder(content_encoding) 

 

    def _decode(self, data, decode_content, flush_decoder): 

        """ 

        Decode the data passed in and potentially flush the decoder. 

        """ 

        try: 

            if decode_content and self._decoder: 

                data = self._decoder.decompress(data) 

        except (IOError, zlib.error) as e: 

            content_encoding = self.headers.get('content-encoding', '').lower() 

            raise DecodeError( 

                "Received response with content-encoding: %s, but " 

                "failed to decode it." % content_encoding, e) 

 

        if flush_decoder and decode_content and self._decoder: 

            buf = self._decoder.decompress(binary_type()) 

            data += buf + self._decoder.flush() 

 

        return data 

 

    def read(self, amt=None, decode_content=None, cache_content=False): 

        """ 

        Similar to :meth:`httplib.HTTPResponse.read`, but with two additional 

        parameters: ``decode_content`` and ``cache_content``. 

 

        :param amt: 

            How much of the content to read. If specified, caching is skipped 

            because it doesn't make sense to cache partial content as the full 

            response. 

 

        :param decode_content: 

            If True, will attempt to decode the body based on the 

            'content-encoding' header. 

 

        :param cache_content: 

            If True, will save the returned data such that the same result is 

            returned despite of the state of the underlying file object. This 

            is useful if you want the ``.data`` property to continue working 

            after having ``.read()`` the file object. (Overridden if ``amt`` is 

            set.) 

        """ 

        self._init_decoder() 

        if decode_content is None: 

            decode_content = self.decode_content 

 

        if self._fp is None: 

            return 

 

        flush_decoder = False 

 

        try: 

            try: 

                if amt is None: 

                    # cStringIO doesn't like amt=None 

                    data = self._fp.read() 

                    flush_decoder = True 

                else: 

                    cache_content = False 

                    data = self._fp.read(amt) 

                    if amt != 0 and not data:  # Platform-specific: Buggy versions of Python. 

                        # Close the connection when no data is returned 

                        # 

                        # This is redundant to what httplib/http.client _should_ 

                        # already do.  However, versions of python released before 

                        # December 15, 2012 (http://bugs.python.org/issue16298) do 

                        # not properly close the connection in all cases. There is 

                        # no harm in redundantly calling close. 

                        self._fp.close() 

                        flush_decoder = True 

 

            except SocketTimeout: 

                # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but 

                # there is yet no clean way to get at it from this context. 

                raise ReadTimeoutError(self._pool, None, 'Read timed out.') 

 

            except BaseSSLError as e: 

                # FIXME: Is there a better way to differentiate between SSLErrors? 

                if 'read operation timed out' not in str(e):  # Defensive: 

                    # This shouldn't happen but just in case we're missing an edge 

                    # case, let's avoid swallowing SSL errors. 

                    raise 

 

                raise ReadTimeoutError(self._pool, None, 'Read timed out.') 

 

            except HTTPException as e: 

                # This includes IncompleteRead. 

                raise ProtocolError('Connection broken: %r' % e, e) 

 

            self._fp_bytes_read += len(data) 

 

            data = self._decode(data, decode_content, flush_decoder) 

 

            if cache_content: 

                self._body = data 

 

            return data 

 

        finally: 

            if self._original_response and self._original_response.isclosed(): 

                self.release_conn() 

 

    def stream(self, amt=2**16, decode_content=None): 

        """ 

        A generator wrapper for the read() method. A call will block until 

        ``amt`` bytes have been read from the connection or until the 

        connection is closed. 

 

        :param amt: 

            How much of the content to read. The generator will return up to 

            much data per iteration, but may return less. This is particularly 

            likely when using compressed data. However, the empty string will 

            never be returned. 

 

        :param decode_content: 

            If True, will attempt to decode the body based on the 

            'content-encoding' header. 

        """ 

        self._init_decoder() 

        if self.chunked: 

            for line in self.read_chunked(amt): 

                yield self._decode(line, decode_content, True) 

        else: 

            while not is_fp_closed(self._fp): 

                data = self.read(amt=amt, decode_content=decode_content) 

 

                if data: 

                    yield data 

 

    @classmethod 

    def from_httplib(ResponseCls, r, **response_kw): 

        """ 

        Given an :class:`httplib.HTTPResponse` instance ``r``, return a 

        corresponding :class:`urllib3.response.HTTPResponse` object. 

 

        Remaining parameters are passed to the HTTPResponse constructor, along 

        with ``original_response=r``. 

        """ 

        headers = r.msg 

        if not isinstance(headers, HTTPHeaderDict): 

            if PY3: # Python 3 

                headers = HTTPHeaderDict(headers.items()) 

            else: # Python 2 

                headers = HTTPHeaderDict.from_httplib(headers) 

 

        # HTTPResponse objects in Python 3 don't have a .strict attribute 

        strict = getattr(r, 'strict', 0) 

        resp = ResponseCls(body=r, 

                           headers=headers, 

                           status=r.status, 

                           version=r.version, 

                           reason=r.reason, 

                           strict=strict, 

                           original_response=r, 

                           **response_kw) 

        return resp 

 

    # Backwards-compatibility methods for httplib.HTTPResponse 

    def getheaders(self): 

        return self.headers 

 

    def getheader(self, name, default=None): 

        return self.headers.get(name, default) 

 

    # Overrides from io.IOBase 

    def close(self): 

        if not self.closed: 

            self._fp.close() 

 

    @property 

    def closed(self): 

        if self._fp is None: 

            return True 

        elif hasattr(self._fp, 'closed'): 

            return self._fp.closed 

        elif hasattr(self._fp, 'isclosed'):  # Python 2 

            return self._fp.isclosed() 

        else: 

            return True 

 

    def fileno(self): 

        if self._fp is None: 

            raise IOError("HTTPResponse has no file to get a fileno from") 

        elif hasattr(self._fp, "fileno"): 

            return self._fp.fileno() 

        else: 

            raise IOError("The file-like object this HTTPResponse is wrapped " 

                          "around has no file descriptor") 

 

    def flush(self): 

        if self._fp is not None and hasattr(self._fp, 'flush'): 

            return self._fp.flush() 

 

    def readable(self): 

        # This method is required for `io` module compatibility. 

        return True 

 

    def readinto(self, b): 

        # This method is required for `io` module compatibility. 

        temp = self.read(len(b)) 

        if len(temp) == 0: 

            return 0 

        else: 

            b[:len(temp)] = temp 

            return len(temp) 

 

    def read_chunked(self, amt=None): 

        # FIXME: Rewrite this method and make it a class with 

        #        a better structured logic. 

        if not self.chunked: 

            raise ResponseNotChunked("Response is not chunked. " 

                "Header 'transfer-encoding: chunked' is missing.") 

        while True: 

            # First, we'll figure out length of a chunk and then 

            # we'll try to read it from socket. 

            if self.chunk_left is None: 

                line = self._fp.fp.readline() 

                line = line.decode() 

                # See RFC 7230: Chunked Transfer Coding. 

                i = line.find(';') 

                if i >= 0: 

                    line = line[:i]  # Strip chunk-extensions. 

                try: 

                    self.chunk_left = int(line, 16) 

                except ValueError: 

                    # Invalid chunked protocol response, abort. 

                    self.close() 

                    raise httplib.IncompleteRead(''.join(line)) 

                if self.chunk_left == 0: 

                    break 

            if amt is None: 

                chunk = self._fp._safe_read(self.chunk_left) 

                yield chunk 

                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk. 

                self.chunk_left = None 

            elif amt < self.chunk_left: 

                value = self._fp._safe_read(amt) 

                self.chunk_left = self.chunk_left - amt 

                yield value 

            elif amt == self.chunk_left: 

                value = self._fp._safe_read(amt) 

                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk. 

                self.chunk_left = None 

                yield value 

            else:  # amt > self.chunk_left 

                yield self._fp._safe_read(self.chunk_left) 

                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk. 

                self.chunk_left = None 

 

        # Chunk content ends with \r\n: discard it. 

        while True: 

            line = self._fp.fp.readline() 

            if not line: 

                # Some sites may not end with '\r\n'. 

                break 

            if line == b'\r\n': 

                break 

 

        # We read everything; close the "file". 

        self.release_conn()