googleapiclient.http

1 # Copyright 2014 Google Inc. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 """Classes to encapsulate a single HTTP request. 16 17 The classes implement a command pattern, with every 18 object supporting an execute() method that does the 19 actual HTTP request. 20 """ 21 from __future__ import absolute_import 22 import six 23 from six.moves import http_client 24 from six.moves import range 25 26 __author__ = "jcgregorio@google.com (Joe Gregorio)" 27 28 from six import BytesIO, StringIO 29 from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 30 31 import base64 32 import copy 33 import gzip 34 import httplib2 35 import json 36 import logging 37 import mimetypes 38 import os 39 import random 40 import socket 41 import sys 42 import time 43 import uuid 44 45 # TODO(issue 221): Remove this conditional import jibbajabba. 46 try: 47 import ssl 48 except ImportError: 49 _ssl_SSLError = object() 50 else: 51 _ssl_SSLError = ssl.SSLError 52 53 from email.generator import Generator 54 from email.mime.multipart import MIMEMultipart 55 from email.mime.nonmultipart import MIMENonMultipart 56 from email.parser import FeedParser 57 58 from googleapiclient import _helpers as util 59 60 from googleapiclient import _auth 61 from googleapiclient.errors import BatchError 62 from googleapiclient.errors import HttpError 63 from googleapiclient.errors import InvalidChunkSizeError 64 from googleapiclient.errors import ResumableUploadError 65 from googleapiclient.errors import UnexpectedBodyError 66 from googleapiclient.errors import UnexpectedMethodError 67 from googleapiclient.model import JsonModel 68 69 70 LOGGER = logging.getLogger(__name__) 71 72 DEFAULT_CHUNK_SIZE = 100 * 1024 * 1024 73 74 MAX_URI_LENGTH = 2048 75 76 MAX_BATCH_LIMIT = 1000 77 78 _TOO_MANY_REQUESTS = 429 79 80 DEFAULT_HTTP_TIMEOUT_SEC = 60 81 82 _LEGACY_BATCH_URI = "https://www.googleapis.com/batch" 83 84 if six.PY2: 85 # That's a builtin python3 exception, nonexistent in python2. 86 # Defined to None to avoid NameError while trying to catch it 87 ConnectionError = None

88 89 90 -def _should_retry_response(resp_status, content):

91 """Determines whether a response should be retried. 92 93 Args: 94 resp_status: The response status received. 95 content: The response content body. 96 97 Returns: 98 True if the response should be retried, otherwise False. 99 """ 100 # Retry on 5xx errors. 101 if resp_status >= 500: 102 return True 103 104 # Retry on 429 errors. 105 if resp_status == _TOO_MANY_REQUESTS: 106 return True 107 108 # For 403 errors, we have to check for the `reason` in the response to 109 # determine if we should retry. 110 if resp_status == six.moves.http_client.FORBIDDEN: 111 # If there's no details about the 403 type, don't retry. 112 if not content: 113 return False 114 115 # Content is in JSON format. 116 try: 117 data = json.loads(content.decode("utf-8")) 118 if isinstance(data, dict): 119 reason = data["error"]["errors"][0]["reason"] 120 else: 121 reason = data[0]["error"]["errors"]["reason"] 122 except (UnicodeDecodeError, ValueError, KeyError): 123 LOGGER.warning("Invalid JSON content from response: %s", content) 124 return False 125 126 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 127 128 # Only retry on rate limit related failures. 129 if reason in ("userRateLimitExceeded", "rateLimitExceeded"): 130 return True 131 132 # Everything else is a success or non-retriable so break. 133 return False

134

135 136 -def _retry_request( 137 http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs 138 ):

139 """Retries an HTTP request multiple times while handling errors. 140 141 If after all retries the request still fails, last error is either returned as 142 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 143 144 Args: 145 http: Http object to be used to execute request. 146 num_retries: Maximum number of retries. 147 req_type: Type of the request (used for logging retries). 148 sleep, rand: Functions to sleep for random time between retries. 149 uri: URI to be requested. 150 method: HTTP method to be used. 151 args, kwargs: Additional arguments passed to http.request. 152 153 Returns: 154 resp, content - Response from the http request (may be HTTP 5xx). 155 """ 156 resp = None 157 content = None 158 exception = None 159 for retry_num in range(num_retries + 1): 160 if retry_num > 0: 161 # Sleep before retrying. 162 sleep_time = rand() * 2 ** retry_num 163 LOGGER.warning( 164 "Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s", 165 sleep_time, 166 retry_num, 167 num_retries, 168 req_type, 169 method, 170 uri, 171 resp.status if resp else exception, 172 ) 173 sleep(sleep_time) 174 175 try: 176 exception = None 177 resp, content = http.request(uri, method, *args, **kwargs) 178 # Retry on SSL errors and socket timeout errors. 179 except _ssl_SSLError as ssl_error: 180 exception = ssl_error 181 except socket.timeout as socket_timeout: 182 # It's important that this be before socket.error as it's a subclass 183 # socket.timeout has no errorcode 184 exception = socket_timeout 185 except ConnectionError as connection_error: 186 # Needs to be before socket.error as it's a subclass of 187 # OSError (socket.error) 188 exception = connection_error 189 except socket.error as socket_error: 190 # errno's contents differ by platform, so we have to match by name. 191 if socket.errno.errorcode.get(socket_error.errno) not in { 192 "WSAETIMEDOUT", 193 "ETIMEDOUT", 194 "EPIPE", 195 "ECONNABORTED", 196 }: 197 raise 198 exception = socket_error 199 except httplib2.ServerNotFoundError as server_not_found_error: 200 exception = server_not_found_error 201 202 if exception: 203 if retry_num == num_retries: 204 raise exception 205 else: 206 continue 207 208 if not _should_retry_response(resp.status, content): 209 break 210 211 return resp, content

212

213 214 -class MediaUploadProgress(object):

215 """Status of a resumable upload.""" 216

217 - def __init__(self, resumable_progress, total_size):

218 """Constructor. 219 220 Args: 221 resumable_progress: int, bytes sent so far. 222 total_size: int, total bytes in complete upload, or None if the total 223 upload size isn't known ahead of time. 224 """ 225 self.resumable_progress = resumable_progress 226 self.total_size = total_size

227

228 - def progress(self):

229 """Percent of upload completed, as a float. 230 231 Returns: 232 the percentage complete as a float, returning 0.0 if the total size of 233 the upload is unknown. 234 """ 235 if self.total_size is not None and self.total_size != 0: 236 return float(self.resumable_progress) / float(self.total_size) 237 else: 238 return 0.0

239

240 241 -class MediaDownloadProgress(object):

242 """Status of a resumable download.""" 243

244 - def __init__(self, resumable_progress, total_size):

245 """Constructor. 246 247 Args: 248 resumable_progress: int, bytes received so far. 249 total_size: int, total bytes in complete download. 250 """ 251 self.resumable_progress = resumable_progress 252 self.total_size = total_size

253

254 - def progress(self):

255 """Percent of download completed, as a float. 256 257 Returns: 258 the percentage complete as a float, returning 0.0 if the total size of 259 the download is unknown. 260 """ 261 if self.total_size is not None and self.total_size != 0: 262 return float(self.resumable_progress) / float(self.total_size) 263 else: 264 return 0.0

265

266 267 -class MediaUpload(object):

268 """Describes a media object to upload. 269 270 Base class that defines the interface of MediaUpload subclasses. 271 272 Note that subclasses of MediaUpload may allow you to control the chunksize 273 when uploading a media object. It is important to keep the size of the chunk 274 as large as possible to keep the upload efficient. Other factors may influence 275 the size of the chunk you use, particularly if you are working in an 276 environment where individual HTTP requests may have a hardcoded time limit, 277 such as under certain classes of requests under Google App Engine. 278 279 Streams are io.Base compatible objects that support seek(). Some MediaUpload 280 subclasses support using streams directly to upload data. Support for 281 streaming may be indicated by a MediaUpload sub-class and if appropriate for a 282 platform that stream will be used for uploading the media object. The support 283 for streaming is indicated by has_stream() returning True. The stream() method 284 should return an io.Base object that supports seek(). On platforms where the 285 underlying httplib module supports streaming, for example Python 2.6 and 286 later, the stream will be passed into the http library which will result in 287 less memory being used and possibly faster uploads. 288 289 If you need to upload media that can't be uploaded using any of the existing 290 MediaUpload sub-class then you can sub-class MediaUpload for your particular 291 needs. 292 """ 293

294 - def chunksize(self):

295 """Chunk size for resumable uploads. 296 297 Returns: 298 Chunk size in bytes. 299 """ 300 raise NotImplementedError()

301

302 - def mimetype(self):

303 """Mime type of the body. 304 305 Returns: 306 Mime type. 307 """ 308 return "application/octet-stream"

309

310 - def size(self):

311 """Size of upload. 312 313 Returns: 314 Size of the body, or None of the size is unknown. 315 """ 316 return None

317

318 - def resumable(self):

319 """Whether this upload is resumable. 320 321 Returns: 322 True if resumable upload or False. 323 """ 324 return False

325

326 - def getbytes(self, begin, end):

327 """Get bytes from the media. 328 329 Args: 330 begin: int, offset from beginning of file. 331 length: int, number of bytes to read, starting at begin. 332 333 Returns: 334 A string of bytes read. May be shorter than length if EOF was reached 335 first. 336 """ 337 raise NotImplementedError()

338

339 - def has_stream(self):

340 """Does the underlying upload support a streaming interface. 341 342 Streaming means it is an io.IOBase subclass that supports seek, i.e. 343 seekable() returns True. 344 345 Returns: 346 True if the call to stream() will return an instance of a seekable io.Base 347 subclass. 348 """ 349 return False

350

351 - def stream(self):

352 """A stream interface to the data being uploaded. 353 354 Returns: 355 The returned value is an io.IOBase subclass that supports seek, i.e. 356 seekable() returns True. 357 """ 358 raise NotImplementedError()

359 360 @util.positional(1)

361 - def _to_json(self, strip=None):

362 """Utility function for creating a JSON representation of a MediaUpload. 363 364 Args: 365 strip: array, An array of names of members to not include in the JSON. 366 367 Returns: 368 string, a JSON representation of this instance, suitable to pass to 369 from_json(). 370 """ 371 t = type(self) 372 d = copy.copy(self.__dict__) 373 if strip is not None: 374 for member in strip: 375 del d[member] 376 d["_class"] = t.__name__ 377 d["_module"] = t.__module__ 378 return json.dumps(d)

379

380 - def to_json(self):

381 """Create a JSON representation of an instance of MediaUpload. 382 383 Returns: 384 string, a JSON representation of this instance, suitable to pass to 385 from_json(). 386 """ 387 return self._to_json()

388 389 @classmethod

390 - def new_from_json(cls, s):

391 """Utility class method to instantiate a MediaUpload subclass from a JSON 392 representation produced by to_json(). 393 394 Args: 395 s: string, JSON from to_json(). 396 397 Returns: 398 An instance of the subclass of MediaUpload that was serialized with 399 to_json(). 400 """ 401 data = json.loads(s) 402 # Find and call the right classmethod from_json() to restore the object. 403 module = data["_module"] 404 m = __import__(module, fromlist=module.split(".")[:-1]) 405 kls = getattr(m, data["_class"]) 406 from_json = getattr(kls, "from_json") 407 return from_json(s)

408

409 410 -class MediaIoBaseUpload(MediaUpload):

411 """A MediaUpload for a io.Base objects. 412 413 Note that the Python file object is compatible with io.Base and can be used 414 with this class also. 415 416 fh = BytesIO('...Some data to upload...') 417 media = MediaIoBaseUpload(fh, mimetype='image/png', 418 chunksize=1024*1024, resumable=True) 419 farm.animals().insert( 420 id='cow', 421 name='cow.png', 422 media_body=media).execute() 423 424 Depending on the platform you are working on, you may pass -1 as the 425 chunksize, which indicates that the entire file should be uploaded in a single 426 request. If the underlying platform supports streams, such as Python 2.6 or 427 later, then this can be very efficient as it avoids multiple connections, and 428 also avoids loading the entire file into memory before sending it. Note that 429 Google App Engine has a 5MB limit on request size, so you should never set 430 your chunksize larger than 5MB, or to -1. 431 """ 432 433 @util.positional(3)

434 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):

435 """Constructor. 436 437 Args: 438 fd: io.Base or file object, The source of the bytes to upload. MUST be 439 opened in blocking mode, do not use streams opened in non-blocking mode. 440 The given stream must be seekable, that is, it must be able to call 441 seek() on fd. 442 mimetype: string, Mime-type of the file. 443 chunksize: int, File will be uploaded in chunks of this many bytes. Only 444 used if resumable=True. Pass in a value of -1 if the file is to be 445 uploaded as a single chunk. Note that Google App Engine has a 5MB limit 446 on request size, so you should never set your chunksize larger than 5MB, 447 or to -1. 448 resumable: bool, True if this is a resumable upload. False means upload 449 in a single request. 450 """ 451 super(MediaIoBaseUpload, self).__init__() 452 self._fd = fd 453 self._mimetype = mimetype 454 if not (chunksize == -1 or chunksize > 0): 455 raise InvalidChunkSizeError() 456 self._chunksize = chunksize 457 self._resumable = resumable 458 459 self._fd.seek(0, os.SEEK_END) 460 self._size = self._fd.tell()

461

462 - def chunksize(self):

463 """Chunk size for resumable uploads. 464 465 Returns: 466 Chunk size in bytes. 467 """ 468 return self._chunksize

469

470 - def mimetype(self):

471 """Mime type of the body. 472 473 Returns: 474 Mime type. 475 """ 476 return self._mimetype

477

478 - def size(self):

479 """Size of upload. 480 481 Returns: 482 Size of the body, or None of the size is unknown. 483 """ 484 return self._size

485

486 - def resumable(self):

487 """Whether this upload is resumable. 488 489 Returns: 490 True if resumable upload or False. 491 """ 492 return self._resumable

493

494 - def getbytes(self, begin, length):

495 """Get bytes from the media. 496 497 Args: 498 begin: int, offset from beginning of file. 499 length: int, number of bytes to read, starting at begin. 500 501 Returns: 502 A string of bytes read. May be shorted than length if EOF was reached 503 first. 504 """ 505 self._fd.seek(begin) 506 return self._fd.read(length)

507

508 - def has_stream(self):

509 """Does the underlying upload support a streaming interface. 510 511 Streaming means it is an io.IOBase subclass that supports seek, i.e. 512 seekable() returns True. 513 514 Returns: 515 True if the call to stream() will return an instance of a seekable io.Base 516 subclass. 517 """ 518 return True

519

520 - def stream(self):

521 """A stream interface to the data being uploaded. 522 523 Returns: 524 The returned value is an io.IOBase subclass that supports seek, i.e. 525 seekable() returns True. 526 """ 527 return self._fd

528

529 - def to_json(self):

530 """This upload type is not serializable.""" 531 raise NotImplementedError("MediaIoBaseUpload is not serializable.")

532

533 534 -class MediaFileUpload(MediaIoBaseUpload):

535 """A MediaUpload for a file. 536 537 Construct a MediaFileUpload and pass as the media_body parameter of the 538 method. For example, if we had a service that allowed uploading images: 539 540 media = MediaFileUpload('cow.png', mimetype='image/png', 541 chunksize=1024*1024, resumable=True) 542 farm.animals().insert( 543 id='cow', 544 name='cow.png', 545 media_body=media).execute() 546 547 Depending on the platform you are working on, you may pass -1 as the 548 chunksize, which indicates that the entire file should be uploaded in a single 549 request. If the underlying platform supports streams, such as Python 2.6 or 550 later, then this can be very efficient as it avoids multiple connections, and 551 also avoids loading the entire file into memory before sending it. Note that 552 Google App Engine has a 5MB limit on request size, so you should never set 553 your chunksize larger than 5MB, or to -1. 554 """ 555 556 @util.positional(2)

557 - def __init__( 558 self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False 559 ):

560 """Constructor. 561 562 Args: 563 filename: string, Name of the file. 564 mimetype: string, Mime-type of the file. If None then a mime-type will be 565 guessed from the file extension. 566 chunksize: int, File will be uploaded in chunks of this many bytes. Only 567 used if resumable=True. Pass in a value of -1 if the file is to be 568 uploaded in a single chunk. Note that Google App Engine has a 5MB limit 569 on request size, so you should never set your chunksize larger than 5MB, 570 or to -1. 571 resumable: bool, True if this is a resumable upload. False means upload 572 in a single request. 573 """ 574 self._filename = filename 575 fd = open(self._filename, "rb") 576 if mimetype is None: 577 # No mimetype provided, make a guess. 578 mimetype, _ = mimetypes.guess_type(filename) 579 if mimetype is None: 580 # Guess failed, use octet-stream. 581 mimetype = "application/octet-stream" 582 super(MediaFileUpload, self).__init__( 583 fd, mimetype, chunksize=chunksize, resumable=resumable 584 )

585

586 - def __del__(self):

587 self._fd.close()

588

589 - def to_json(self):

590 """Creating a JSON representation of an instance of MediaFileUpload. 591 592 Returns: 593 string, a JSON representation of this instance, suitable to pass to 594 from_json(). 595 """ 596 return self._to_json(strip=["_fd"])

597 598 @staticmethod

599 - def from_json(s):

600 d = json.loads(s) 601 return MediaFileUpload( 602 d["_filename"], 603 mimetype=d["_mimetype"], 604 chunksize=d["_chunksize"], 605 resumable=d["_resumable"], 606 )

607

608 609 -class MediaInMemoryUpload(MediaIoBaseUpload):

610 """MediaUpload for a chunk of bytes. 611 612 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 613 the stream. 614 """ 615 616 @util.positional(2)

617 - def __init__( 618 self, 619 body, 620 mimetype="application/octet-stream", 621 chunksize=DEFAULT_CHUNK_SIZE, 622 resumable=False, 623 ):

624 """Create a new MediaInMemoryUpload. 625 626 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 627 the stream. 628 629 Args: 630 body: string, Bytes of body content. 631 mimetype: string, Mime-type of the file or default of 632 'application/octet-stream'. 633 chunksize: int, File will be uploaded in chunks of this many bytes. Only 634 used if resumable=True. 635 resumable: bool, True if this is a resumable upload. False means upload 636 in a single request. 637 """ 638 fd = BytesIO(body) 639 super(MediaInMemoryUpload, self).__init__( 640 fd, mimetype, chunksize=chunksize, resumable=resumable 641 )

642

643 644 -class MediaIoBaseDownload(object):

645 """"Download media resources. 646 647 Note that the Python file object is compatible with io.Base and can be used 648 with this class also. 649 650 651 Example: 652 request = farms.animals().get_media(id='cow') 653 fh = io.FileIO('cow.png', mode='wb') 654 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 655 656 done = False 657 while done is False: 658 status, done = downloader.next_chunk() 659 if status: 660 print "Download %d%%." % int(status.progress() * 100) 661 print "Download Complete!" 662 """ 663 664 @util.positional(3)

665 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):

666 """Constructor. 667 668 Args: 669 fd: io.Base or file object, The stream in which to write the downloaded 670 bytes. 671 request: googleapiclient.http.HttpRequest, the media request to perform in 672 chunks. 673 chunksize: int, File will be downloaded in chunks of this many bytes. 674 """ 675 self._fd = fd 676 self._request = request 677 self._uri = request.uri 678 self._chunksize = chunksize 679 self._progress = 0 680 self._total_size = None 681 self._done = False 682 683 # Stubs for testing. 684 self._sleep = time.sleep 685 self._rand = random.random 686 687 self._headers = {} 688 for k, v in six.iteritems(request.headers): 689 # allow users to supply custom headers by setting them on the request 690 # but strip out the ones that are set by default on requests generated by 691 # API methods like Drive's files().get(fileId=...) 692 if not k.lower() in ("accept", "accept-encoding", "user-agent"): 693 self._headers[k] = v

694 695 @util.positional(1)

696 - def next_chunk(self, num_retries=0):

697 """Get the next chunk of the download. 698 699 Args: 700 num_retries: Integer, number of times to retry with randomized 701 exponential backoff. If all retries fail, the raised HttpError 702 represents the last request. If zero (default), we attempt the 703 request only once. 704 705 Returns: 706 (status, done): (MediaDownloadProgress, boolean) 707 The value of 'done' will be True when the media has been fully 708 downloaded or the total size of the media is unknown. 709 710 Raises: 711 googleapiclient.errors.HttpError if the response was not a 2xx. 712 httplib2.HttpLib2Error if a transport error has occurred. 713 """ 714 headers = self._headers.copy() 715 headers["range"] = "bytes=%d-%d" % ( 716 self._progress, 717 self._progress + self._chunksize, 718 ) 719 http = self._request.http 720 721 resp, content = _retry_request( 722 http, 723 num_retries, 724 "media download", 725 self._sleep, 726 self._rand, 727 self._uri, 728 "GET", 729 headers=headers, 730 ) 731 732 if resp.status in [200, 206]: 733 if "content-location" in resp and resp["content-location"] != self._uri: 734 self._uri = resp["content-location"] 735 self._progress += len(content) 736 self._fd.write(content) 737 738 if "content-range" in resp: 739 content_range = resp["content-range"] 740 length = content_range.rsplit("/", 1)[1] 741 self._total_size = int(length) 742 elif "content-length" in resp: 743 self._total_size = int(resp["content-length"]) 744 745 if self._total_size is None or self._progress == self._total_size: 746 self._done = True 747 return MediaDownloadProgress(self._progress, self._total_size), self._done 748 else: 749 raise HttpError(resp, content, uri=self._uri)

750

751 752 -class _StreamSlice(object):

753 """Truncated stream. 754 755 Takes a stream and presents a stream that is a slice of the original stream. 756 This is used when uploading media in chunks. In later versions of Python a 757 stream can be passed to httplib in place of the string of data to send. The 758 problem is that httplib just blindly reads to the end of the stream. This 759 wrapper presents a virtual stream that only reads to the end of the chunk. 760 """ 761

762 - def __init__(self, stream, begin, chunksize):

763 """Constructor. 764 765 Args: 766 stream: (io.Base, file object), the stream to wrap. 767 begin: int, the seek position the chunk begins at. 768 chunksize: int, the size of the chunk. 769 """ 770 self._stream = stream 771 self._begin = begin 772 self._chunksize = chunksize 773 self._stream.seek(begin)

774

775 - def read(self, n=-1):

776 """Read n bytes. 777 778 Args: 779 n, int, the number of bytes to read. 780 781 Returns: 782 A string of length 'n', or less if EOF is reached. 783 """ 784 # The data left available to read sits in [cur, end) 785 cur = self._stream.tell() 786 end = self._begin + self._chunksize 787 if n == -1 or cur + n > end: 788 n = end - cur 789 return self._stream.read(n)

790

791 792 -class HttpRequest(object):

793 """Encapsulates a single HTTP request.""" 794 795 @util.positional(4)

796 - def __init__( 797 self, 798 http, 799 postproc, 800 uri, 801 method="GET", 802 body=None, 803 headers=None, 804 methodId=None, 805 resumable=None, 806 ):

807 """Constructor for an HttpRequest. 808 809 Args: 810 http: httplib2.Http, the transport object to use to make a request 811 postproc: callable, called on the HTTP response and content to transform 812 it into a data object before returning, or raising an exception 813 on an error. 814 uri: string, the absolute URI to send the request to 815 method: string, the HTTP method to use 816 body: string, the request body of the HTTP request, 817 headers: dict, the HTTP request headers 818 methodId: string, a unique identifier for the API method being called. 819 resumable: MediaUpload, None if this is not a resumbale request. 820 """ 821 self.uri = uri 822 self.method = method 823 self.body = body 824 self.headers = headers or {} 825 self.methodId = methodId 826 self.http = http 827 self.postproc = postproc 828 self.resumable = resumable 829 self.response_callbacks = [] 830 self._in_error_state = False 831 832 # The size of the non-media part of the request. 833 self.body_size = len(self.body or "") 834 835 # The resumable URI to send chunks to. 836 self.resumable_uri = None 837 838 # The bytes that have been uploaded. 839 self.resumable_progress = 0 840 841 # Stubs for testing. 842 self._rand = random.random 843 self._sleep = time.sleep

844 845 @util.positional(1)

846 - def execute(self, http=None, num_retries=0):

847 """Execute the request. 848 849 Args: 850 http: httplib2.Http, an http object to be used in place of the 851 one the HttpRequest request object was constructed with. 852 num_retries: Integer, number of times to retry with randomized 853 exponential backoff. If all retries fail, the raised HttpError 854 represents the last request. If zero (default), we attempt the 855 request only once. 856 857 Returns: 858 A deserialized object model of the response body as determined 859 by the postproc. 860 861 Raises: 862 googleapiclient.errors.HttpError if the response was not a 2xx. 863 httplib2.HttpLib2Error if a transport error has occurred. 864 """ 865 if http is None: 866 http = self.http 867 868 if self.resumable: 869 body = None 870 while body is None: 871 _, body = self.next_chunk(http=http, num_retries=num_retries) 872 return body 873 874 # Non-resumable case. 875 876 if "content-length" not in self.headers: 877 self.headers["content-length"] = str(self.body_size) 878 # If the request URI is too long then turn it into a POST request. 879 # Assume that a GET request never contains a request body. 880 if len(self.uri) > MAX_URI_LENGTH and self.method == "GET": 881 self.method = "POST" 882 self.headers["x-http-method-override"] = "GET" 883 self.headers["content-type"] = "application/x-www-form-urlencoded" 884 parsed = urlparse(self.uri) 885 self.uri = urlunparse( 886 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, None) 887 ) 888 self.body = parsed.query 889 self.headers["content-length"] = str(len(self.body)) 890 891 # Handle retries for server-side errors. 892 resp, content = _retry_request( 893 http, 894 num_retries, 895 "request", 896 self._sleep, 897 self._rand, 898 str(self.uri), 899 method=str(self.method), 900 body=self.body, 901 headers=self.headers, 902 ) 903 904 for callback in self.response_callbacks: 905 callback(resp) 906 if resp.status >= 300: 907 raise HttpError(resp, content, uri=self.uri) 908 return self.postproc(resp, content)

909 910 @util.positional(2)

911 - def add_response_callback(self, cb):

912 """add_response_headers_callback 913 914 Args: 915 cb: Callback to be called on receiving the response headers, of signature: 916 917 def cb(resp): 918 # Where resp is an instance of httplib2.Response 919 """ 920 self.response_callbacks.append(cb)

921 922 @util.positional(1)

923 - def next_chunk(self, http=None, num_retries=0):

924 """Execute the next step of a resumable upload. 925 926 Can only be used if the method being executed supports media uploads and 927 the MediaUpload object passed in was flagged as using resumable upload. 928 929 Example: 930 931 media = MediaFileUpload('cow.png', mimetype='image/png', 932 chunksize=1000, resumable=True) 933 request = farm.animals().insert( 934 id='cow', 935 name='cow.png', 936 media_body=media) 937 938 response = None 939 while response is None: 940 status, response = request.next_chunk() 941 if status: 942 print "Upload %d%% complete." % int(status.progress() * 100) 943 944 945 Args: 946 http: httplib2.Http, an http object to be used in place of the 947 one the HttpRequest request object was constructed with. 948 num_retries: Integer, number of times to retry with randomized 949 exponential backoff. If all retries fail, the raised HttpError 950 represents the last request. If zero (default), we attempt the 951 request only once. 952 953 Returns: 954 (status, body): (ResumableMediaStatus, object) 955 The body will be None until the resumable media is fully uploaded. 956 957 Raises: 958 googleapiclient.errors.HttpError if the response was not a 2xx. 959 httplib2.HttpLib2Error if a transport error has occurred. 960 """ 961 if http is None: 962 http = self.http 963 964 if self.resumable.size() is None: 965 size = "*" 966 else: 967 size = str(self.resumable.size()) 968 969 if self.resumable_uri is None: 970 start_headers = copy.copy(self.headers) 971 start_headers["X-Upload-Content-Type"] = self.resumable.mimetype() 972 if size != "*": 973 start_headers["X-Upload-Content-Length"] = size 974 start_headers["content-length"] = str(self.body_size) 975 976 resp, content = _retry_request( 977 http, 978 num_retries, 979 "resumable URI request", 980 self._sleep, 981 self._rand, 982 self.uri, 983 method=self.method, 984 body=self.body, 985 headers=start_headers, 986 ) 987 988 if resp.status == 200 and "location" in resp: 989 self.resumable_uri = resp["location"] 990 else: 991 raise ResumableUploadError(resp, content) 992 elif self._in_error_state: 993 # If we are in an error state then query the server for current state of 994 # the upload by sending an empty PUT and reading the 'range' header in 995 # the response. 996 headers = {"Content-Range": "bytes */%s" % size, "content-length": "0"} 997 resp, content = http.request(self.resumable_uri, "PUT", headers=headers) 998 status, body = self._process_response(resp, content) 999 if body: 1000 # The upload was complete. 1001 return (status, body) 1002 1003 if self.resumable.has_stream(): 1004 data = self.resumable.stream() 1005 if self.resumable.chunksize() == -1: 1006 data.seek(self.resumable_progress) 1007 chunk_end = self.resumable.size() - self.resumable_progress - 1 1008 else: 1009 # Doing chunking with a stream, so wrap a slice of the stream. 1010 data = _StreamSlice( 1011 data, self.resumable_progress, self.resumable.chunksize() 1012 ) 1013 chunk_end = min( 1014 self.resumable_progress + self.resumable.chunksize() - 1, 1015 self.resumable.size() - 1, 1016 ) 1017 else: 1018 data = self.resumable.getbytes( 1019 self.resumable_progress, self.resumable.chunksize() 1020 ) 1021 1022 # A short read implies that we are at EOF, so finish the upload. 1023 if len(data) < self.resumable.chunksize(): 1024 size = str(self.resumable_progress + len(data)) 1025 1026 chunk_end = self.resumable_progress + len(data) - 1 1027 1028 headers = { 1029 "Content-Range": "bytes %d-%d/%s" 1030 % (self.resumable_progress, chunk_end, size), 1031 # Must set the content-length header here because httplib can't 1032 # calculate the size when working with _StreamSlice. 1033 "Content-Length": str(chunk_end - self.resumable_progress + 1), 1034 } 1035 1036 for retry_num in range(num_retries + 1): 1037 if retry_num > 0: 1038 self._sleep(self._rand() * 2 ** retry_num) 1039 LOGGER.warning( 1040 "Retry #%d for media upload: %s %s, following status: %d" 1041 % (retry_num, self.method, self.uri, resp.status) 1042 ) 1043 1044 try: 1045 resp, content = http.request( 1046 self.resumable_uri, method="PUT", body=data, headers=headers 1047 ) 1048 except: 1049 self._in_error_state = True 1050 raise 1051 if not _should_retry_response(resp.status, content): 1052 break 1053 1054 return self._process_response(resp, content)

1055

1056 - def _process_response(self, resp, content):

1057 """Process the response from a single chunk upload. 1058 1059 Args: 1060 resp: httplib2.Response, the response object. 1061 content: string, the content of the response. 1062 1063 Returns: 1064 (status, body): (ResumableMediaStatus, object) 1065 The body will be None until the resumable media is fully uploaded. 1066 1067 Raises: 1068 googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 1069 """ 1070 if resp.status in [200, 201]: 1071 self._in_error_state = False 1072 return None, self.postproc(resp, content) 1073 elif resp.status == 308: 1074 self._in_error_state = False 1075 # A "308 Resume Incomplete" indicates we are not done. 1076 try: 1077 self.resumable_progress = int(resp["range"].split("-")[1]) + 1 1078 except KeyError: 1079 # If resp doesn't contain range header, resumable progress is 0 1080 self.resumable_progress = 0 1081 if "location" in resp: 1082 self.resumable_uri = resp["location"] 1083 else: 1084 self._in_error_state = True 1085 raise HttpError(resp, content, uri=self.uri) 1086 1087 return ( 1088 MediaUploadProgress(self.resumable_progress, self.resumable.size()), 1089 None, 1090 )

1091

1092 - def to_json(self):

1093 """Returns a JSON representation of the HttpRequest.""" 1094 d = copy.copy(self.__dict__) 1095 if d["resumable"] is not None: 1096 d["resumable"] = self.resumable.to_json() 1097 del d["http"] 1098 del d["postproc"] 1099 del d["_sleep"] 1100 del d["_rand"] 1101 1102 return json.dumps(d)

1103 1104 @staticmethod

1105 - def from_json(s, http, postproc):

1106 """Returns an HttpRequest populated with info from a JSON object.""" 1107 d = json.loads(s) 1108 if d["resumable"] is not None: 1109 d["resumable"] = MediaUpload.new_from_json(d["resumable"]) 1110 return HttpRequest( 1111 http, 1112 postproc, 1113 uri=d["uri"], 1114 method=d["method"], 1115 body=d["body"], 1116 headers=d["headers"], 1117 methodId=d["methodId"], 1118 resumable=d["resumable"], 1119 )

1120 1121 @staticmethod

1122 - def null_postproc(resp, contents):

1123 return resp, contents

1124

1125 1126 -class BatchHttpRequest(object):

1127 """Batches multiple HttpRequest objects into a single HTTP request. 1128 1129 Example: 1130 from googleapiclient.http import BatchHttpRequest 1131 1132 def list_animals(request_id, response, exception): 1133 \"\"\"Do something with the animals list response.\"\"\" 1134 if exception is not None: 1135 # Do something with the exception. 1136 pass 1137 else: 1138 # Do something with the response. 1139 pass 1140 1141 def list_farmers(request_id, response, exception): 1142 \"\"\"Do something with the farmers list response.\"\"\" 1143 if exception is not None: 1144 # Do something with the exception. 1145 pass 1146 else: 1147 # Do something with the response. 1148 pass 1149 1150 service = build('farm', 'v2') 1151 1152 batch = BatchHttpRequest() 1153 1154 batch.add(service.animals().list(), list_animals) 1155 batch.add(service.farmers().list(), list_farmers) 1156 batch.execute(http=http) 1157 """ 1158 1159 @util.positional(1)

1160 - def __init__(self, callback=None, batch_uri=None):

1161 """Constructor for a BatchHttpRequest. 1162 1163 Args: 1164 callback: callable, A callback to be called for each response, of the 1165 form callback(id, response, exception). The first parameter is the 1166 request id, and the second is the deserialized response object. The 1167 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1168 occurred while processing the request, or None if no error occurred. 1169 batch_uri: string, URI to send batch requests to. 1170 """ 1171 if batch_uri is None: 1172 batch_uri = _LEGACY_BATCH_URI 1173 1174 if batch_uri == _LEGACY_BATCH_URI: 1175 LOGGER.warning( 1176 "You have constructed a BatchHttpRequest using the legacy batch " 1177 "endpoint %s. This endpoint will be turned down on August 12, 2020. " 1178 "Please provide the API-specific endpoint or use " 1179 "service.new_batch_http_request(). For more details see " 1180 "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" 1181 "and https://developers.google.com/api-client-library/python/guide/batch.", 1182 _LEGACY_BATCH_URI, 1183 ) 1184 self._batch_uri = batch_uri 1185 1186 # Global callback to be called for each individual response in the batch. 1187 self._callback = callback 1188 1189 # A map from id to request. 1190 self._requests = {} 1191 1192 # A map from id to callback. 1193 self._callbacks = {} 1194 1195 # List of request ids, in the order in which they were added. 1196 self._order = [] 1197 1198 # The last auto generated id. 1199 self._last_auto_id = 0 1200 1201 # Unique ID on which to base the Content-ID headers. 1202 self._base_id = None 1203 1204 # A map from request id to (httplib2.Response, content) response pairs 1205 self._responses = {} 1206 1207 # A map of id(Credentials) that have been refreshed. 1208 self._refreshed_credentials = {}

1209

1210 - def _refresh_and_apply_credentials(self, request, http):

1211 """Refresh the credentials and apply to the request. 1212 1213 Args: 1214 request: HttpRequest, the request. 1215 http: httplib2.Http, the global http object for the batch. 1216 """ 1217 # For the credentials to refresh, but only once per refresh_token 1218 # If there is no http per the request then refresh the http passed in 1219 # via execute() 1220 creds = None 1221 request_credentials = False 1222 1223 if request.http is not None: 1224 creds = _auth.get_credentials_from_http(request.http) 1225 request_credentials = True 1226 1227 if creds is None and http is not None: 1228 creds = _auth.get_credentials_from_http(http) 1229 1230 if creds is not None: 1231 if id(creds) not in self._refreshed_credentials: 1232 _auth.refresh_credentials(creds) 1233 self._refreshed_credentials[id(creds)] = 1 1234 1235 # Only apply the credentials if we are using the http object passed in, 1236 # otherwise apply() will get called during _serialize_request(). 1237 if request.http is None or not request_credentials: 1238 _auth.apply_credentials(creds, request.headers)

1239

1240 - def _id_to_header(self, id_):

1241 """Convert an id to a Content-ID header value. 1242 1243 Args: 1244 id_: string, identifier of individual request. 1245 1246 Returns: 1247 A Content-ID header with the id_ encoded into it. A UUID is prepended to 1248 the value because Content-ID headers are supposed to be universally 1249 unique. 1250 """ 1251 if self._base_id is None: 1252 self._base_id = uuid.uuid4() 1253 1254 # NB: we intentionally leave whitespace between base/id and '+', so RFC2822 1255 # line folding works properly on Python 3; see 1256 # https://github.com/googleapis/google-api-python-client/issues/164 1257 return "<%s + %s>" % (self._base_id, quote(id_))

1258

1259 - def _header_to_id(self, header):

1260 """Convert a Content-ID header value to an id. 1261 1262 Presumes the Content-ID header conforms to the format that _id_to_header() 1263 returns. 1264 1265 Args: 1266 header: string, Content-ID header value. 1267 1268 Returns: 1269 The extracted id value. 1270 1271 Raises: 1272 BatchError if the header is not in the expected format. 1273 """ 1274 if header[0] != "<" or header[-1] != ">": 1275 raise BatchError("Invalid value for Content-ID: %s" % header) 1276 if "+" not in header: 1277 raise BatchError("Invalid value for Content-ID: %s" % header) 1278 base, id_ = header[1:-1].split(" + ", 1) 1279 1280 return unquote(id_)

1281

1282 - def _serialize_request(self, request):

1283 """Convert an HttpRequest object into a string. 1284 1285 Args: 1286 request: HttpRequest, the request to serialize. 1287 1288 Returns: 1289 The request as a string in application/http format. 1290 """ 1291 # Construct status line 1292 parsed = urlparse(request.uri) 1293 request_line = urlunparse( 1294 ("", "", parsed.path, parsed.params, parsed.query, "") 1295 ) 1296 status_line = request.method + " " + request_line + " HTTP/1.1\n" 1297 major, minor = request.headers.get("content-type", "application/json").split( 1298 "/" 1299 ) 1300 msg = MIMENonMultipart(major, minor) 1301 headers = request.headers.copy() 1302 1303 if request.http is not None: 1304 credentials = _auth.get_credentials_from_http(request.http) 1305 if credentials is not None: 1306 _auth.apply_credentials(credentials, headers) 1307 1308 # MIMENonMultipart adds its own Content-Type header. 1309 if "content-type" in headers: 1310 del headers["content-type"] 1311 1312 for key, value in six.iteritems(headers): 1313 msg[key] = value 1314 msg["Host"] = parsed.netloc 1315 msg.set_unixfrom(None) 1316 1317 if request.body is not None: 1318 msg.set_payload(request.body) 1319 msg["content-length"] = str(len(request.body)) 1320 1321 # Serialize the mime message. 1322 fp = StringIO() 1323 # maxheaderlen=0 means don't line wrap headers. 1324 g = Generator(fp, maxheaderlen=0) 1325 g.flatten(msg, unixfrom=False) 1326 body = fp.getvalue() 1327 1328 return status_line + body

1329

1330 - def _deserialize_response(self, payload):

1331 """Convert string into httplib2 response and content. 1332 1333 Args: 1334 payload: string, headers and body as a string. 1335 1336 Returns: 1337 A pair (resp, content), such as would be returned from httplib2.request. 1338 """ 1339 # Strip off the status line 1340 status_line, payload = payload.split("\n", 1) 1341 protocol, status, reason = status_line.split(" ", 2) 1342 1343 # Parse the rest of the response 1344 parser = FeedParser() 1345 parser.feed(payload) 1346 msg = parser.close() 1347 msg["status"] = status 1348 1349 # Create httplib2.Response from the parsed headers. 1350 resp = httplib2.Response(msg) 1351 resp.reason = reason 1352 resp.version = int(protocol.split("/", 1)[1].replace(".", "")) 1353 1354 content = payload.split("\r\n\r\n", 1)[1] 1355 1356 return resp, content

1357

1358 - def _new_id(self):

1359 """Create a new id. 1360 1361 Auto incrementing number that avoids conflicts with ids already used. 1362 1363 Returns: 1364 string, a new unique id. 1365 """ 1366 self._last_auto_id += 1 1367 while str(self._last_auto_id) in self._requests: 1368 self._last_auto_id += 1 1369 return str(self._last_auto_id)

1370 1371 @util.positional(2)

1372 - def add(self, request, callback=None, request_id=None):

1373 """Add a new request. 1374 1375 Every callback added will be paired with a unique id, the request_id. That 1376 unique id will be passed back to the callback when the response comes back 1377 from the server. The default behavior is to have the library generate it's 1378 own unique id. If the caller passes in a request_id then they must ensure 1379 uniqueness for each request_id, and if they are not an exception is 1380 raised. Callers should either supply all request_ids or never supply a 1381 request id, to avoid such an error. 1382 1383 Args: 1384 request: HttpRequest, Request to add to the batch. 1385 callback: callable, A callback to be called for this response, of the 1386 form callback(id, response, exception). The first parameter is the 1387 request id, and the second is the deserialized response object. The 1388 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1389 occurred while processing the request, or None if no errors occurred. 1390 request_id: string, A unique id for the request. The id will be passed 1391 to the callback with the response. 1392 1393 Returns: 1394 None 1395 1396 Raises: 1397 BatchError if a media request is added to a batch. 1398 KeyError is the request_id is not unique. 1399 """ 1400 1401 if len(self._order) >= MAX_BATCH_LIMIT: 1402 raise BatchError( 1403 "Exceeded the maximum calls(%d) in a single batch request." 1404 % MAX_BATCH_LIMIT 1405 ) 1406 if request_id is None: 1407 request_id = self._new_id() 1408 if request.resumable is not None: 1409 raise BatchError("Media requests cannot be used in a batch request.") 1410 if request_id in self._requests: 1411 raise KeyError("A request with this ID already exists: %s" % request_id) 1412 self._requests[request_id] = request 1413 self._callbacks[request_id] = callback 1414 self._order.append(request_id)

1415

1416 - def _execute(self, http, order, requests):

1417 """Serialize batch request, send to server, process response. 1418 1419 Args: 1420 http: httplib2.Http, an http object to be used to make the request with. 1421 order: list, list of request ids in the order they were added to the 1422 batch. 1423 requests: list, list of request objects to send. 1424 1425 Raises: 1426 httplib2.HttpLib2Error if a transport error has occurred. 1427 googleapiclient.errors.BatchError if the response is the wrong format. 1428 """ 1429 message = MIMEMultipart("mixed") 1430 # Message should not write out it's own headers. 1431 setattr(message, "_write_headers", lambda self: None) 1432 1433 # Add all the individual requests. 1434 for request_id in order: 1435 request = requests[request_id] 1436 1437 msg = MIMENonMultipart("application", "http") 1438 msg["Content-Transfer-Encoding"] = "binary" 1439 msg["Content-ID"] = self._id_to_header(request_id) 1440 1441 body = self._serialize_request(request) 1442 msg.set_payload(body) 1443 message.attach(msg) 1444 1445 # encode the body: note that we can't use `as_string`, because 1446 # it plays games with `From ` lines. 1447 fp = StringIO() 1448 g = Generator(fp, mangle_from_=False) 1449 g.flatten(message, unixfrom=False) 1450 body = fp.getvalue() 1451 1452 headers = {} 1453 headers["content-type"] = ( 1454 "multipart/mixed; " 'boundary="%s"' 1455 ) % message.get_boundary() 1456 1457 resp, content = http.request( 1458 self._batch_uri, method="POST", body=body, headers=headers 1459 ) 1460 1461 if resp.status >= 300: 1462 raise HttpError(resp, content, uri=self._batch_uri) 1463 1464 # Prepend with a content-type header so FeedParser can handle it. 1465 header = "content-type: %s\r\n\r\n" % resp["content-type"] 1466 # PY3's FeedParser only accepts unicode. So we should decode content 1467 # here, and encode each payload again. 1468 if six.PY3: 1469 content = content.decode("utf-8") 1470 for_parser = header + content 1471 1472 parser = FeedParser() 1473 parser.feed(for_parser) 1474 mime_response = parser.close() 1475 1476 if not mime_response.is_multipart(): 1477 raise BatchError( 1478 "Response not in multipart/mixed format.", resp=resp, content=content 1479 ) 1480 1481 for part in mime_response.get_payload(): 1482 request_id = self._header_to_id(part["Content-ID"]) 1483 response, content = self._deserialize_response(part.get_payload()) 1484 # We encode content here to emulate normal http response. 1485 if isinstance(content, six.text_type): 1486 content = content.encode("utf-8") 1487 self._responses[request_id] = (response, content)

1488 1489 @util.positional(1)

1490 - def execute(self, http=None):

1491 """Execute all the requests as a single batched HTTP request. 1492 1493 Args: 1494 http: httplib2.Http, an http object to be used in place of the one the 1495 HttpRequest request object was constructed with. If one isn't supplied 1496 then use a http object from the requests in this batch. 1497 1498 Returns: 1499 None 1500 1501 Raises: 1502 httplib2.HttpLib2Error if a transport error has occurred. 1503 googleapiclient.errors.BatchError if the response is the wrong format. 1504 """ 1505 # If we have no requests return 1506 if len(self._order) == 0: 1507 return None 1508 1509 # If http is not supplied use the first valid one given in the requests. 1510 if http is None: 1511 for request_id in self._order: 1512 request = self._requests[request_id] 1513 if request is not None: 1514 http = request.http 1515 break 1516 1517 if http is None: 1518 raise ValueError("Missing a valid http object.") 1519 1520 # Special case for OAuth2Credentials-style objects which have not yet been 1521 # refreshed with an initial access_token. 1522 creds = _auth.get_credentials_from_http(http) 1523 if creds is not None: 1524 if not _auth.is_valid(creds): 1525 LOGGER.info("Attempting refresh to obtain initial access_token") 1526 _auth.refresh_credentials(creds) 1527 1528 self._execute(http, self._order, self._requests) 1529 1530 # Loop over all the requests and check for 401s. For each 401 request the 1531 # credentials should be refreshed and then sent again in a separate batch. 1532 redo_requests = {} 1533 redo_order = [] 1534 1535 for request_id in self._order: 1536 resp, content = self._responses[request_id] 1537 if resp["status"] == "401": 1538 redo_order.append(request_id) 1539 request = self._requests[request_id] 1540 self._refresh_and_apply_credentials(request, http) 1541 redo_requests[request_id] = request 1542 1543 if redo_requests: 1544 self._execute(http, redo_order, redo_requests) 1545 1546 # Now process all callbacks that are erroring, and raise an exception for 1547 # ones that return a non-2xx response? Or add extra parameter to callback 1548 # that contains an HttpError? 1549 1550 for request_id in self._order: 1551 resp, content = self._responses[request_id] 1552 1553 request = self._requests[request_id] 1554 callback = self._callbacks[request_id] 1555 1556 response = None 1557 exception = None 1558 try: 1559 if resp.status >= 300: 1560 raise HttpError(resp, content, uri=request.uri) 1561 response = request.postproc(resp, content) 1562 except HttpError as e: 1563 exception = e 1564 1565 if callback is not None: 1566 callback(request_id, response, exception) 1567 if self._callback is not None: 1568 self._callback(request_id, response, exception)

1569

1570 1571 -class HttpRequestMock(object):

1572 """Mock of HttpRequest. 1573 1574 Do not construct directly, instead use RequestMockBuilder. 1575 """ 1576

1577 - def __init__(self, resp, content, postproc):

1578 """Constructor for HttpRequestMock 1579 1580 Args: 1581 resp: httplib2.Response, the response to emulate coming from the request 1582 content: string, the response body 1583 postproc: callable, the post processing function usually supplied by 1584 the model class. See model.JsonModel.response() as an example. 1585 """ 1586 self.resp = resp 1587 self.content = content 1588 self.postproc = postproc 1589 if resp is None: 1590 self.resp = httplib2.Response({"status": 200, "reason": "OK"}) 1591 if "reason" in self.resp: 1592 self.resp.reason = self.resp["reason"]

1593

1594 - def execute(self, http=None):

1595 """Execute the request. 1596 1597 Same behavior as HttpRequest.execute(), but the response is 1598 mocked and not really from an HTTP request/response. 1599 """ 1600 return self.postproc(self.resp, self.content)

1601

1602 1603 -class RequestMockBuilder(object):

1604 """A simple mock of HttpRequest 1605 1606 Pass in a dictionary to the constructor that maps request methodIds to 1607 tuples of (httplib2.Response, content, opt_expected_body) that should be 1608 returned when that method is called. None may also be passed in for the 1609 httplib2.Response, in which case a 200 OK response will be generated. 1610 If an opt_expected_body (str or dict) is provided, it will be compared to 1611 the body and UnexpectedBodyError will be raised on inequality. 1612 1613 Example: 1614 response = '{"data": {"id": "tag:google.c...' 1615 requestBuilder = RequestMockBuilder( 1616 { 1617 'plus.activities.get': (None, response), 1618 } 1619 ) 1620 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1621 1622 Methods that you do not supply a response for will return a 1623 200 OK with an empty string as the response content or raise an excpetion 1624 if check_unexpected is set to True. The methodId is taken from the rpcName 1625 in the discovery document. 1626 1627 For more details see the project wiki. 1628 """ 1629

1630 - def __init__(self, responses, check_unexpected=False):

1631 """Constructor for RequestMockBuilder 1632 1633 The constructed object should be a callable object 1634 that can replace the class HttpResponse. 1635 1636 responses - A dictionary that maps methodIds into tuples 1637 of (httplib2.Response, content). The methodId 1638 comes from the 'rpcName' field in the discovery 1639 document. 1640 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1641 should be raised on unsupplied method. 1642 """ 1643 self.responses = responses 1644 self.check_unexpected = check_unexpected

1645

1646 - def __call__( 1647 self, 1648 http, 1649 postproc, 1650 uri, 1651 method="GET", 1652 body=None, 1653 headers=None, 1654 methodId=None, 1655 resumable=None, 1656 ):

1657 """Implements the callable interface that discovery.build() expects 1658 of requestBuilder, which is to build an object compatible with 1659 HttpRequest.execute(). See that method for the description of the 1660 parameters and the expected response. 1661 """ 1662 if methodId in self.responses: 1663 response = self.responses[methodId] 1664 resp, content = response[:2] 1665 if len(response) > 2: 1666 # Test the body against the supplied expected_body. 1667 expected_body = response[2] 1668 if bool(expected_body) != bool(body): 1669 # Not expecting a body and provided one 1670 # or expecting a body and not provided one. 1671 raise UnexpectedBodyError(expected_body, body) 1672 if isinstance(expected_body, str): 1673 expected_body = json.loads(expected_body) 1674 body = json.loads(body) 1675 if body != expected_body: 1676 raise UnexpectedBodyError(expected_body, body) 1677 return HttpRequestMock(resp, content, postproc) 1678 elif self.check_unexpected: 1679 raise UnexpectedMethodError(methodId=methodId) 1680 else: 1681 model = JsonModel(False) 1682 return HttpRequestMock(None, "{}", model.response)

1683

1684 1685 -class HttpMock(object):

1686 """Mock of httplib2.Http""" 1687

1688 - def __init__(self, filename=None, headers=None):

1689 """ 1690 Args: 1691 filename: string, absolute filename to read response from 1692 headers: dict, header to return with response 1693 """ 1694 if headers is None: 1695 headers = {"status": "200"} 1696 if filename: 1697 with open(filename, "rb") as f: 1698 self.data = f.read() 1699 else: 1700 self.data = None 1701 self.response_headers = headers 1702 self.headers = None 1703 self.uri = None 1704 self.method = None 1705 self.body = None 1706 self.headers = None

1707

1708 - def request( 1709 self, 1710 uri, 1711 method="GET", 1712 body=None, 1713 headers=None, 1714 redirections=1, 1715 connection_type=None, 1716 ):

1717 self.uri = uri 1718 self.method = method 1719 self.body = body 1720 self.headers = headers 1721 return httplib2.Response(self.response_headers), self.data

1722

1723 - def close(self):

1724 return None

1725

1726 -class HttpMockSequence(object):

1727 """Mock of httplib2.Http 1728 1729 Mocks a sequence of calls to request returning different responses for each 1730 call. Create an instance initialized with the desired response headers 1731 and content and then use as if an httplib2.Http instance. 1732 1733 http = HttpMockSequence([ 1734 ({'status': '401'}, ''), 1735 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1736 ({'status': '200'}, 'echo_request_headers'), 1737 ]) 1738 resp, content = http.request("http://examples.com") 1739 1740 There are special values you can pass in for content to trigger 1741 behavours that are helpful in testing. 1742 1743 'echo_request_headers' means return the request headers in the response body 1744 'echo_request_headers_as_json' means return the request headers in 1745 the response body 1746 'echo_request_body' means return the request body in the response body 1747 'echo_request_uri' means return the request uri in the response body 1748 """ 1749

1750 - def __init__(self, iterable):

1751 """ 1752 Args: 1753 iterable: iterable, a sequence of pairs of (headers, body) 1754 """ 1755 self._iterable = iterable 1756 self.follow_redirects = True 1757 self.request_sequence = list()

1758

1759 - def request( 1760 self, 1761 uri, 1762 method="GET", 1763 body=None, 1764 headers=None, 1765 redirections=1, 1766 connection_type=None, 1767 ):

1768 # Remember the request so after the fact this mock can be examined 1769 self.request_sequence.append((uri, method, body, headers)) 1770 resp, content = self._iterable.pop(0) 1771 content = six.ensure_binary(content) 1772 1773 if content == b"echo_request_headers": 1774 content = headers 1775 elif content == b"echo_request_headers_as_json": 1776 content = json.dumps(headers) 1777 elif content == b"echo_request_body": 1778 if hasattr(body, "read"): 1779 content = body.read() 1780 else: 1781 content = body 1782 elif content == b"echo_request_uri": 1783 content = uri 1784 if isinstance(content, six.text_type): 1785 content = content.encode("utf-8") 1786 return httplib2.Response(resp), content

1787

1788 1789 -def set_user_agent(http, user_agent):

1790 """Set the user-agent on every request. 1791 1792 Args: 1793 http - An instance of httplib2.Http 1794 or something that acts like it. 1795 user_agent: string, the value for the user-agent header. 1796 1797 Returns: 1798 A modified instance of http that was passed in. 1799 1800 Example: 1801 1802 h = httplib2.Http() 1803 h = set_user_agent(h, "my-app-name/6.0") 1804 1805 Most of the time the user-agent will be set doing auth, this is for the rare 1806 cases where you are accessing an unauthenticated endpoint. 1807 """ 1808 request_orig = http.request 1809 1810 # The closure that will replace 'httplib2.Http.request'. 1811 def new_request( 1812 uri, 1813 method="GET", 1814 body=None, 1815 headers=None, 1816 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1817 connection_type=None, 1818 ): 1819 """Modify the request headers to add the user-agent.""" 1820 if headers is None: 1821 headers = {} 1822 if "user-agent" in headers: 1823 headers["user-agent"] = user_agent + " " + headers["user-agent"] 1824 else: 1825 headers["user-agent"] = user_agent 1826 resp, content = request_orig( 1827 uri, 1828 method=method, 1829 body=body, 1830 headers=headers, 1831 redirections=redirections, 1832 connection_type=connection_type, 1833 ) 1834 return resp, content

1835 1836 http.request = new_request 1837 return http 1838

1839 1840 -def tunnel_patch(http):

1841 """Tunnel PATCH requests over POST. 1842 Args: 1843 http - An instance of httplib2.Http 1844 or something that acts like it. 1845 1846 Returns: 1847 A modified instance of http that was passed in. 1848 1849 Example: 1850 1851 h = httplib2.Http() 1852 h = tunnel_patch(h, "my-app-name/6.0") 1853 1854 Useful if you are running on a platform that doesn't support PATCH. 1855 Apply this last if you are using OAuth 1.0, as changing the method 1856 will result in a different signature. 1857 """ 1858 request_orig = http.request 1859 1860 # The closure that will replace 'httplib2.Http.request'. 1861 def new_request( 1862 uri, 1863 method="GET", 1864 body=None, 1865 headers=None, 1866 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1867 connection_type=None, 1868 ): 1869 """Modify the request headers to add the user-agent.""" 1870 if headers is None: 1871 headers = {} 1872 if method == "PATCH": 1873 if "oauth_token" in headers.get("authorization", ""): 1874 LOGGER.warning( 1875 "OAuth 1.0 request made with Credentials after tunnel_patch." 1876 ) 1877 headers["x-http-method-override"] = "PATCH" 1878 method = "POST" 1879 resp, content = request_orig( 1880 uri, 1881 method=method, 1882 body=body, 1883 headers=headers, 1884 redirections=redirections, 1885 connection_type=connection_type, 1886 ) 1887 return resp, content

1888 1889 http.request = new_request 1890 return http 1891

1892 1893 -def build_http():

1894 """Builds httplib2.Http object 1895 1896 Returns: 1897 A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 1898 To override default timeout call 1899 1900 socket.setdefaulttimeout(timeout_in_sec) 1901 1902 before interacting with this method. 1903 """ 1904 if socket.getdefaulttimeout() is not None: 1905 http_timeout = socket.getdefaulttimeout() 1906 else: 1907 http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 1908 http = httplib2.Http(timeout=http_timeout) 1909 # 308's are used by several Google APIs (Drive, YouTube) 1910 # for Resumable Uploads rather than Permanent Redirects. 1911 # This asks httplib2 to exclude 308s from the status codes 1912 # it treats as redirects 1913 try: 1914 http.redirect_codes = http.redirect_codes - {308} 1915 except AttributeError: 1916 # Apache Beam tests depend on this library and cannot 1917 # currently upgrade their httplib2 version 1918 # http.redirect_codes does not exist in previous versions 1919 # of httplib2, so pass 1920 pass 1921 1922 return http

1923

Source Code for Module googleapiclient.http