Browse Source

timeout for http fetch on py3

Thomas B 6 months ago
parent
commit
8ba783c882
1 changed files with 40 additions and 17 deletions
  1. 40
    17
      macros.py

+ 40
- 17
macros.py View File

20
 if PY3:
20
 if PY3:
21
     import urllib
21
     import urllib
22
     import urllib.request
22
     import urllib.request
23
+    from urllib.error import HTTPError, URLError
23
     def urlparse_foo(link):
24
     def urlparse_foo(link):
24
         return urllib.parse.parse_qs(urllib.parse.urlparse(link).query)['v'][0]
25
         return urllib.parse.parse_qs(urllib.parse.urlparse(link).query)['v'][0]
25
 else:
26
 else:
494
 
495
 
495
 import json, sys
496
 import json, sys
496
 
497
 
497
-def restRequest(url):
498
-    response = urllib.request.urlopen(url) if PY3 else urllib.urlopen(url)
498
+def print_cnsl_error(s, url):
499
+    sys.stderr.write("\n")
500
+    sys.stderr.write("warning: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
501
+    sys.stderr.write("warning: !!!!!!!                  WARNING                 !!!!!\n")
502
+    sys.stderr.write("warning: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
503
+    sys.stderr.write("warning: " + s + "\n")
504
+    sys.stderr.write("warning: URL: \"" + url + "\"\n")
505
+    sys.stderr.write("warning: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
506
+    sys.stderr.write("warning: !!!!!!!                  WARNING                 !!!!!\n")
507
+    sys.stderr.write("warning: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
508
+    sys.stderr.write("\n")
509
+
510
+def http_request(url):
511
+    sys.stderr.write('sub    : fetching %s\n' % url)
512
+
513
+    if PY3:
514
+        try:
515
+            response = urllib.request.urlopen(url, timeout = 10)
516
+        except HTTPError as error:
517
+            print_cnsl_error("HTTPError: '%s'" % error)
518
+            return ""
519
+        except URLError as error:
520
+            print_cnsl_error("URLError: '%s'" % error)
521
+            return ""
522
+    else:
523
+        try:
524
+            response = urllib.urlopen(url)
525
+        except IOError as error:
526
+            print_cnsl_error("HTTPError: '%s'" % error)
527
+            return ""
528
+
499
     if response.getcode() != 200:
529
     if response.getcode() != 200:
500
-        sys.stderr.write("\n")
501
-        sys.stderr.write("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
502
-        sys.stderr.write("!!!!!!!                  WARNING                 !!!!!\n")
503
-        sys.stderr.write("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
504
-        sys.stderr.write("invalid response code: " + str(response.getcode()) + "\n")
505
-        sys.stderr.write("url: \"" + url + "\"\n")
506
-        sys.stderr.write("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
507
-        sys.stderr.write("!!!!!!!                  WARNING                 !!!!!\n")
508
-        sys.stderr.write("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
509
-        sys.stderr.write("\n")
530
+        print_cnsl_error("invalid response code: " + str(response.getcode()), url)
510
         return ""
531
         return ""
511
-    data = json.loads(response.read().decode("utf-8"))
532
+    else:
533
+        data = response.read().decode("utf-8")
534
+        return data
535
+
536
+def restRequest(url):
537
+    data = json.loads(http_request(url))
512
     return data
538
     return data
513
 
539
 
514
 def restReleases(user, repo):
540
 def restReleases(user, repo):
556
     print("</ul></div>")
582
     print("</ul></div>")
557
 
583
 
558
 def include_url(url):
584
 def include_url(url):
559
-    response = urllib.request.urlopen(url) if PY3 else urllib.urlopen(url)
560
-    if response.getcode() != 200:
561
-        raise Exception("invalid response code", response.getcode())
562
-    data = response.read().decode("utf-8")
585
+    data = http_request(url)
563
     print(data, end="")
586
     print(data, end="")
564
 
587
 
565
 # -----------------------------------------------------------------------------
588
 # -----------------------------------------------------------------------------

Loading…
Cancel
Save