A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
"""HTTP utilities for devable-research-agent (stdlib only, adapted from last30days)."""
import json
import os
import sys
import time
import urllib.error
import urllib.request
from contextlib import nullcontext
from typing import Any, Dict, Optional
from urllib.parse import urlencode
from .net import force_ipv4_only, is_ipv6_unreachable
DEFAULT_TIMEOUT = 30
MAX_RETRIES = 5
RETRY_DELAY = 2.0
USER_AGENT = "devable-research-agent/1.0 (AI Industry Tracker)"
DEBUG = os.environ.get("MORNING_AI_DEBUG", "").lower() in ("1", "true", "yes")
def log(msg: str):
if DEBUG:
sys.stderr.write(f"[HTTP] {msg}\n")
sys.stderr.flush()
class HTTPError(Exception):
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
super().__init__(message)
self.status_code = status_code
self.body = body
def request(
method: str,
url: str,
headers: Optional[Dict[str, str]] = None,
json_data: Optional[Dict[str, Any]] = None,
timeout: int = DEFAULT_TIMEOUT,
retries: int = MAX_RETRIES,
raw: bool = False,
) -> Any:
headers = headers or {}
headers.setdefault("User-Agent", USER_AGENT)
data = None
if json_data is not None:
data = json.dumps(json_data).encode("utf-8")
headers.setdefault("Content-Type", "application/json")
req = urllib.request.Request(url, data=data, headers=headers, method=method)
log(f"{method} {url}")
last_error = None
ipv4_fallback = False
for attempt in range(retries):
ctx = force_ipv4_only() if ipv4_fallback else nullcontext()
try:
with ctx, urllib.request.urlopen(req, timeout=timeout) as response:
body = response.read().decode("utf-8")
log(f"Response: {response.status} ({len(body)} bytes)")
if raw:
return body
return json.loads(body) if body else {}
except urllib.error.HTTPError as e:
body = None
try:
body = e.read().decode("utf-8")
except Exception:
pass
log(f"HTTP Error {e.code}: {e.reason}")
last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
if 400 <= e.code < 500 and e.code != 429:
raise last_error
if attempt < retries - 1:
if e.code == 429:
retry_after = e.headers.get("Retry-After") if hasattr(e, "headers") else None
if retry_after:
try:
delay = float(retry_after)
except ValueError:
delay = RETRY_DELAY * (2 ** attempt) + 1
else:
delay = RETRY_DELAY * (2 ** attempt) + 1
log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
else:
delay = RETRY_DELAY * (2 ** attempt)
time.sleep(delay)
except urllib.error.URLError as e:
log(f"URL Error: {e.reason}")
last_error = HTTPError(f"URL Error: {e.reason}")
# Broken IPv6 routing: retry once IPv4-only without burning the backoff.
if not ipv4_fallback and is_ipv6_unreachable(e):
log("IPv6 unreachable, switching to IPv4-only DNS for remaining retries")
ipv4_fallback = True
continue
if attempt < retries - 1:
time.sleep(RETRY_DELAY * (attempt + 1))
except json.JSONDecodeError as e:
last_error = HTTPError(f"Invalid JSON response: {e}")
raise last_error
except (OSError, TimeoutError, ConnectionResetError) as e:
log(f"Connection error: {type(e).__name__}: {e}")
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
if not ipv4_fallback and is_ipv6_unreachable(e):
log("IPv6 unreachable, switching to IPv4-only DNS for remaining retries")
ipv4_fallback = True
continue
if attempt < retries - 1:
time.sleep(RETRY_DELAY * (attempt + 1))
if last_error:
raise last_error
raise HTTPError("Request failed with no error details")
def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Any:
return request("GET", url, headers=headers, **kwargs)
def get_text(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> str:
"""GET request returning raw text (not parsed as JSON)."""
return request("GET", url, headers=headers, raw=True, **kwargs)
def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Any:
return request("POST", url, headers=headers, json_data=json_data, **kwargs)