Skip to content

FinceptTerminal is a modern finance application offering advanced market analytics, investment research, and economic data tools, designed for interactive exploration and data-driven decision-making in a user-friendly environment.

Add data.gov.hk (Hong Kong Gov) API wrapper

Tanvi Kamath66b0dc435c8c
diff --git a/fincept-terminal-desktop/src-tauri/resources/scripts/data_gov_hk_api.py b/fincept-terminal-desktop/src-tauri/resources/scripts/data_gov_hk_api.py
new file mode 100644
index 000000000..938ff3078
--- /dev/null
+++ b/fincept-terminal-desktop/src-tauri/resources/scripts/data_gov_hk_api.py
@@ -0,0 +1,675 @@
+#!/usr/bin/env python3
+"""
+Data.gov.hk API Wrapper
+A comprehensive wrapper for Hong Kong Government Data Portal APIs
+
+Provides access to:
+1. CKAN Catalog API - for dataset metadata and categories
+2. Historical Archive API - for historical data files
+
+Features:
+- Multi-language support with English translation
+- Modular endpoint design
+- Comprehensive error handling
+- Independent endpoint failure handling
+- Rich metadata and response formatting
+"""
+
+import sys
+import json
+import requests
+import os
+from datetime import datetime
+from typing import Dict, List, Optional, Any, Union
+from urllib.parse import urlencode
+
+class DataGovHKWrapper:
+ """
+ Comprehensive wrapper for Data.gov.hk APIs
+ """
+
+ def __init__(self, language: str = "en"):
+ """
+ Initialize the Data.gov.hk API wrapper
+
+ Args:
+ language: Language preference - 'en' (English), 'tc' (Traditional Chinese), 'sc' (Simplified Chinese)
+ """
+ self.language = language.lower()
+
+ # Base URLs for different languages
+ self.base_urls = {
+ 'en': 'https://data.gov.hk/en-data/api/3/action/',
+ 'tc': 'https://data.gov.hk/tc-data/api/3/action/',
+ 'sc': 'https://data.gov.hk/sc-data/api/3/action/'
+ }
+
+ # Historical Archive API (language independent)
+ self.historical_base_url = 'https://app.data.gov.hk'
+
+ # Validate language
+ if self.language not in self.base_urls:
+ raise ValueError(f"Unsupported language: {language}. Use 'en', 'tc', or 'sc'")
+
+ self.base_url = self.base_urls[self.language]
+
+ # Session configuration
+ self.session = requests.Session()
+ self.session.headers.update({
+ 'User-Agent': 'Fincept-Terminal/1.0',
+ 'Accept': 'application/json',
+ 'Accept-Language': f'{self.language}-HK,{self.language};q=0.9,en;q=0.8'
+ })
+
+ self.timeout = 30
+
+ def _make_ckan_request(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+ """
+ Make a request to CKAN API endpoint
+
+ Args:
+ endpoint: CKAN action endpoint name
+ params: Query parameters
+
+ Returns:
+ Standardized response dictionary
+ """
+ try:
+ url = f"{self.base_url}{endpoint}"
+
+ # Clean up params (remove None values)
+ clean_params = {k: v for k, v in (params or {}).items() if v is not None}
+
+ response = self.session.get(url, params=clean_params, timeout=self.timeout)
+ response.raise_for_status()
+
+ data = response.json()
+
+ # CKAN API standard response format: {success: bool, result: any, help: string}
+ if data.get('success'):
+ return {
+ "data": data.get('result', []),
+ "metadata": {
+ "source": "Data.gov.hk CKAN API",
+ "endpoint": endpoint,
+ "language": self.language,
+ "last_updated": datetime.now().isoformat(),
+ "request_params": clean_params,
+ "help": data.get('help', '')
+ },
+ "error": None
+ }
+ else:
+ return {
+ "data": [],
+ "metadata": {
+ "source": "Data.gov.hk CKAN API",
+ "endpoint": endpoint,
+ "language": self.language,
+ "last_updated": datetime.now().isoformat(),
+ "request_params": clean_params
+ },
+ "error": data.get('error', {}).get('message', 'Unknown CKAN API error')
+ }
+
+ except requests.exceptions.Timeout:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk CKAN API", "endpoint": endpoint},
+ "error": "Request timeout. The API took too long to respond."
+ }
+ except requests.exceptions.ConnectionError:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk CKAN API", "endpoint": endpoint},
+ "error": "Connection error. Unable to connect to Data.gov.hk API."
+ }
+ except requests.exceptions.HTTPError as e:
+ status_code = e.response.status_code
+ error_messages = {
+ 400: "Bad request. Invalid parameters.",
+ 404: "Endpoint not found.",
+ 429: "Rate limit exceeded. Please try again later.",
+ 500: "Internal server error."
+ }
+ error_msg = error_messages.get(status_code, f"HTTP Error {status_code}")
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk CKAN API", "endpoint": endpoint},
+ "error": error_msg
+ }
+ except json.JSONDecodeError:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk CKAN API", "endpoint": endpoint},
+ "error": "Invalid JSON response from API."
+ }
+ except Exception as e:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk CKAN API", "endpoint": endpoint},
+ "error": f"Unexpected error: {str(e)}"
+ }
+
+ def _make_historical_request(self, params: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Make a request to Historical Archive API
+
+ Args:
+ params: Query parameters for historical archive
+
+ Returns:
+ Standardized response dictionary
+ """
+ try:
+ url = f"{self.historical_base_url}/v1/historical-archive/list-files"
+
+ # Clean up params
+ clean_params = {k: v for k, v in params.items() if v is not None}
+
+ response = self.session.get(url, params=clean_params, timeout=self.timeout)
+ response.raise_for_status()
+
+ data = response.json()
+
+ return {
+ "data": data,
+ "metadata": {
+ "source": "Data.gov.hk Historical Archive API",
+ "endpoint": "list-files",
+ "last_updated": datetime.now().isoformat(),
+ "request_params": clean_params
+ },
+ "error": None
+ }
+
+ except requests.exceptions.Timeout:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API", "endpoint": "list-files"},
+ "error": "Request timeout. The historical archive API took too long to respond."
+ }
+ except requests.exceptions.ConnectionError:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API", "endpoint": "list-files"},
+ "error": "Connection error. Unable to connect to historical archive API."
+ }
+ except requests.exceptions.HTTPError as e:
+ status_code = e.response.status_code
+ error_messages = {
+ 400: "Bad request. Invalid date range or parameters.",
+ 404: "Historical archive endpoint not found.",
+ 429: "Rate limit exceeded. Please try again later.",
+ 500: "Internal server error in historical archive."
+ }
+ error_msg = error_messages.get(status_code, f"HTTP Error {status_code}")
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API", "endpoint": "list-files"},
+ "error": error_msg
+ }
+ except json.JSONDecodeError:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API", "endpoint": "list-files"},
+ "error": "Invalid JSON response from historical archive API."
+ }
+ except Exception as e:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API", "endpoint": "list-files"},
+ "error": f"Unexpected error: {str(e)}"
+ }
+
+ def _translate_text_if_needed(self, text: str) -> str:
+ """
+ Simple translation for common Chinese terms to English
+ In a real implementation, you might use a translation service
+
+ Args:
+ text: Text to translate
+
+ Returns:
+ Translated text if it's Chinese, original text if English
+ """
+ if self.language == 'en':
+ return text
+
+ # Common translations for data.gov.hk
+ translations = {
+ # Traditional Chinese
+ '環境': 'Environment',
+ '交通': 'Transportation',
+ '政府': 'Government',
+ '經濟': 'Economy',
+ '社會': 'Social',
+ '教育': 'Education',
+ '醫療': 'Healthcare',
+ '房屋': 'Housing',
+ '規劃': 'Planning',
+ '統計': 'Statistics',
+ # Simplified Chinese
+ '环境': 'Environment',
+ '交通': 'Transportation',
+ '政府': 'Government',
+ '经济': 'Economy',
+ '社会': 'Social',
+ '教育': 'Education',
+ '医疗': 'Healthcare',
+ '房屋': 'Housing',
+ '规划': 'Planning',
+ '统计': 'Statistics'
+ }
+
+ return translations.get(text, text)
+
+ # === CATALOGUE/CATEGORY ENDPOINTS ===
+
+ def get_categories_list(self) -> Dict[str, Any]:
+ """
+ Get list of all available categories (groups)
+
+ Returns:
+ List of category names
+ """
+ return self._make_ckan_request('group_list')
+
+ def get_category_details(self, category_id: str) -> Dict[str, Any]:
+ """
+ Get detailed information about a specific category including datasets
+
+ Args:
+ category_id: Unique category identifier (e.g., 'environment', 'transport')
+
+ Returns:
+ Category details with datasets list
+ """
+ if not category_id:
+ return {
+ "data": {},
+ "metadata": {"source": "Data.gov.hk CKAN API"},
+ "error": "Category ID is required"
+ }
+
+ return self._make_ckan_request('group_show', {'id': category_id})
+
+ # === DATASET ENDPOINTS ===
+
+ def get_datasets_list(self, limit: Optional[int] = None, offset: Optional[int] = None) -> Dict[str, Any]:
+ """
+ Get list of all datasets (packages) with pagination
+
+ Args:
+ limit: Maximum number of datasets to return
+ offset: Number of datasets to skip (for pagination)
+
+ Returns:
+ List of dataset names
+ """
+ params = {}
+ if limit is not None:
+ params['limit'] = limit
+ if offset is not None:
+ params['offset'] = offset
+
+ return self._make_ckan_request('package_list', params)
+
+ def get_dataset_details(self, dataset_id: str) -> Dict[str, Any]:
+ """
+ Get full metadata for a specific dataset
+
+ Args:
+ dataset_id: Unique dataset identifier or name
+
+ Returns:
+ Complete dataset metadata including resources, tags, etc.
+ """
+ if not dataset_id:
+ return {
+ "data": {},
+ "metadata": {"source": "Data.gov.hk CKAN API"},
+ "error": "Dataset ID is required"
+ }
+
+ return self._make_ckan_request('package_show', {'id': dataset_id})
+
+ # === HISTORICAL ARCHIVE ENDPOINTS ===
+
+ def get_historical_files(self, start_date: str, end_date: str, category: Optional[str] = None,
+ format_filter: Optional[str] = None, search: Optional[str] = None,
+ order: Optional[str] = None, skip: Optional[int] = None,
+ max_results: Optional[int] = None) -> Dict[str, Any]:
+ """
+ Get historical data files within date range
+
+ Args:
+ start_date: Start date in YYYYMMDD format (required)
+ end_date: End date in YYYYMMDD format (required)
+ category: Category ID filter (optional)
+ format_filter: File format extension (e.g., 'xls', 'csv') (optional)
+ search: Keyword search for dataset names (optional)
+ order: Sort order (optional)
+ skip: Number of results to skip (optional)
+ max_results: Maximum results to return (optional)
+
+ Returns:
+ List of historical files matching criteria
+ """
+ # Validate required parameters
+ if not start_date or not end_date:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API"},
+ "error": "Both start_date and end_date are required in YYYYMMDD format"
+ }
+
+ # Validate date format
+ if len(start_date) != 8 or len(end_date) != 8:
+ return {
+ "data": [],
+ "metadata": {"source": "Data.gov.hk Historical Archive API"},
+ "error": "Dates must be in YYYYMMDD format"
+ }
+
+ params = {
+ 'start': start_date,
+ 'end': end_date
+ }
+
+ # Add optional parameters
+ if category:
+ params['category'] = category
+ if format_filter:
+ params['format'] = format_filter
+ if search:
+ params['search'] = search
+ if order:
+ params['order'] = order
+ if skip is not None:
+ params['skip'] = skip
+ if max_results is not None:
+ params['max'] = max_results
+
+ return self._make_historical_request(params)
+
+ # === COMPOSITE METHODS ===
+
+ def get_catalogue_overview(self) -> Dict[str, Any]:
+ """
+ Get complete catalogue overview with categories and sample datasets
+
+ Returns:
+ Comprehensive catalogue data with categories and dataset counts
+ """
+ result = {
+ "success": True,
+ "timestamp": datetime.now().isoformat(),
+ "catalogue": {},
+ "failed_operations": []
+ }
+
+ # Get categories list
+ try:
+ categories_result = self.get_categories_list()
+ if categories_result.get('error'):
+ result["failed_operations"].append({
+ "operation": "get_categories_list",
+ "error": categories_result['error']
+ })
+ result["success"] = False
+ else:
+ categories = categories_result.get('data', [])
+ result["catalogue"]["categories"] = categories
+ result["catalogue"]["category_count"] = len(categories)
+
+ # Get details for first 5 categories as samples
+ sample_categories = categories[:5]
+ category_details = {}
+
+ for category_id in sample_categories:
+ try:
+ category_result = self.get_category_details(category_id)
+ if not category_result.get('error'):
+ category_details[category_id] = {
+ "name": self._translate_text_if_needed(category_id),
+ "dataset_count": len(category_result.get('data', {}).get('packages', [])),
+ "details": category_result['data']
+ }
+ else:
+ result["failed_operations"].append({
+ "operation": f"get_category_details({category_id})",
+ "error": category_result['error']
+ })
+ except Exception as e:
+ result["failed_operations"].append({
+ "operation": f"get_category_details({category_id})",
+ "error": str(e)
+ })
+
+ result["catalogue"]["sample_category_details"] = category_details
+
+ except Exception as e:
+ result["failed_operations"].append({
+ "operation": "get_categories_list",
+ "error": str(e)
+ })
+ result["success"] = False
+
+ # Get sample datasets
+ try:
+ datasets_result = self.get_datasets_list(limit=10)
+ if datasets_result.get('error'):
+ result["failed_operations"].append({
+ "operation": "get_datasets_list",
+ "error": datasets_result['error']
+ })
+ else:
+ result["catalogue"]["sample_datasets"] = datasets_result.get('data', [])
+
+ except Exception as e:
+ result["failed_operations"].append({
+ "operation": "get_datasets_list",
+ "error": str(e)
+ })
+
+ result["metadata"] = {
+ "source": "Data.gov.hk API",
+ "language": self.language,
+ "operation": "catalogue_overview",
+ "last_updated": datetime.now().isoformat()
+ }
+
+ return result
+
+ def get_all_category_datasets(self, category_id: str) -> Dict[str, Any]:
+ """
+ Get all datasets within a specific category
+
+ Args:
+ category_id: Category identifier
+
+ Returns:
+ All datasets in the category with enhanced metadata
+ """
+ result = {
+ "success": True,
+ "timestamp": datetime.now().isoformat(),
+ "category_id": category_id,
+ "datasets": {},
+ "failed_datasets": []
+ }
+
+ try:
+ # Get category details
+ category_result = self.get_category_details(category_id)
+ if category_result.get('error'):
+ return {
+ "success": False,
+ "error": category_result['error'],
+ "category_id": category_id
+ }
+
+ category_data = category_result.get('data', {})
+ packages = category_data.get('packages', [])
+
+ result["category_info"] = {
+ "name": self._translate_text_if_needed(category_id),
+ "title": category_data.get('title', category_id),
+ "description": category_data.get('description', ''),
+ "dataset_count": len(packages)
+ }
+
+ # Get details for each dataset
+ for package in packages:
+ package_name = package.get('name', '')
+ if package_name:
+ try:
+ dataset_result = self.get_dataset_details(package_name)
+ if not dataset_result.get('error'):
+ result["datasets"][package_name] = {
+ "name": package_name,
+ "title": dataset_result['data'].get('title', ''),
+ "notes": dataset_result['data'].get('notes', ''),
+ "resources": dataset_result['data'].get('resources', []),
+ "tags": [tag.get('display_name', '') for tag in dataset_result['data'].get('tags', [])],
+ "metadata": dataset_result['data']
+ }
+ else:
+ result["failed_datasets"].append({
+ "dataset": package_name,
+ "error": dataset_result['error']
+ })
+ except Exception as e:
+ result["failed_datasets"].append({
+ "dataset": package_name,
+ "error": str(e)
+ })
+
+ result["success"] = len(result["failed_datasets"]) < len(packages)
+
+ except Exception as e:
+ return {
+ "success": False,
+ "error": f"Failed to retrieve category datasets: {str(e)}",
+ "category_id": category_id
+ }
+
+ result["metadata"] = {
+ "source": "Data.gov.hk API",
+ "language": self.language,
+ "operation": "get_all_category_datasets",
+ "last_updated": datetime.now().isoformat()
+ }
+
+ return result
+
+
+def main():
+ """
+ CLI interface for Data.gov.hk API wrapper
+ """
+ if len(sys.argv) < 2:
+ print(json.dumps({
+ "error": "Usage: python data_gov_hk_api.py <command> [args]",
+ "available_commands": [
+ "categories_list",
+ "category_details <category_id>",
+ "datasets_list [limit] [offset]",
+ "dataset_details <dataset_id>",
+ "historical_files <start_date> <end_date> [category] [format]",
+ "catalogue_overview",
+ "category_datasets <category_id>",
+ "test_all"
+ ]
+ }, indent=2))
+ sys.exit(1)
+
+ command = sys.argv[1]
+
+ # Initialize wrapper with English language by default
+ language = os.environ.get('DATA_GOV_HK_LANGUAGE', 'en')
+ wrapper = DataGovHKWrapper(language=language)
+
+ try:
+ if command == "categories_list":
+ result = wrapper.get_categories_list()
+
+ elif command == "category_details":
+ if len(sys.argv) < 3:
+ print(json.dumps({"error": "Category ID is required"}))
+ sys.exit(1)
+ category_id = sys.argv[2]
+ result = wrapper.get_category_details(category_id)
+
+ elif command == "datasets_list":
+ limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
+ offset = int(sys.argv[3]) if len(sys.argv) > 3 else None
+ result = wrapper.get_datasets_list(limit=limit, offset=offset)
+
+ elif command == "dataset_details":
+ if len(sys.argv) < 3:
+ print(json.dumps({"error": "Dataset ID is required"}))
+ sys.exit(1)
+ dataset_id = sys.argv[2]
+ result = wrapper.get_dataset_details(dataset_id)
+
+ elif command == "historical_files":
+ if len(sys.argv) < 4:
+ print(json.dumps({"error": "Start date and end date are required"}))
+ sys.exit(1)
+ start_date = sys.argv[2]
+ end_date = sys.argv[3]
+ category = sys.argv[4] if len(sys.argv) > 4 else None
+ format_filter = sys.argv[5] if len(sys.argv) > 5 else None
+ result = wrapper.get_historical_files(start_date, end_date, category, format_filter)
+
+ elif command == "catalogue_overview":
+ result = wrapper.get_catalogue_overview()
+
+ elif command == "category_datasets":
+ if len(sys.argv) < 3:
+ print(json.dumps({"error": "Category ID is required"}))
+ sys.exit(1)
+ category_id = sys.argv[2]
+ result = wrapper.get_all_category_datasets(category_id)
+
+ elif command == "test_all":
+ # Test all endpoints
+ test_results = {
+ "test_suite": "Data.gov.hk API",
+ "timestamp": datetime.now().isoformat(),
+ "tests": {}
+ }
+
+ # Test categories list
+ test_results["tests"]["categories_list"] = wrapper.get_categories_list()
+
+ # Test datasets list
+ test_results["tests"]["datasets_list"] = wrapper.get_datasets_list(limit=5)
+
+ # Test historical files (last 30 days)
+ end_date = datetime.now().strftime('%Y%m%d')
+ start_date = (datetime.now().replace(day=1)).strftime('%Y%m%d') # First day of current month
+ test_results["tests"]["historical_files"] = wrapper.get_historical_files(start_date, end_date)
+
+ # Test catalogue overview
+ test_results["tests"]["catalogue_overview"] = wrapper.get_catalogue_overview()
+
+ result = test_results
+
+ else:
+ result = {"error": f"Unknown command: {command}"}
+
+ print(json.dumps(result, indent=2, ensure_ascii=False))
+
+ except ValueError as e:
+ print(json.dumps({"error": f"Invalid parameter: {str(e)}"}))
+ sys.exit(1)
+ except Exception as e:
+ print(json.dumps({"error": f"Command execution failed: {str(e)}"}))
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file