Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove exc checks (hash)
NTFSvolumecfe0b48edd6e
diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.pyindex 8e8fb5d0b..95d2d6306 100644--- a/cyberdrop_dl/database/hash.py+++ b/cyberdrop_dl/database/hash.py@@ -5,6 +5,8 @@ import loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, cast+from cyberdrop_dl import aio+from .common import Tablefrom .definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEX@@ -31,133 +33,133 @@ class HashTable(Table, name="hash"):await self.db_conn.commit()async def get_file_hash_exists(self, path: Path | str, hash_type: str) -> str | None:- query = "SELECT hash FROM hash WHERE folder=? AND download_filename=? AND hash_type=? AND hash IS NOT NULL"- try:- path = self.cwd / path- folder = str(path.parent)- filename = path.name-- # Check if the file exists with matching folder, filename, and size- cursor = await self.db_conn.execute(query, (folder, filename, hash_type))- if row := await cursor.fetchone():- return row[0]-- except Exception:- logger.exception("Error checking file")+ query = "SELECT hash FROM hash WHERE folder= ? AND download_filename= ? AND hash_type= ? AND hash IS NOT NULL LIMIT 1;"+ path = self.cwd / path+ folder = str(path.parent)+ filename = path.name+ cursor = await self.db_conn.execute(query, (folder, filename, hash_type))+ if row := await cursor.fetchone():+ return row["hash"]async def get_files_with_hash_matches(- self, hash_value: str, size: int, hash_type: str | None = None+ self,+ hash_value: str,+ size: int,+ hash_algo: str | None = None,) -> list[aiosqlite.Row]:- """Retrieves a list of (folder, filename) tuples based on a given hash.-- Args:- hash_value: The hash value to search for.- size: file size-- Returns:- A list of (folder, filename) tuples, or an empty list if no matches found.- """- if hash_type:+ if hash_algo:query = """- SELECT files.folder, files.download_filename,files.date- FROM hash JOIN files ON hash.folder = files.folder AND hash.download_filename = files.download_filename- WHERE hash.hash = ? AND files.file_size = ? AND hash.hash_type = ?;+ SELECT+ files.folder,+ files.download_filename,+ files.date+ FROM+ hash+ JOIN files ON hash.folder = files.folder+ AND hash.download_filename = files.download_filename+ WHERE+ hash.hash = ?+ AND files.file_size = ?+ AND hash.hash_type = ?;"""else:query = """- SELECT files.folder, files.download_filename FROM hash JOIN files- ON hash.folder = files.folder AND hash.download_filename = files.download_filename- WHERE hash.hash = ? AND files.file_size = ? AND hash.hash_type = ?;+ SELECT+ files.folder,+ files.download_filename+ FROM+ hash+ JOIN files ON hash.folder = files.folder+ AND hash.download_filename = files.download_filename+ WHERE+ hash.hash = ?+ AND files.file_size = ?+ AND hash.hash_type = ?;"""- try:- rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_type))- return cast("list[aiosqlite.Row]", rows)-- except Exception:- logger.exception("Error retrieving folder and filename")- return []+ rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_algo))+ return cast("list[aiosqlite.Row]", rows)async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:if self.ignore_history:return False- query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"+ query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1;"cursor = await self.db_conn.execute(query, (hash_type, hash_value))- result = await cursor.fetchone()- return result is not None+ return await cursor.fetchone() is not Noneasync def insert_or_update_hash_db(- self, hash_value: str, hash_type: str, file: Path | str, original_filename: str | None, referer: URL | None- ) -> bool:- """Inserts or updates a record in the specified SQLite database.-- Args:- hash_value: The calculated hash of the file.- file: The file path- original_filename: The name original name of the file.- referer: referer URL- hash_type: The hash type (e.g., md5, sha256)-- Returns:- True if all the record was inserted or updated successfully, False otherwise.- """-- hashed = await self.insert_or_update_hashes(hash_value, hash_type, file)- existed = await self.insert_or_update_file(original_filename, referer, file)- return existed and hashed-- async def insert_or_update_hashes(self, hash_value: str, hash_type: str, file: Path | str) -> bool:+ self,+ hash_value: str,+ hash_algo: str,+ file: Path | str,+ original_filename: str | None,+ referer: URL | None,+ ) -> None:+ await self.insert_or_update_hashes(hash_value, hash_algo, file)+ await self.insert_or_update_file(original_filename, referer, file)++ async def insert_or_update_hashes(self, hash_value: str, hash_type: str, file: Path | str) -> None:query = """- INSERT INTO hash (hash, hash_type, folder, download_filename)- VALUES (?, ?, ?, ?) ON CONFLICT(download_filename, folder, hash_type) DO UPDATE SET hash = ?;+ INSERT INTO hash (+ hash, hash_type, folder, download_filename+ )+ VALUES+ (?, ?, ?, ?) ON CONFLICT(+ download_filename, folder, hash_type+ ) DO+ UPDATE+ SET+ hash = ?;"""- try:- full_path = self.cwd / file- download_filename = full_path.name- folder = str(full_path.parent)- await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))- await self.db_conn.commit()- except Exception:- logger.exception("Error inserting/updating record")- return False- return True+ full_path = self.cwd / file+ download_filename = full_path.name+ folder = str(full_path.parent)+ await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))+ await self.db_conn.commit()async def insert_or_update_file(- self, original_filename: str | None, referer: URL | str | None, file: Path | str- ) -> bool:+ self,+ original_filename: str | None,+ referer: URL | str | None,+ file: Path | str,+ ) -> None:query = """- INSERT INTO files (folder, original_filename, download_filename, file_size, referer, date)- VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(download_filename, folder)- DO UPDATE SET original_filename = ?, file_size = ?, referer = ?, date = ?;+ INSERT INTO files (+ folder, original_filename, download_filename,+ file_size, referer, date+ )+ VALUES+ (?, ?, ?, ?, ?, ?) ON CONFLICT(download_filename, folder) DO+ UPDATE+ SET+ original_filename = ?,+ file_size = ?,+ referer = ?,+ date = ?;"""referer_ = str(referer) if referer else None- try:- full_path = self.cwd / file- download_filename = full_path.name- folder = str(full_path.parent)- stat = full_path.stat()- file_size = stat.st_size- file_date = int(stat.st_mtime)- await self.db_conn.execute(- query,- (- folder,- original_filename,- download_filename,- file_size,- referer_,- file_date,- original_filename,- file_size,- referer_,- file_date,- ),- )- await self.db_conn.commit()- except Exception:- logger.exception("Error inserting/updating record")- return False- return True+ full_path = self.cwd / file+ download_filename = full_path.name+ folder = str(full_path.parent)+ stat = await aio.stat(full_path)+ file_size = stat.st_size+ file_date = int(stat.st_mtime)+ await self.db_conn.execute(+ query,+ (+ folder,+ original_filename,+ download_filename,+ file_size,+ referer_,+ file_date,+ original_filename,+ file_size,+ referer_,+ file_date,+ ),+ )+ await self.db_conn.commit()