Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove exc checks (hash)

NTFSvolumecfe0b48edd6e
diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.py
index 8e8fb5d0b..95d2d6306 100644
--- a/cyberdrop_dl/database/hash.py
+++ b/cyberdrop_dl/database/hash.py
@@ -5,6 +5,8 @@ import logging
from pathlib import Path
from typing import TYPE_CHECKING, cast
+from cyberdrop_dl import aio
+
from .common import Table
from .definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEX
@@ -31,133 +33,133 @@ class HashTable(Table, name="hash"):
await self.db_conn.commit()
async def get_file_hash_exists(self, path: Path | str, hash_type: str) -> str | None:
- query = "SELECT hash FROM hash WHERE folder=? AND download_filename=? AND hash_type=? AND hash IS NOT NULL"
- try:
- path = self.cwd / path
- folder = str(path.parent)
- filename = path.name
-
- # Check if the file exists with matching folder, filename, and size
- cursor = await self.db_conn.execute(query, (folder, filename, hash_type))
- if row := await cursor.fetchone():
- return row[0]
-
- except Exception:
- logger.exception("Error checking file")
+ query = "SELECT hash FROM hash WHERE folder= ? AND download_filename= ? AND hash_type= ? AND hash IS NOT NULL LIMIT 1;"
+ path = self.cwd / path
+ folder = str(path.parent)
+ filename = path.name
+ cursor = await self.db_conn.execute(query, (folder, filename, hash_type))
+ if row := await cursor.fetchone():
+ return row["hash"]
async def get_files_with_hash_matches(
- self, hash_value: str, size: int, hash_type: str | None = None
+ self,
+ hash_value: str,
+ size: int,
+ hash_algo: str | None = None,
) -> list[aiosqlite.Row]:
- """Retrieves a list of (folder, filename) tuples based on a given hash.
-
- Args:
- hash_value: The hash value to search for.
- size: file size
-
- Returns:
- A list of (folder, filename) tuples, or an empty list if no matches found.
- """
- if hash_type:
+ if hash_algo:
query = """
- SELECT files.folder, files.download_filename,files.date
- FROM hash JOIN files ON hash.folder = files.folder AND hash.download_filename = files.download_filename
- WHERE hash.hash = ? AND files.file_size = ? AND hash.hash_type = ?;
+ SELECT
+ files.folder,
+ files.download_filename,
+ files.date
+ FROM
+ hash
+ JOIN files ON hash.folder = files.folder
+ AND hash.download_filename = files.download_filename
+ WHERE
+ hash.hash = ?
+ AND files.file_size = ?
+ AND hash.hash_type = ?;
"""
else:
query = """
- SELECT files.folder, files.download_filename FROM hash JOIN files
- ON hash.folder = files.folder AND hash.download_filename = files.download_filename
- WHERE hash.hash = ? AND files.file_size = ? AND hash.hash_type = ?;
+ SELECT
+ files.folder,
+ files.download_filename
+ FROM
+ hash
+ JOIN files ON hash.folder = files.folder
+ AND hash.download_filename = files.download_filename
+ WHERE
+ hash.hash = ?
+ AND files.file_size = ?
+ AND hash.hash_type = ?;
"""
- try:
- rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_type))
- return cast("list[aiosqlite.Row]", rows)
-
- except Exception:
- logger.exception("Error retrieving folder and filename")
- return []
+ rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_algo))
+ return cast("list[aiosqlite.Row]", rows)
async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:
if self.ignore_history:
return False
- query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"
+ query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1;"
cursor = await self.db_conn.execute(query, (hash_type, hash_value))
- result = await cursor.fetchone()
- return result is not None
+ return await cursor.fetchone() is not None
async def insert_or_update_hash_db(
- self, hash_value: str, hash_type: str, file: Path | str, original_filename: str | None, referer: URL | None
- ) -> bool:
- """Inserts or updates a record in the specified SQLite database.
-
- Args:
- hash_value: The calculated hash of the file.
- file: The file path
- original_filename: The name original name of the file.
- referer: referer URL
- hash_type: The hash type (e.g., md5, sha256)
-
- Returns:
- True if all the record was inserted or updated successfully, False otherwise.
- """
-
- hashed = await self.insert_or_update_hashes(hash_value, hash_type, file)
- existed = await self.insert_or_update_file(original_filename, referer, file)
- return existed and hashed
-
- async def insert_or_update_hashes(self, hash_value: str, hash_type: str, file: Path | str) -> bool:
+ self,
+ hash_value: str,
+ hash_algo: str,
+ file: Path | str,
+ original_filename: str | None,
+ referer: URL | None,
+ ) -> None:
+ await self.insert_or_update_hashes(hash_value, hash_algo, file)
+ await self.insert_or_update_file(original_filename, referer, file)
+
+ async def insert_or_update_hashes(self, hash_value: str, hash_type: str, file: Path | str) -> None:
query = """
- INSERT INTO hash (hash, hash_type, folder, download_filename)
- VALUES (?, ?, ?, ?) ON CONFLICT(download_filename, folder, hash_type) DO UPDATE SET hash = ?;
+ INSERT INTO hash (
+ hash, hash_type, folder, download_filename
+ )
+ VALUES
+ (?, ?, ?, ?) ON CONFLICT(
+ download_filename, folder, hash_type
+ ) DO
+ UPDATE
+ SET
+ hash = ?;
"""
- try:
- full_path = self.cwd / file
- download_filename = full_path.name
- folder = str(full_path.parent)
- await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))
- await self.db_conn.commit()
- except Exception:
- logger.exception("Error inserting/updating record")
- return False
- return True
+ full_path = self.cwd / file
+ download_filename = full_path.name
+ folder = str(full_path.parent)
+ await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))
+ await self.db_conn.commit()
async def insert_or_update_file(
- self, original_filename: str | None, referer: URL | str | None, file: Path | str
- ) -> bool:
+ self,
+ original_filename: str | None,
+ referer: URL | str | None,
+ file: Path | str,
+ ) -> None:
query = """
- INSERT INTO files (folder, original_filename, download_filename, file_size, referer, date)
- VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(download_filename, folder)
- DO UPDATE SET original_filename = ?, file_size = ?, referer = ?, date = ?;
+ INSERT INTO files (
+ folder, original_filename, download_filename,
+ file_size, referer, date
+ )
+ VALUES
+ (?, ?, ?, ?, ?, ?) ON CONFLICT(download_filename, folder) DO
+ UPDATE
+ SET
+ original_filename = ?,
+ file_size = ?,
+ referer = ?,
+ date = ?;
"""
referer_ = str(referer) if referer else None
- try:
- full_path = self.cwd / file
- download_filename = full_path.name
- folder = str(full_path.parent)
- stat = full_path.stat()
- file_size = stat.st_size
- file_date = int(stat.st_mtime)
- await self.db_conn.execute(
- query,
- (
- folder,
- original_filename,
- download_filename,
- file_size,
- referer_,
- file_date,
- original_filename,
- file_size,
- referer_,
- file_date,
- ),
- )
- await self.db_conn.commit()
- except Exception:
- logger.exception("Error inserting/updating record")
- return False
- return True
+ full_path = self.cwd / file
+ download_filename = full_path.name
+ folder = str(full_path.parent)
+ stat = await aio.stat(full_path)
+ file_size = stat.st_size
+ file_date = int(stat.st_mtime)
+ await self.db_conn.execute(
+ query,
+ (
+ folder,
+ original_filename,
+ download_filename,
+ file_size,
+ referer_,
+ file_date,
+ original_filename,
+ file_size,
+ referer_,
+ file_date,
+ ),
+ )
+ await self.db_conn.commit()