Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

39.0 KB
Raw Blame
use anyhow::{Context, Result};
use aws_lambda_events::{event::sqs::SqsEvent, sqs::SqsMessage};
use aws_sdk_s3::{Client as S3Client, primitives::ByteStream};
use connection_gateway_client::client::ConnectionGatewayClient;
use document_storage_service_client::DocumentStorageServiceClient;
use dynamodb_client::DynamodbClient;
use futures::{
    TryFutureExt,
    future::{join_all, try_join_all},
};
use lambda_runtime::{
    Error, LambdaEvent, run, service_fn,
    tracing::{self},
};
use macro_entrypoint::MacroEntrypoint;
use macro_env_var::env_vars;
use macro_service_urls::ConnectionGatewayUrl;
use model::{
    document::{FileType, FileTypeExt},
    folder::{FileSystemNodeWithIds, FolderItem, S3Destination, S3DestinationMap},
};
use model_entity::EntityType;
use models_bulk_upload::{UploadDocumentStatus, UploadFolderStatus, UploadFolderStatusUpdate};
use sha2::{Digest, Sha256};
use sqs_client::upload_extractor::UploadExtractQueueMessage;
use std::ffi::OsStr;
use std::path::Path;
use std::{fs, str::FromStr};
use std::{path::PathBuf, sync::Arc};
use tempfile::tempdir;
use tokio::fs::File;
use tokio_stream::StreamExt;
use tokio_util::io::ReaderStream;
use zip::read::root_dir_common_filter;

env_vars! {
    struct LocalZipPath;
    struct UploadBucketName;
    struct InternalApiSecretKey;
    struct DssUrl;
    struct DynamodbTable;
}

fn get_upload_message_from_sqs_record(record: SqsMessage) -> Result<UploadExtractQueueMessage> {
    let body = record.body.as_deref().unwrap_or_default();
    let message = serde_json::from_str::<UploadExtractQueueMessage>(body)?;
    Ok(message)
}

#[tracing::instrument(skip_all)]
async fn handler(
    s3_client: Arc<S3Client>,
    dss_client: Arc<DocumentStorageServiceClient>,
    dynamodb_client: Arc<DynamodbClient>,
    conn_gateway_client: Arc<ConnectionGatewayClient>,
    upload_bucket_name: &str,
    event: LambdaEvent<SqsEvent>,
) -> Result<(), Error> {
    tracing::debug!("Processing SQS event");

    for record in event.payload.records {
        let upload_request = match get_upload_message_from_sqs_record(record) {
            Ok(req) => req,
            Err(err) => {
                tracing::error!("Failed to get upload request from SQS record: {:?}", err);
                continue;
            }
        };

        tracing::debug!("Processing upload request {:?}", upload_request);

        let request_id = upload_request.upload_request_id();
        let user_id = upload_request.user_id().as_str();
        let zip_name = upload_request.name();
        let parent_id = match &upload_request {
            UploadExtractQueueMessage::ExtractZip(msg) => msg.parent_id.clone(),
        };

        let upload_status = match process_zipped_s3_object(
            &s3_client,
            &dss_client,
            &dynamodb_client,
            upload_request.key(),
            upload_bucket_name,
            request_id,
            user_id,
            zip_name,
            parent_id.as_deref(),
        )
        .await
        {
            Ok((upload_status, root_project_id)) => {
                let pid = root_project_id.as_str();
                let update_dynamodb_fut = dynamodb_client
                    .bulk_upload
                    .update_bulk_upload_request_status(
                        request_id,
                        upload_status.clone(),
                        None,
                        Some(pid),
                    );

                // A partial upload still has a usable tree; leaving it pending would hide
                // every folder because of one bad file.
                let (update_dynamodb_res, update_dss_res) = match &upload_status {
                    UploadFolderStatus::Completed | UploadFolderStatus::PartiallyCompleted => {
                        tokio::join!(
                            update_dynamodb_fut,
                            dss_client
                                .mark_projects_uploaded(user_id, pid)
                                .map_ok(|_| ())
                        )
                    }
                    _ => tokio::join!(update_dynamodb_fut, futures::future::ok(())),
                };

                update_dynamodb_res
                    .inspect_err(|e| tracing::error!("Failed to update dynamo status: {:?}", e))
                    .ok();

                match (update_dss_res, &upload_status) {
                    (Ok(_), UploadFolderStatus::Completed) => UploadFolderStatusUpdate::Completed {
                        request_id: request_id.clone(),
                        project_id: root_project_id.clone(),
                    },
                    (Ok(_), UploadFolderStatus::PartiallyCompleted) => {
                        UploadFolderStatusUpdate::PartiallyCompleted {
                            request_id: request_id.clone(),
                            project_id: root_project_id.clone(),
                        }
                    }
                    (Ok(_), _) => UploadFolderStatusUpdate::Failed {
                        request_id: request_id.clone(),
                    },
                    (Err(e), _) => {
                        tracing::error!("Failed to update dss status: {:?}", e);
                        UploadFolderStatusUpdate::Unknown {
                            request_id: request_id.clone(),
                        }
                    }
                }
            }
            Err(err) => {
                tracing::error!(
                    "Failed to process upload request {:?}: {:?}",
                    upload_request,
                    err
                );
                dynamodb_client
                    .bulk_upload
                    .update_bulk_upload_request_status(
                        request_id,
                        UploadFolderStatus::Failed,
                        Some(&err.to_string()),
                        None,
                    )
                    .await
                    .inspect_err(|e| tracing::error!("Failed to update dynamo status: {:?}", e))
                    .ok();

                UploadFolderStatusUpdate::Failed {
                    request_id: request_id.clone(),
                }
            }
        };

        update_bulk_upload_request_state(conn_gateway_client.as_ref(), user_id, upload_status)
            .await;
    }

    Ok(())
}

#[tracing::instrument(skip_all)]
async fn update_bulk_upload_request_state(
    client: &ConnectionGatewayClient,
    user_id: &str,
    message: UploadFolderStatusUpdate,
) -> () {
    if cfg!(feature = "local") {
        tracing::info!("bypassing connection gateway");
    } else {
        let entities = vec![EntityType::User.with_entity_str(user_id)];

        match serde_json::to_value(message) {
            Ok(message) => {
                client
                    .batch_send_message("bulk_upload".to_string(), message, entities)
                    .await
                    .inspect_err(|e| {
                        tracing::error!(error = ?e, "failed to send message to connection gateway");
                    })
                    .ok();
            }
            Err(e) => {
                tracing::error!(error = ?e, "failed to serialize message");
            }
        }
    }
}

#[tracing::instrument(skip_all, fields(request_id=request_id))]
#[expect(clippy::too_many_arguments, reason = "too annoying to fix")]
async fn process_zipped_s3_object(
    s3_client: &S3Client,
    dss_client: &DocumentStorageServiceClient,
    dynamodb_client: &DynamodbClient,
    object_key: &str,
    staging_bucket: &str,
    request_id: &str,
    user_id: &str,
    zip_name: Option<&str>,
    parent_id: Option<&str>,
) -> Result<(UploadFolderStatus, String), Error> {
    tracing::debug!("Processing object: {}", object_key);

    dynamodb_client
        .bulk_upload
        .update_bulk_upload_request_status(request_id, UploadFolderStatus::Processing, None, None)
        .await?;

    // Create a temporary directory to extract files
    // automatically cleaned up when the process exits
    let temp_dir = tempdir()?;
    let zip_path = temp_dir.path().join("upload.zip");

    tracing::debug!("Downloading zip file to {:?}", zip_path);

    if cfg!(feature = "local") {
        tracing::info!("using local zip file, remove local feature to download s3 object");
        // Update this path to your zip file location
        let file_path = LocalZipPath::new().context("LOCAL_ZIP_PATH must be set")?;
        tracing::info!("Loading local zip file from: {}", file_path.as_ref());
        let file = File::open(file_path.as_ref()).await?;
        let mut body = tokio::io::BufReader::new(file);

        // Write the file to disk
        tokio::io::copy(&mut body, &mut tokio::fs::File::create(&zip_path).await?).await?;
    } else {
        let obj = s3_client
            .get_object()
            .bucket(staging_bucket)
            .key(object_key)
            .send()
            .await?;

        let mut body = obj.body.into_async_read();

        // Write the file to disk
        tokio::io::copy(&mut body, &mut tokio::fs::File::create(&zip_path).await?).await?;
    }

    // Extract zip file
    let extract_dir = temp_dir.path().join("extracted");
    fs::create_dir_all(&extract_dir)?;

    tracing::debug!("Extracting zip file to {:?}", extract_dir);
    let root_folder_name = extract_zip_file(&zip_path, &extract_dir, zip_name)?;

    // Build file system structure
    tracing::debug!(
        "Building file system structure for folder: {}",
        root_folder_name
    );
    let ExtractedTree {
        items: folder_items,
        folders,
    } = build_file_system(&extract_dir).await?;
    tracing::debug!("Upload file system: {:?} {:?}", folder_items, folders);

    let upload_result = dss_client
        .upload_unnested_folder(
            user_id.to_string(),
            root_folder_name,
            folder_items,
            folders,
            request_id.to_string(),
            parent_id.map(|s| s.to_string()),
        )
        .await?;

    let file_system = upload_result.file_system;
    let root_project_id = file_system
        .get_project_id()
        .ok_or_else(|| anyhow::anyhow!("No project found for root folder"))?;

    tracing::debug!("returned file system: {:?}", file_system);
    tracing::debug!("root project id: {:?}", root_project_id);

    let destination_map = upload_result.destination_map;

    let request_status = move_files_to_cloud_storage_bucket(
        s3_client,
        dynamodb_client,
        request_id,
        &extract_dir,
        &file_system,
        &destination_map,
    )
    .await?;
    tracing::debug!("Request status: {:?}", request_status);

    return Ok((request_status, root_project_id.clone()));
}

#[tracing::instrument(skip_all, fields(request_id=request_id))]
async fn move_files_to_cloud_storage_bucket(
    s3_client: &S3Client,
    dynamodb_client: &DynamodbClient,
    request_id: &str,
    extract_dir: &Path,
    file_system: &FileSystemNodeWithIds,
    destination_map: &S3DestinationMap,
) -> Result<UploadFolderStatus, Error> {
    let folder_items = file_system.get_folder_items();

    let document_ids: Vec<String> = folder_items.keys().cloned().collect();
    let document_count = document_ids.len();
    if document_count == 0 {
        tracing::error!("No documents found in file system {}", request_id);
        return Ok(UploadFolderStatus::Completed);
    }

    dynamodb_client
        .bulk_upload
        .bulk_update_file_status(
            request_id,
            document_ids.clone(),
            UploadDocumentStatus::Pending,
        )
        .await?;

    let mut upload_tasks = Vec::new();

    for (document_id, item) in folder_items {
        let destination = match destination_map.get(&document_id) {
            Some(dest) => dest.clone(),
            None => {
                tracing::error!("No destination found for {}", document_id);
                continue;
            }
        };

        let s3_client = s3_client.clone();
        let extract_dir = extract_dir.to_path_buf();

        let fut = async move {
            match destination {
                S3Destination::External(_) => {
                    let error_message = format!(
                        "External destination not implemented for file {}",
                        document_id
                    );
                    (document_id, false, Some(error_message))
                }
                S3Destination::Internal(s3_info) => {
                    let path = get_file_path_for_item(&extract_dir, &item);

                    if !path.is_file() {
                        let error_message = format!("Not a file: {}", path.display());
                        return (document_id, false, Some(error_message));
                    }

                    let byte_stream = match ByteStream::from_path(&path).await {
                        Ok(bs) => bs,
                        Err(e) => {
                            let error_message = format!(
                                "Failed to read file for upload: {:?} {} {} {} {:?}",
                                e,
                                document_id,
                                s3_info.bucket,
                                s3_info.key,
                                path.display()
                            );
                            return (document_id, false, Some(error_message));
                        }
                    };

                    match s3_client
                        .put_object()
                        .bucket(&s3_info.bucket)
                        .key(&s3_info.key)
                        .body(byte_stream)
                        .send()
                        .await
                    {
                        Ok(_) => (document_id, true, None),
                        Err(e) => {
                            let error_message =
                                format!("Failed to upload {}: {:?}", document_id, e);
                            (document_id, false, Some(error_message))
                        }
                    }
                }
            }
        };

        upload_tasks.push(fut);
    }

    let results = join_all(upload_tasks).await;

    let mut success_ids = Vec::new();
    let mut failed_ids = Vec::new();

    for result in results {
        let (document_id, success, error_message) = result;
        match success {
            true => success_ids.push(document_id),
            false => {
                match error_message {
                    Some(error_message) => tracing::error!(error_message),
                    None => tracing::error!("Failed to upload {}", document_id),
                }
                failed_ids.push(document_id)
            }
        };
    }

    let success_count = success_ids.len();
    let failed_count = failed_ids.len();

    match (success_count == 0, failed_count == 0) {
        (true, true) => Ok(UploadFolderStatus::Completed),
        (false, true) => {
            dynamodb_client
                .bulk_upload
                .bulk_update_file_status(request_id, success_ids, UploadDocumentStatus::Completed)
                .await?;
            Ok(UploadFolderStatus::Completed)
        }
        (true, false) => {
            dynamodb_client
                .bulk_upload
                .bulk_update_file_status(request_id, failed_ids, UploadDocumentStatus::Failed)
                .await?;
            Ok(UploadFolderStatus::Failed)
        }
        (false, false) => {
            let (success_res, failed_res) = tokio::join!(
                dynamodb_client.bulk_upload.bulk_update_file_status(
                    request_id,
                    success_ids,
                    UploadDocumentStatus::Completed
                ),
                dynamodb_client.bulk_upload.bulk_update_file_status(
                    request_id,
                    failed_ids,
                    UploadDocumentStatus::Failed
                )
            );

            if let Err(e) = success_res {
                tracing::error!("Failed to update completed statuses: {:?}", e);
            }
            if let Err(e) = failed_res {
                tracing::error!("Failed to update failed statuses: {:?}", e);
            }

            Ok(UploadFolderStatus::PartiallyCompleted)
        }
    }
}

// returns the root folder name
fn extract_zip_file(
    zip_path: &Path,
    extract_dir: &Path,
    zip_name: Option<&str>,
) -> Result<String, Error> {
    let file = fs::File::open(zip_path)?;
    let mut archive = zip::ZipArchive::new(file)?;

    let extract_to: PathBuf;
    let root_dir_name = match archive
        .root_dir(root_dir_common_filter)?
        .and_then(|root| root.file_name().map(|f| f.to_string_lossy().to_string()))
    {
        Some(root_dir_name) => {
            extract_to = extract_dir.join(&root_dir_name);
            root_dir_name
        }
        None => {
            tracing::error!("No root directory found, using provided root folder name");
            let extract_folder = match zip_name {
                Some(zip_name) => zip_name.to_string(),
                None => {
                    let default_extract_folder = "Folder Upload".to_string();
                    tracing::error!(
                        "No zip name provided, using default extract folder {}",
                        default_extract_folder
                    );
                    default_extract_folder
                }
            };
            extract_to = extract_dir.join(&extract_folder);
            extract_folder
        }
    };
    fs::create_dir_all(&extract_to)?;

    tracing::debug!("Extracting to: {:?}", extract_to);
    archive.extract_unwrapped_root_dir(&extract_to, root_dir_common_filter)?;

    Ok(root_dir_name)
}

/// Files and folders collected from an extracted upload.
#[derive(Debug, Default)]
struct ExtractedTree {
    items: Vec<FolderItem>,
    /// Every kept folder, as a path relative to the extract dir, so empty
    /// folders still become projects.
    folders: Vec<String>,
}

/// OS and editor clutter that should never become a document or project.
fn is_ignored_name(name: &OsStr) -> bool {
    let name = name.to_string_lossy();
    let lower = name.to_ascii_lowercase();
    // Hidden files/folders, including `.DS_Store`, `._*` and `.~lock.*#`.
    name.starts_with('.')
        || name == "__MACOSX"
        // Office owner/lock files written next to open documents.
        || name.starts_with("~$")
        || lower == "thumbs.db"
        || lower == "ehthumbs.db"
        || lower == "desktop.ini"
        // macOS custom folder icon file.
        || name == "Icon\r"
}

/// Whether a path under the extract dir should be skipped. The first component
/// is the root folder the user picked, which is kept whatever its name.
fn is_ignored_entry(relative_path: &Path) -> bool {
    relative_path.iter().skip(1).any(is_ignored_name)
}

async fn build_file_system(extract_dir: &Path) -> Result<ExtractedTree, Error> {
    let mut tasks = Vec::new();
    let mut folders = Vec::new();

    let walker = walkdir::WalkDir::new(extract_dir)
        .min_depth(1)
        .sort_by_file_name()
        .into_iter()
        // Pruning here also skips everything inside an ignored folder.
        .filter_entry(|entry| {
            entry
                .path()
                .strip_prefix(extract_dir)
                .map(|relative| !is_ignored_entry(relative))
                .unwrap_or(false)
        });

    for entry in walker.filter_map(|e| e.ok()) {
        let path = entry.path().to_path_buf();
        let relative = path
            .strip_prefix(extract_dir)
            .map_err(|e| Error::from(format!("strip_prefix failed: {:?}", e)))?
            .to_path_buf();

        if entry.file_type().is_dir() {
            let folder = relative
                .to_str()
                .ok_or_else(|| Error::from(format!("Invalid folder name: {:?}", relative)))?;
            folders.push(folder.to_string());
            continue;
        }

        if !entry.file_type().is_file() {
            continue;
        }

        let fut = async move {
            let full_name = path
                .file_name()
                .and_then(|s| s.to_str())
                .ok_or_else(|| Error::from(format!("Invalid file name: {:?}", path)))?;

            let (file_name, file_type) = match FileType::split_suffix_match(full_name) {
                Some((file, extension)) => {
                    let file_type = FileType::from_str(extension).ok();
                    (file, file_type)
                }
                None => (full_name, None),
            };

            let relative_path_without_filename = relative
                .parent()
                .ok_or_else(|| Error::from(format!("No parent for path: {:?}", path)))?
                .to_string_lossy()
                .to_string();

            let sha = compute_sha256(&path).await?;

            Ok::<_, Error>(FolderItem {
                name: file_name.to_string(),
                full_name: full_name.to_string(),
                file_type,
                relative_path: relative_path_without_filename,
                sha,
            })
        };

        tasks.push(fut);
    }

    // exits immediately if any task fails
    let items = try_join_all(tasks).await?;
    Ok(ExtractedTree { items, folders })
}

async fn compute_sha256(path: &Path) -> Result<String, Error> {
    let file = File::open(path).await?;
    let mut stream = ReaderStream::new(file);
    let mut hasher = Sha256::new();

    while let Some(chunk) = stream.next().await {
        let bytes = chunk?;
        hasher.update(&bytes);
    }

    let result = hasher.finalize();
    Ok(format!("{:x}", result)) // hex string
}

/// `full_name` is the name on disk; rebuilding it from the parsed type would
/// lowercase extensions like `.DOCX` and miss the file.
fn get_file_path_for_item(extract_dir: &Path, item: &FolderItem) -> PathBuf {
    extract_dir.join(&item.relative_path).join(&item.full_name)
}

#[tokio::main]
async fn main() -> Result<(), Error> {
    MacroEntrypoint::default().init();
    tracing::trace!("initiating lambda");

    let upload_staging_bucket = UploadBucketName::new()
        .context("UPLOAD_BUCKET_NAME must be set")?
        .to_string();
    let internal_api_secret_key = InternalApiSecretKey::new()
        .context("INTERNAL_API_SECRET_KEY must be set")?
        .to_string();
    let dss_url = DssUrl::new().context("DSS_URL must be set")?.to_string();
    let dynamo_table_name = DynamodbTable::new()
        .context("DYNAMODB_TABLE must be set")?
        .to_string();
    let connection_gateway_url = ConnectionGatewayUrl::new()?.to_string();

    let config = macro_aws_config::get_macro_aws_config().await;
    let s3_client = S3Client::new(&config);

    let dss_client = DocumentStorageServiceClient::new(internal_api_secret_key.clone(), dss_url);

    let dynamodb_client = DynamodbClient::new(&config, Some(dynamo_table_name));

    let conn_gateway_client =
        ConnectionGatewayClient::new(internal_api_secret_key, connection_gateway_url);

    let shared_s3_client = Arc::new(s3_client);
    let shared_dss_client = Arc::new(dss_client);
    let shared_dynamodb_client = Arc::new(dynamodb_client);
    let shared_conn_gateway_client = Arc::new(conn_gateway_client);
    let upload_staging_bucket_str = upload_staging_bucket.as_str();

    let func = service_fn(move |event: LambdaEvent<SqsEvent>| {
        handler(
            shared_s3_client.clone(),
            shared_dss_client.clone(),
            shared_dynamodb_client.clone(),
            shared_conn_gateway_client.clone(),
            upload_staging_bucket_str,
            event,
        )
    });

    run(func).await
}

#[cfg(test)]
mod tests {
    use super::*;
    use model::document::FileType;
    use model::folder::FolderItem;
    use std::fs::{self, File};
    use std::io::Write;
    use std::path::PathBuf;
    use tempfile::tempdir;

    #[tokio::test]
    async fn test_build_file_system() {
        // Create a temporary directory for testing
        let temp_dir = tempdir().unwrap();
        let extract_dir = temp_dir.path();

        // Create a test file structure
        let root_dir = extract_dir.join("root");
        fs::create_dir_all(&root_dir).unwrap();

        // Create a file in the root directory
        let root_file_path = root_dir.join("root_file.txt");
        let mut root_file = File::create(&root_file_path).unwrap();
        root_file.write_all(b"root file content").unwrap();

        // Create a subdirectory
        let subdir_path = root_dir.join("subdir");
        fs::create_dir_all(&subdir_path).unwrap();

        // Create a file in the subdirectory
        let subdir_file_path = subdir_path.join("subdir_file.pdf");
        let mut subdir_file = File::create(&subdir_file_path).unwrap();
        subdir_file.write_all(b"subdir file content").unwrap();

        // Create a nested subdirectory
        let nested_subdir_path = subdir_path.join("nested");
        fs::create_dir_all(&nested_subdir_path).unwrap();

        // Create a file in the nested subdirectory
        let nested_file_path = nested_subdir_path.join("nested_file.docx");
        let mut nested_file = File::create(&nested_file_path).unwrap();
        nested_file.write_all(b"nested file content").unwrap();

        // Run the build_file_system function
        let result = build_file_system(extract_dir).await.unwrap().items;

        // Verify the results
        assert_eq!(result.len(), 3); // Should have 3 files total

        // Check that each file has the correct relative path and name
        let root_file_item = result.iter().find(|item| item.name == "root_file").unwrap();
        assert_eq!(root_file_item.relative_path, "root");
        assert_eq!(root_file_item.file_type, Some(FileType::Txt));

        let subdir_file_item = result
            .iter()
            .find(|item| item.name == "subdir_file")
            .unwrap();
        assert_eq!(subdir_file_item.relative_path, "root/subdir");
        assert_eq!(subdir_file_item.file_type, Some(FileType::Pdf));

        let nested_file_item = result
            .iter()
            .find(|item| item.name == "nested_file")
            .unwrap();
        assert_eq!(nested_file_item.relative_path, "root/subdir/nested");
        assert_eq!(nested_file_item.file_type, Some(FileType::Docx));
    }

    #[test]
    fn test_get_file_path_for_item_with_file_type() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "document".to_string(),
            full_name: "document.pdf".to_string(),
            file_type: Some(FileType::Pdf),
            relative_path: "folder/subfolder".to_string(),
            sha: "abcd1234".to_string(),
        };

        let path = get_file_path_for_item(&extract_dir, &item);
        assert_eq!(
            path,
            PathBuf::from("/tmp/extract/folder/subfolder/document.pdf")
        );
    }

    #[test]
    fn test_get_file_path_for_item_with_multi_part_extension() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "document".to_string(),
            full_name: "document.tar.gz".to_string(),
            file_type: Some(FileType::TarGz),
            relative_path: "folder/subfolder".to_string(),
            sha: "abcd1234".to_string(),
        };

        let path = get_file_path_for_item(&extract_dir, &item);
        assert_eq!(
            path,
            PathBuf::from("/tmp/extract/folder/subfolder/document.tar.gz")
        );
    }

    #[test]
    fn test_get_file_path_for_item_without_file_type() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "config".to_string(),
            full_name: "config".to_string(),
            file_type: None,
            relative_path: "settings".to_string(),
            sha: "efgh5678".to_string(),
        };

        let path = get_file_path_for_item(&extract_dir, &item);
        assert_eq!(path, PathBuf::from("/tmp/extract/settings/config"));
    }

    #[test]
    fn test_get_file_path_for_item_with_periods_in_name() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "test.1.5.5".to_string(),
            full_name: "test.1.5.5.tar.gz".to_string(),
            file_type: Some(FileType::TarGz),
            relative_path: "settings".to_string(),
            sha: "efgh5678".to_string(),
        };

        let path = get_file_path_for_item(&extract_dir, &item);
        assert_eq!(
            path,
            PathBuf::from("/tmp/extract/settings/test.1.5.5.tar.gz")
        );
    }

    #[test]
    fn test_get_file_path_for_item_with_empty_relative_path() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "root_file".to_string(),
            full_name: "root_file.txt".to_string(),
            file_type: Some(FileType::Txt),
            relative_path: "".to_string(),
            sha: "ijkl9012".to_string(),
        };

        let path = get_file_path_for_item(&extract_dir, &item);
        assert_eq!(path, PathBuf::from("/tmp/extract/root_file.txt"));
    }

    #[tokio::test]
    async fn test_build_file_system_handles_empty_directory() {
        // Create an empty temporary directory
        let temp_dir = tempdir().unwrap();
        let extract_dir = temp_dir.path();

        // Run the build_file_system function
        let result = build_file_system(extract_dir).await.unwrap().items;

        // Verify the results
        assert_eq!(result.len(), 0); // Should be empty
        let tree = build_file_system(extract_dir).await.unwrap();
        assert!(tree.folders.is_empty());
    }

    #[tokio::test]
    async fn test_build_file_system_skips_hidden_files() {
        // Create a temporary directory
        let temp_dir = tempdir().unwrap();
        let extract_dir = temp_dir.path();

        let root_dir = extract_dir.join("root");
        fs::create_dir_all(&root_dir).unwrap();

        // Create a visible file
        let visible_file_path = root_dir.join("visible.txt");
        let mut visible_file = File::create(&visible_file_path).unwrap();
        visible_file.write_all(b"visible content").unwrap();

        // Create a hidden file
        let hidden_file_path = root_dir.join(".hidden.txt");
        let mut hidden_file = File::create(&hidden_file_path).unwrap();
        hidden_file.write_all(b"hidden content").unwrap();

        // Run the build_file_system function
        let result = build_file_system(extract_dir).await.unwrap().items;

        // Verify that only the visible file is included
        assert_eq!(result.len(), 1);
        assert_eq!(result[0].name, "visible");
        assert_eq!(result[0].file_type, Some(FileType::Txt));
    }

    #[tokio::test]
    async fn test_build_file_system_handles_file_without_extension() {
        // Create a temporary directory
        let temp_dir = tempdir().unwrap();
        let extract_dir = temp_dir.path();

        // Create a file without extension
        let file_path = extract_dir.join("no_extension");
        let mut file = File::create(&file_path).unwrap();
        file.write_all(b"content without extension").unwrap();

        // Run the build_file_system function
        let result = build_file_system(extract_dir).await.unwrap().items;

        // Verify the results
        assert_eq!(result.len(), 1);
        assert_eq!(result[0].name, "no_extension");
        assert_eq!(result[0].file_type, None);
    }

    #[tokio::test]
    async fn test_build_file_system_computes_correct_sha() {
        // Create a temporary directory
        let temp_dir = tempdir().unwrap();
        let extract_dir = temp_dir.path();

        // Create a file with known content for SHA verification
        let file_path = extract_dir.join("test.txt");
        let mut file = File::create(&file_path).unwrap();
        file.write_all(b"test content for SHA").unwrap();

        // Manually compute the SHA256 hash for comparison
        use sha2::{Digest, Sha256};
        let mut hasher = Sha256::new();
        hasher.update(b"test content for SHA");
        let expected_sha = format!("{:x}", hasher.finalize());

        // Run the build_file_system function
        let result = build_file_system(extract_dir).await.unwrap().items;

        // Verify the SHA256 hash
        assert_eq!(result.len(), 1);
        assert_eq!(result[0].sha, expected_sha);
    }

    #[test]
    fn test_get_file_path_for_item_keeps_extension_case() {
        let extract_dir = PathBuf::from("/tmp/extract");
        let item = FolderItem {
            name: "REPORT".to_string(),
            full_name: "REPORT.DOCX".to_string(),
            file_type: Some(FileType::Docx),
            relative_path: "Root".to_string(),
            sha: "abcd".to_string(),
        };

        assert_eq!(
            get_file_path_for_item(&extract_dir, &item),
            PathBuf::from("/tmp/extract/Root/REPORT.DOCX")
        );
    }

    #[test]
    fn test_is_ignored_entry() {
        for ignored in [
            "Root/.DS_Store",
            "Root/~$Summary.docx",
            "Root/Sub/~$Deck.pptx",
            "Root/Thumbs.db",
            "Root/THUMBS.DB",
            "Root/desktop.ini",
            "Root/Desktop.ini",
            "Root/ehthumbs.db",
            "Root/Icon\r",
            "Root/.git/config",
            "Root/__MACOSX/._a.docx",
            "Root/Sub/.~lock.Budget.xlsx#",
        ] {
            assert!(is_ignored_entry(Path::new(ignored)), "{ignored}");
        }
        for kept in [
            "Root",
            ".dotted-root",
            ".dotted-root/a.docx",
            "Root/a.docx",
            "Root/Sub/b.pptx",
            "Root/Thumbs.db.xlsx",
            "Root/report~$.docx",
            "Root/Icon.png",
        ] {
            assert!(!is_ignored_entry(Path::new(kept)), "{kept}");
        }
    }

    /// Zip entries as the web app's zip worker writes them for a dropped folder.
    fn write_browser_zip(path: &Path, entries: &[(&str, Option<&[u8]>)]) {
        let file = File::create(path).unwrap();
        let mut zip = zip::ZipWriter::new(file);
        let options = zip::write::SimpleFileOptions::default();
        for (name, contents) in entries {
            match contents {
                Some(bytes) => {
                    zip.start_file(*name, options).unwrap();
                    zip.write_all(bytes).unwrap();
                }
                None => zip.add_directory(*name, options).unwrap(),
            }
        }
        zip.finish().unwrap();
    }

    async fn extract_tree(
        entries: &[(&str, Option<&[u8]>)],
        zip_name: &str,
    ) -> (String, ExtractedTree, PathBuf, tempfile::TempDir) {
        let temp_dir = tempdir().unwrap();
        let zip_path = temp_dir.path().join("upload.zip");
        write_browser_zip(&zip_path, entries);
        let extract_dir = temp_dir.path().join("extracted");
        fs::create_dir_all(&extract_dir).unwrap();
        let root = extract_zip_file(&zip_path, &extract_dir, Some(zip_name)).unwrap();
        let tree = build_file_system(&extract_dir).await.unwrap();
        (root, tree, extract_dir, temp_dir)
    }

    #[tokio::test]
    async fn test_nested_office_folder_upload() {
        let entries: &[(&str, Option<&[u8]>)] = &[
            ("Board Pack/", None),
            ("Board Pack/Summary.docx", Some(b"docx")),
            ("Board Pack/REPORT UPPER.DOCX", Some(b"upper")),
            ("Board Pack/Deck.PPTX", Some(b"upper pptx")),
            ("Board Pack/Café Menü.docx", Some(b"unicode")),
            ("Board Pack/~$Summary.docx", Some(b"lock")),
            ("Board Pack/.DS_Store", Some(b"ds")),
            ("Board Pack/Thumbs.db", Some(b"thumbs")),
            ("Board Pack/desktop.ini", Some(b"ini")),
            ("Board Pack/.git/", None),
            ("Board Pack/.git/config", Some(b"git")),
            ("Board Pack/Empty Folder/", None),
            ("Board Pack/Finance/", None),
            ("Board Pack/Finance/Budget.xlsx", Some(b"xlsx")),
            ("Board Pack/Finance/Forecasts/", None),
            ("Board Pack/Finance/Forecasts/Scenarios/", None),
            (
                "Board Pack/Finance/Forecasts/Scenarios/Downside.xlsx",
                Some(b"xlsx2"),
            ),
            ("Board Pack/Finance/Forecasts/Scenarios/Empty Leaf/", None),
            ("Board Pack/Decks/", None),
            ("Board Pack/Decks/Archive/", None),
            ("Board Pack/Decks/Archive/Old.pptx", Some(b"pptx")),
            ("Board Pack/Decks/Archive/~$Old.pptx", Some(b"lock")),
        ];
        let (root, tree, extract_dir, _guard) = extract_tree(entries, "Board Pack").await;
        assert_eq!(root, "Board Pack");

        let mut files: Vec<String> = tree
            .items
            .iter()
            .map(|item| format!("{}/{}", item.relative_path, item.full_name))
            .collect();
        files.sort();
        assert_eq!(
            files,
            vec![
                "Board Pack/Café Menü.docx",
                "Board Pack/Deck.PPTX",
                "Board Pack/Decks/Archive/Old.pptx",
                "Board Pack/Finance/Budget.xlsx",
                "Board Pack/Finance/Forecasts/Scenarios/Downside.xlsx",
                "Board Pack/REPORT UPPER.DOCX",
                "Board Pack/Summary.docx",
            ]
        );

        // Every kept file must be found again when it is copied to storage.
        for item in &tree.items {
            assert!(
                get_file_path_for_item(&extract_dir, item).is_file(),
                "{item:?}"
            );
        }
        let upper = tree
            .items
            .iter()
            .find(|item| item.full_name == "REPORT UPPER.DOCX")
            .unwrap();
        assert_eq!(upper.file_type, Some(FileType::Docx));
        assert_eq!(upper.name, "REPORT UPPER");
        let deck = tree
            .items
            .iter()
            .find(|item| item.full_name == "Deck.PPTX")
            .unwrap();
        assert_eq!(deck.file_type, Some(FileType::Pptx));

        assert_eq!(
            tree.folders,
            vec![
                "Board Pack",
                "Board Pack/Decks",
                "Board Pack/Decks/Archive",
                "Board Pack/Empty Folder",
                "Board Pack/Finance",
                "Board Pack/Finance/Forecasts",
                "Board Pack/Finance/Forecasts/Scenarios",
                "Board Pack/Finance/Forecasts/Scenarios/Empty Leaf",
            ]
        );
    }

    #[tokio::test]
    async fn test_folder_with_only_empty_subfolders() {
        let entries: &[(&str, Option<&[u8]>)] =
            &[("Shell/", None), ("Shell/A/", None), ("Shell/A/B/", None)];
        let (root, tree, _extract_dir, _guard) = extract_tree(entries, "Shell").await;
        assert_eq!(root, "Shell");
        assert!(tree.items.is_empty());
        assert_eq!(tree.folders, vec!["Shell", "Shell/A", "Shell/A/B"]);
    }

    #[tokio::test]
    async fn test_single_subfolder_keeps_picked_root() {
        // The picked folder holds one subfolder and no files of its own.
        let entries: &[(&str, Option<&[u8]>)] = &[
            ("Wrapper/", None),
            ("Wrapper/Inner/", None),
            ("Wrapper/Inner/a.docx", Some(b"a")),
        ];
        let (root, tree, _extract_dir, _guard) = extract_tree(entries, "Wrapper").await;
        assert_eq!(root, "Wrapper");
        assert_eq!(tree.items.len(), 1);
        assert_eq!(tree.items[0].relative_path, "Wrapper/Inner");
        assert_eq!(tree.folders, vec!["Wrapper", "Wrapper/Inner"]);
    }
}