#!/bin/bash

###############################################################################
# Glacier Archive Split Script
#
# Archives source folders into ~100GB tar.gz chunks across two transit disks.
# Prepares data for later upload to AWS S3 Glacier Deep Archive.
#
# Features:
#   - Disks may have different capacities (detected automatically)
#   - Large folders (e.g. GoPro) can be split into subfolders
#   - All operations strictly sequential: one archive at a time
#   - Full file/folder map written to master manifest per disk
#   - Resume support via completed.txt state tracking
#
# Usage:
#   ./glacier_archive_split.sh [--dry-run] [--yes] [--cleanup]
#
# Options:
#   --dry-run   Show the archiving plan and generate preview manifest
#   --yes       Skip confirmation prompt (for background/nohup execution)
#   --cleanup   Remove all archives, manifests, and state from transit disks
###############################################################################

set -euo pipefail

# Configuration - EDIT THESE VALUES
SOURCE_DIR="/mnt/source_hdd/PhotosVideos"
TRANSIT_DISK_1="/mnt/sdd_transit"
TRANSIT_DISK_2="/mnt/sdc_transit"
MAX_ARCHIVE_SIZE=$((100 * 1024 * 1024 * 1024))  # 100GB in bytes

# Folders to split into subfolders with custom archive naming.
# Format: "FolderName:ArchivePrefix"
# GoPro subfolders will be batched into GoPro_1.tar.gz, GoPro_2.tar.gz, etc.
SPLIT_FOLDERS=("GoPro:GoPro")

# State directory name (created on each transit disk)
STATE_DIR_NAME=".archive_state"
MASTER_MANIFEST_NAME="archive_master_manifest.txt"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

# Flags
DRY_RUN=false
AUTO_YES=false
CLEANUP=false
for arg in "$@"; do
    case "$arg" in
        --dry-run) DRY_RUN=true ;;
        --yes)     AUTO_YES=true ;;
        --cleanup) CLEANUP=true ;;
        *) echo "ERROR: Unknown option: $arg" >&2; exit 1 ;;
    esac
done

###############################################################################
# Utility Functions
###############################################################################

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
}

log_color() {
    echo -e "${2}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}"
}

error_exit() {
    log_color "ERROR: $1" "$RED" >&2
    exit 1
}

human_size() {
    local bytes=$1
    if (( bytes >= 1099511627776 )); then
        echo "$(echo "scale=1; $bytes / 1099511627776" | bc)T"
    elif (( bytes >= 1073741824 )); then
        echo "$(echo "scale=1; $bytes / 1073741824" | bc)G"
    elif (( bytes >= 1048576 )); then
        echo "$(echo "scale=1; $bytes / 1048576" | bc)M"
    else
        echo "${bytes}B"
    fi
}

get_disk_free_bytes() {
    local path="$1"
    df -B1 --output=avail "$path" 2>/dev/null | tail -1 | tr -d ' '
}

# Check if a folder name is in SPLIT_FOLDERS, return the prefix
get_split_prefix() {
    local folder_name="$1"
    for entry in "${SPLIT_FOLDERS[@]}"; do
        local name="${entry%%:*}"
        local prefix="${entry##*:}"
        if [[ "$folder_name" == "$name" ]]; then
            echo "$prefix"
            return 0
        fi
    done
    return 1
}

###############################################################################
# Phase 1: Scan & Plan
###############################################################################

scan_folders() {
    # Output format: group\tfolder_path\tsize_bytes
    # group "main" = regular folders, archived as archive_001, archive_002, ...
    # group "GoPro" (etc) = split folder subfolders, archived as GoPro_1, GoPro_2, ...
    # NOTE: stdout is piped, so all log/status messages go to stderr
    log_color "Scanning folders..." "$BLUE" >&2

    if [ ! -d "$SOURCE_DIR" ]; then
        error_exit "Source directory not found: $SOURCE_DIR"
    fi

    local scan_file
    scan_file=$(mktemp)
    local count=0

    while IFS= read -r -d '' folder; do
        local name
        name=$(basename "$folder")
        [[ "$name" == .* ]] && continue

        local prefix
        if prefix=$(get_split_prefix "$name"); then
            # Split this folder: scan its subfolders
            log_color "Splitting folder '$name' into subfolders (prefix: ${prefix})" "$CYAN" >&2
            local subcount=0
            while IFS= read -r -d '' subfolder; do
                local subname
                subname=$(basename "$subfolder")
                [[ "$subname" == .* ]] && continue
                local subsize
                subsize=$(du -sb "$subfolder" 2>/dev/null | cut -f1)
                echo -e "${prefix}\t${name}/${subname}\t${subsize}" >> "$scan_file"
                subcount=$((subcount + 1))
            done < <(find "$SOURCE_DIR/$name" -mindepth 1 -maxdepth 1 -type d -print0)
            log_color "  Found $subcount subfolders in '$name'" "$CYAN" >&2
            count=$((count + subcount))
        else
            local size
            size=$(du -sb "$folder" 2>/dev/null | cut -f1)
            echo -e "main\t${name}\t${size}" >> "$scan_file"
            count=$((count + 1))
        fi
    done < <(find "$SOURCE_DIR" -mindepth 1 -maxdepth 1 -type d -print0)

    if [ "$count" -eq 0 ]; then
        rm -f "$scan_file"
        error_exit "No folders found in $SOURCE_DIR"
    fi

    log "Found $count items to archive" >&2

    # Sort by group, then natural name order (so archive_001 < archive_002)
    sort -t$'\t' -k1,1 -k2,2V "$scan_file"
    rm -f "$scan_file"
}

build_batches() {
    # Input:  group\tfolder_path\tsize_bytes (sorted by group, then name naturally)
    # Output: group\tbatch_num_within_group\tfolder_path\tsize_bytes
    # Batches are built independently per group.
    # Sequential filling: folders are added in natural name order to the current
    # batch until it would exceed MAX_ARCHIVE_SIZE, then a new batch starts.
    # This keeps naturally-sorted folders together in consecutive archives.

    local current_group=""
    local current_batch=0
    local current_batch_size=0

    while IFS=$'\t' read -r group folder_path size_bytes; do
        [ -z "$folder_path" ] && continue

        # New group — reset batch counter
        if [[ "$group" != "$current_group" ]]; then
            current_group="$group"
            current_batch=0
            current_batch_size=0
        fi

        # Start new batch if adding this folder would exceed limit
        # (unless current batch is empty — always add at least one folder)
        if (( current_batch_size > 0 && current_batch_size + size_bytes > MAX_ARCHIVE_SIZE )); then
            current_batch=$((current_batch + 1))
            current_batch_size=0
        fi

        current_batch_size=$((current_batch_size + size_bytes))
        echo -e "${group}\t${current_batch}\t${folder_path}\t${size_bytes}"
    done
}

assign_disks() {
    # Input:  group\tbatch_num\tfolder_path\tsize_bytes
    # Output: disk_path\tgroup\tbatch_num\tfolder_path\tsize_bytes

    local disk1_cap=$1
    local disk2_cap=$2

    local -a disk_remaining=( "$disk1_cap" "$disk2_cap" )
    local -a disk_paths=( "$TRANSIT_DISK_1" "$TRANSIT_DISK_2" )

    # Collect all lines
    local lines=()
    while IFS=$'\t' read -r group batch_num folder_path size_bytes; do
        lines+=( "${group}	${batch_num}	${folder_path}	${size_bytes}" )
    done

    # Compute batch totals per (group, batch_num) — use composite key
    local -A batch_totals=()
    for line in "${lines[@]}"; do
        IFS=$'\t' read -r group batch_num folder_path size_bytes <<< "$line"
        local key="${group}:${batch_num}"
        batch_totals[$key]=$(( ${batch_totals[$key]:-0} + size_bytes ))
    done

    # Assign each (group, batch) to disk with most remaining space
    local -A batch_disk_map=()
    local sorted_keys
    sorted_keys=$(for k in "${!batch_totals[@]}"; do
        echo -e "${k}\t${batch_totals[$k]}"
    done | sort -t$'\t' -k2 -n -r | cut -f1)

    while IFS= read -r key; do
        [ -z "$key" ] && continue
        local batch_size=${batch_totals[$key]}
        if (( disk_remaining[0] >= disk_remaining[1] )); then
            if (( batch_size <= disk_remaining[0] )); then
                batch_disk_map[$key]="${disk_paths[0]}"
                disk_remaining[0]=$(( disk_remaining[0] - batch_size ))
            elif (( batch_size <= disk_remaining[1] )); then
                batch_disk_map[$key]="${disk_paths[1]}"
                disk_remaining[1]=$(( disk_remaining[1] - batch_size ))
            else
                echo "WARNING: $key ($(human_size "$batch_size")) does not fit on either disk!" >&2
                batch_disk_map[$key]="${disk_paths[0]}"
                disk_remaining[0]=$(( disk_remaining[0] - batch_size ))
            fi
        else
            if (( batch_size <= disk_remaining[1] )); then
                batch_disk_map[$key]="${disk_paths[1]}"
                disk_remaining[1]=$(( disk_remaining[1] - batch_size ))
            elif (( batch_size <= disk_remaining[0] )); then
                batch_disk_map[$key]="${disk_paths[0]}"
                disk_remaining[0]=$(( disk_remaining[0] - batch_size ))
            else
                echo "WARNING: $key ($(human_size "$batch_size")) does not fit on either disk!" >&2
                batch_disk_map[$key]="${disk_paths[1]}"
                disk_remaining[1]=$(( disk_remaining[1] - batch_size ))
            fi
        fi
    done <<< "$sorted_keys"

    # Output with disk assignment
    for line in "${lines[@]}"; do
        IFS=$'\t' read -r group batch_num folder_path size_bytes <<< "$line"
        local key="${group}:${batch_num}"
        echo -e "${batch_disk_map[$key]}\t${group}\t${batch_num}\t${folder_path}\t${size_bytes}"
    done
}

get_archive_name() {
    local group="$1"
    local batch_num="$2"
    if [[ "$group" == "main" ]]; then
        printf "archive_%03d.tar.gz" $((batch_num + 1))
    else
        # Custom prefix: GoPro_1.tar.gz, GoPro_2.tar.gz, etc.
        printf "%s_%d.tar.gz" "$group" $((batch_num + 1))
    fi
}

display_plan() {
    local plan_file="$1"
    local disk1_free="$2"
    local disk2_free="$3"
    local prev_key=""
    local prev_disk=""
    local batch_size=0
    local batch_folders=0
    local total_size=0
    local total_folders=0
    local total_batches=0

    echo ""
    log_color "==================== ARCHIVING PLAN ====================" "$CYAN"
    echo ""

    echo -e "  ${BLUE}Disk capacities:${NC}"
    echo -e "    $(basename "$TRANSIT_DISK_1"): $(human_size "$disk1_free") available"
    echo -e "    $(basename "$TRANSIT_DISK_2"): $(human_size "$disk2_free") available"
    echo ""

    # Sort by disk, then group, then batch
    while IFS=$'\t' read -r disk_path group batch_num folder_path size_bytes; do
        local key="${group}:${batch_num}"
        if [[ "$key" != "$prev_key" || "$disk_path" != "$prev_disk" ]]; then
            if [ -n "$prev_key" ]; then
                echo -e "    ${YELLOW}--- Batch total: $(human_size $batch_size), $batch_folders folder(s) ---${NC}"
                echo ""
            fi
            local archive_name
            archive_name=$(get_archive_name "$group" "$batch_num")
            echo -e "${GREEN}${archive_name}  ->  ${disk_path}/${NC}"
            batch_size=0
            batch_folders=0
            total_batches=$((total_batches + 1))
            prev_key="$key"
            prev_disk="$disk_path"
        fi
        echo "    $(human_size "$size_bytes")	$folder_path"
        batch_size=$((batch_size + size_bytes))
        batch_folders=$((batch_folders + 1))
        total_size=$((total_size + size_bytes))
        total_folders=$((total_folders + 1))
    done < <(sort -t$'\t' -k2,2V -k3,3n "$plan_file")

    if [ -n "$prev_key" ]; then
        echo -e "    ${YELLOW}--- Batch total: $(human_size $batch_size), $batch_folders folder(s) ---${NC}"
        echo ""
    fi

    echo -e "${CYAN}========================================================${NC}"
    echo -e "${CYAN}Total: $total_folders items, $(human_size $total_size) raw data${NC}"
    echo -e "${CYAN}Archives: $total_batches across 2 disks${NC}"

    # Per-disk summary
    local disk1_size=0 disk1_archives=0 disk2_size=0 disk2_archives=0
    local prev_count_key=""
    while IFS=$'\t' read -r disk_path group batch_num folder_path size_bytes; do
        local key="${disk_path}:${group}:${batch_num}"
        if [[ "$disk_path" == "$TRANSIT_DISK_1" ]]; then
            disk1_size=$((disk1_size + size_bytes))
            if [[ "$key" != "$prev_count_key" ]]; then
                disk1_archives=$((disk1_archives + 1))
            fi
        else
            disk2_size=$((disk2_size + size_bytes))
            if [[ "$key" != "$prev_count_key" ]]; then
                disk2_archives=$((disk2_archives + 1))
            fi
        fi
        prev_count_key="$key"
    done < <(sort -t$'\t' -k2,2V -k3,3n "$plan_file")

    echo ""
    echo -e "  ${BLUE}$(basename "$TRANSIT_DISK_1"):${NC} $disk1_archives archives, ~$(human_size $disk1_size) raw"
    echo -e "  ${BLUE}$(basename "$TRANSIT_DISK_2"):${NC} $disk2_archives archives, ~$(human_size $disk2_size) raw"
    echo -e "${CYAN}========================================================${NC}"
    echo ""
}

###############################################################################
# Phase 2: Sequential Archiving
###############################################################################

is_archive_completed() {
    local disk_path="$1"
    local archive_name="$2"
    local completed_file="$disk_path/$STATE_DIR_NAME/completed.txt"
    grep -Fxq "$archive_name" "$completed_file" 2>/dev/null
}

mark_archive_completed() {
    local disk_path="$1"
    local archive_name="$2"
    local completed_file="$disk_path/$STATE_DIR_NAME/completed.txt"
    echo "$archive_name" >> "$completed_file"
}

create_archive() {
    # Creates one archive on the given transit disk.
    # All steps sequential: create -> verify -> md5 -> manifest -> update master.
    local disk_path="$1"
    local group="$2"
    local batch_num="$3"
    shift 3
    local folders=("$@")

    local archive_name
    archive_name=$(get_archive_name "$group" "$batch_num")
    local archive_path="$disk_path/$archive_name"
    local state_dir="$disk_path/$STATE_DIR_NAME"
    local log_file="$state_dir/${archive_name%.tar.gz}.log"
    local manifest_file="$disk_path/${archive_name%.tar.gz}.manifest.txt"
    local master_manifest="$disk_path/$MASTER_MANIFEST_NAME"
    local disk_name
    disk_name=$(basename "$disk_path")

    mkdir -p "$state_dir"

    # Skip if already completed
    if is_archive_completed "$disk_path" "$archive_name"; then
        log_color "SKIP: $archive_name on $disk_name (already completed)" "$YELLOW"
        return 0
    fi

    # Remove partial archive if it exists
    if [ -f "$archive_path" ]; then
        log_color "Removing partial archive: $archive_name" "$YELLOW"
        rm -f "$archive_path"
    fi

    # Compute expected raw size
    local expected_raw=0
    for f in "${folders[@]}"; do
        local fsize
        fsize=$(du -sb "$SOURCE_DIR/$f" 2>/dev/null | cut -f1 || echo 0)
        expected_raw=$((expected_raw + fsize))
    done

    # Check disk free space
    local disk_free
    disk_free=$(get_disk_free_bytes "$disk_path")
    if (( expected_raw > disk_free )); then
        log_color "WARNING: $archive_name needs ~$(human_size $expected_raw) but $disk_name has $(human_size "$disk_free") free" "$RED"
        log_color "Skipping this archive — not enough space" "$RED"
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] SKIP: not enough disk space (need ~$(human_size $expected_raw), have $(human_size "$disk_free"))" >> "$log_file"
        return 1
    fi

    echo ""
    log_color "================================================================" "$BLUE"
    log_color "$archive_name -> $disk_name (${#folders[@]} folders, ~$(human_size $expected_raw) raw)" "$BLUE"
    log_color "================================================================" "$BLUE"

    echo "[$(date '+%Y-%m-%d %H:%M:%S')] START: $archive_name (${#folders[@]} folders, ~$(human_size $expected_raw) raw)" >> "$log_file"

    # Step 1: Create tar.gz archive with pigz
    log_color "Step 1/4: Creating archive with pigz..." "$BLUE"
    local start_time
    start_time=$(date +%s)

    if ! tar -cf - -C "$SOURCE_DIR" "${folders[@]}" 2>>"$log_file" | pigz -1 > "$archive_path"; then
        log_color "FAIL: Archive creation failed for $archive_name" "$RED"
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAIL: archive creation" >> "$log_file"
        rm -f "$archive_path"
        return 1
    fi

    local end_time elapsed
    end_time=$(date +%s)
    elapsed=$((end_time - start_time))
    local archive_size_bytes
    archive_size_bytes=$(stat -c%s "$archive_path" 2>/dev/null || stat -f%z "$archive_path" 2>/dev/null)
    log_color "Archive created: $(human_size "$archive_size_bytes") in ${elapsed}s" "$GREEN"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] Created: $(human_size "$archive_size_bytes") in ${elapsed}s" >> "$log_file"

    # Step 2: Verify archive
    log_color "Step 2/4: Verifying archive (tar -tzf)..." "$BLUE"
    local file_listing
    file_listing=$(mktemp)
    if ! tar -tzf "$archive_path" > "$file_listing" 2>>"$log_file"; then
        log_color "FAIL: Archive verification failed for $archive_name" "$RED"
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAIL: verification" >> "$log_file"
        rm -f "$archive_path" "$file_listing"
        return 1
    fi
    local file_count
    file_count=$(wc -l < "$file_listing" | tr -d ' ')
    log_color "Verified OK: $file_count entries" "$GREEN"

    # Step 3: Compute md5sum
    log_color "Step 3/4: Computing md5sum..." "$BLUE"
    local md5
    md5=$(md5sum "$archive_path" 2>/dev/null | cut -d' ' -f1 || md5 -q "$archive_path" 2>/dev/null)
    log_color "MD5: $md5" "$GREEN"

    # Step 4: Write per-archive manifest
    log_color "Step 4/4: Writing manifest..." "$BLUE"
    {
        echo "================================================================================"
        echo "ARCHIVE MANIFEST: $archive_name"
        echo "================================================================================"
        echo "Created: $(date '+%Y-%m-%d %H:%M:%S')"
        echo "Archive size: $(human_size "$archive_size_bytes") ($archive_size_bytes bytes)"
        echo "MD5: $md5"
        echo "Files/entries: $file_count"
        echo "Transit disk: $disk_path"
        echo "Compression time: ${elapsed}s"
        echo ""
        echo "Source folders:"
        for f in "${folders[@]}"; do
            local folder_size
            folder_size=$(du -sb "$SOURCE_DIR/$f" 2>/dev/null | cut -f1)
            echo "  $f  ($(human_size "$folder_size"))"
        done
        echo ""
        local archive_prefix="${archive_name%.tar.gz}"
        echo "Directory structure:"
        echo "--------------------------------------------------------------------------------"
        grep '/$' "$file_listing" | sed "s|^|${archive_prefix}/|"
        echo "--------------------------------------------------------------------------------"
        echo ""
        echo "Full file listing:"
        echo "--------------------------------------------------------------------------------"
        sed "s|^|${archive_prefix}/|" "$file_listing"
        echo ""
        echo "================================================================================"
    } > "$manifest_file"

    # Append full entry to master manifest (including complete file tree)
    {
        echo ""
        echo "================================================================================"
        echo "ARCHIVE: $archive_name"
        echo "================================================================================"
        echo "Created:          $(date '+%Y-%m-%d %H:%M:%S')"
        echo "Archive size:     $(human_size "$archive_size_bytes") ($archive_size_bytes bytes)"
        echo "MD5:              $md5"
        echo "Files/entries:    $file_count"
        echo "Compression time: ${elapsed}s"
        echo ""
        echo "Source folders:"
        for f in "${folders[@]}"; do
            local folder_size
            folder_size=$(du -sb "$SOURCE_DIR/$f" 2>/dev/null | cut -f1)
            echo "  $f  ($(human_size "$folder_size"))"
        done
        echo ""
        echo "Directory structure:"
        echo "--------------------------------------------------------------------------------"
        grep '/$' "$file_listing" | sed "s|^|${archive_prefix}/|"
        echo "--------------------------------------------------------------------------------"
        echo ""
        echo "Complete file/folder map:"
        echo "--------------------------------------------------------------------------------"
        sed "s|^|${archive_prefix}/|" "$file_listing"
        echo "--------------------------------------------------------------------------------"
    } >> "$master_manifest"

    rm -f "$file_listing"

    # Mark completed
    mark_archive_completed "$disk_path" "$archive_name"
    log_color "DONE: $archive_name" "$GREEN"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] DONE: $archive_name md5:$md5" >> "$log_file"
    return 0
}

###############################################################################
# Main
###############################################################################

cleanup_transit_disks() {
    log_color "========================================" "$BLUE"
    log_color "Cleaning up transit disks" "$BLUE"
    log_color "========================================" "$BLUE"
    echo ""

    for disk in "$TRANSIT_DISK_1" "$TRANSIT_DISK_2"; do
        local disk_name
        disk_name=$(basename "$disk")
        if [ ! -d "$disk" ]; then
            log_color "$disk_name: not mounted, skipping" "$YELLOW"
            continue
        fi

        echo -e "${BLUE}$disk_name ($disk):${NC}"

        # Count what will be removed
        local archive_count manifest_count
        archive_count=$(find "$disk" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | wc -l | tr -d ' ')
        manifest_count=$(find "$disk" -maxdepth 1 -name "*.manifest.txt" 2>/dev/null | wc -l | tr -d ' ')
        local has_state=false
        [ -d "$disk/$STATE_DIR_NAME" ] && has_state=true
        local has_master=false
        [ -f "$disk/$MASTER_MANIFEST_NAME" ] && has_master=true

        echo "  Archives:  $archive_count"
        echo "  Manifests: $manifest_count"
        echo "  State dir: $has_state"
        echo "  Master manifest: $has_master"

        if (( archive_count == 0 )) && ! $has_state && ! $has_master; then
            log_color "  Nothing to clean" "$GREEN"
            echo ""
            continue
        fi

        if ! $AUTO_YES; then
            read -p "  Delete all of the above from $disk_name? (yes/no): " confirm
            if [[ "$confirm" != "yes" ]]; then
                log "  Skipped $disk_name"
                echo ""
                continue
            fi
        fi

        # Remove archives
        find "$disk" -maxdepth 1 -name "*.tar.gz" -delete 2>/dev/null
        # Remove per-archive manifests
        find "$disk" -maxdepth 1 -name "*.manifest.txt" -delete 2>/dev/null
        # Remove master manifest
        rm -f "$disk/$MASTER_MANIFEST_NAME"
        # Remove state directory
        rm -rf "$disk/$STATE_DIR_NAME"

        log_color "  Cleaned $disk_name" "$GREEN"
        echo ""
    done

    log_color "Cleanup complete" "$GREEN"
}

main() {
    # Handle --cleanup before anything else
    if $CLEANUP; then
        cleanup_transit_disks
        exit 0
    fi

    log_color "========================================" "$BLUE"
    log_color "Glacier Archive Split Script" "$BLUE"
    log_color "========================================" "$BLUE"
    echo ""

    # Check prerequisites
    if ! command -v pigz &> /dev/null; then
        error_exit "pigz is not installed. Install it with: sudo apt install pigz"
    fi

    if [ ! -d "$SOURCE_DIR" ]; then
        error_exit "Source directory not found: $SOURCE_DIR"
    fi

    # Get disk capacities (required even for dry-run)
    local disk1_free=0 disk2_free=0
    for disk in "$TRANSIT_DISK_1" "$TRANSIT_DISK_2"; do
        if [ ! -d "$disk" ]; then
            error_exit "Transit disk not found: $disk"
        fi
    done
    disk1_free=$(get_disk_free_bytes "$TRANSIT_DISK_1")
    disk2_free=$(get_disk_free_bytes "$TRANSIT_DISK_2")
    if [ -z "$disk1_free" ] || [ "$disk1_free" -eq 0 ]; then
        error_exit "Cannot determine free space on $TRANSIT_DISK_1"
    fi
    if [ -z "$disk2_free" ] || [ "$disk2_free" -eq 0 ]; then
        error_exit "Cannot determine free space on $TRANSIT_DISK_2"
    fi
    log "Disk 1 ($(basename "$TRANSIT_DISK_1")): $(human_size "$disk1_free") free"
    log "Disk 2 ($(basename "$TRANSIT_DISK_2")): $(human_size "$disk2_free") free"

    # Phase 1: Scan and plan
    log_color "Phase 1: Building archiving plan..." "$BLUE"
    local plan_file
    plan_file=$(mktemp)

    scan_folders | build_batches | assign_disks "$disk1_free" "$disk2_free" > "$plan_file"

    if [ ! -s "$plan_file" ]; then
        rm -f "$plan_file"
        error_exit "No folders to archive"
    fi

    # Count completed archives
    local total_archives=0
    local completed=0
    while IFS=$'\t' read -r disk_path group batch_num; do
        total_archives=$((total_archives + 1))
        local archive_name
        archive_name=$(get_archive_name "$group" "$batch_num")
        if is_archive_completed "$disk_path" "$archive_name"; then
            completed=$((completed + 1))
        fi
    done < <(awk -F'\t' '{print $1"\t"$2"\t"$3}' "$plan_file" | sort -u)

    display_plan "$plan_file" "$disk1_free" "$disk2_free"

    if (( completed > 0 )); then
        echo -e "${YELLOW}Previously completed: $completed / $total_archives archives (will be skipped)${NC}"
        echo ""
    fi

    if $DRY_RUN; then
        log_color "DRY RUN mode - no archives will be created" "$YELLOW"
        echo ""

        # Generate preview manifest with full file/folder maps
        local preview_manifest="$SOURCE_DIR/archive_master_manifest_PREVIEW.txt"
        log_color "Generating preview manifest with full file maps..." "$BLUE"
        log_color "Output: $preview_manifest" "$BLUE"

        {
            echo "================================================================================"
            echo "ARCHIVE MASTER MANIFEST — PREVIEW (dry-run)"
            echo "================================================================================"
            echo "Source: $SOURCE_DIR"
            echo "Generated: $(date '+%Y-%m-%d %H:%M:%S')"
            echo ""
            echo "This is a DRY-RUN preview. No archives were created."
            echo "It shows which files/folders would go into each archive."
            echo "================================================================================"
        } > "$preview_manifest"

        local prev_key="" prev_disk="" prev_group="" prev_batch=""
        local current_folders=()

        while IFS=$'\t' read -r disk_path group batch_num folder_path size_bytes; do
            local key="${group}:${batch_num}"
            if [[ "$key" != "$prev_key" || "$disk_path" != "$prev_disk" ]]; then
                # Write previous batch to manifest
                if [ -n "$prev_key" ] && [ ${#current_folders[@]} -gt 0 ]; then
                    local aname
                    aname=$(get_archive_name "$prev_group" "$prev_batch")
                    log_color "  Scanning files for $aname..." "$CYAN"
                    {
                        echo ""
                        echo "================================================================================"
                        echo "ARCHIVE: $aname"
                        echo "================================================================================"
                        echo "Transit disk: $prev_disk"
                        echo ""
                        echo "Source folders:"
                        for f in "${current_folders[@]}"; do
                            local folder_size
                            folder_size=$(du -sb "$SOURCE_DIR/$f" 2>/dev/null | cut -f1)
                            echo "  $f  ($(human_size "$folder_size"))"
                        done
                        echo ""
                        local aprefix="${aname%.tar.gz}"
                        echo "Directory structure:"
                        echo "--------------------------------------------------------------------------------"
                        for f in "${current_folders[@]}"; do
                            find "$SOURCE_DIR/$f" -type d -printf "%P\n" 2>/dev/null | sort | while IFS= read -r dpath; do
                                [ -z "$dpath" ] && echo "${aprefix}/$f/" && continue
                                echo "${aprefix}/$f/$dpath/"
                            done
                        done
                        echo "--------------------------------------------------------------------------------"
                        echo ""
                        echo "Complete file/folder map:"
                        echo "--------------------------------------------------------------------------------"
                        for f in "${current_folders[@]}"; do
                            find "$SOURCE_DIR/$f" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r fpath; do
                                [ -z "$fpath" ] && continue
                                echo "${aprefix}/$f/$fpath"
                            done
                        done
                        echo "--------------------------------------------------------------------------------"
                    } >> "$preview_manifest"
                fi
                prev_key="$key"
                prev_disk="$disk_path"
                prev_group="$group"
                prev_batch="$batch_num"
                current_folders=()
            fi
            current_folders+=( "$folder_path" )
        done < <(sort -t$'\t' -k2,2V -k3,3n "$plan_file")

        # Last batch
        if [ -n "$prev_key" ] && [ ${#current_folders[@]} -gt 0 ]; then
            local aname
            aname=$(get_archive_name "$prev_group" "$prev_batch")
            log_color "  Scanning files for $aname..." "$CYAN"
            {
                echo ""
                echo "================================================================================"
                echo "ARCHIVE: $aname"
                echo "================================================================================"
                echo "Transit disk: $prev_disk"
                echo ""
                echo "Source folders:"
                for f in "${current_folders[@]}"; do
                    local folder_size
                    folder_size=$(du -sb "$SOURCE_DIR/$f" 2>/dev/null | cut -f1)
                    echo "  $f  ($(human_size "$folder_size"))"
                done
                echo ""
                local aprefix="${aname%.tar.gz}"
                echo "Directory structure:"
                echo "--------------------------------------------------------------------------------"
                for f in "${current_folders[@]}"; do
                    find "$SOURCE_DIR/$f" -type d -printf "%P\n" 2>/dev/null | sort | while IFS= read -r dpath; do
                        [ -z "$dpath" ] && echo "${aprefix}/$f/" && continue
                        echo "${aprefix}/$f/$dpath/"
                    done
                done
                echo "--------------------------------------------------------------------------------"
                echo ""
                echo "Complete file/folder map:"
                echo "--------------------------------------------------------------------------------"
                for f in "${current_folders[@]}"; do
                    find "$SOURCE_DIR/$f" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r fpath; do
                        [ -z "$fpath" ] && continue
                        echo "${aprefix}/$f/$fpath"
                    done
                done
                echo "--------------------------------------------------------------------------------"
            } >> "$preview_manifest"
        fi

        local manifest_size
        manifest_size=$(du -sh "$preview_manifest" 2>/dev/null | cut -f1)
        echo ""
        log_color "Preview manifest written: $preview_manifest ($manifest_size)" "$GREEN"
        log_color "Review it with: less $preview_manifest" "$GREEN"

        rm -f "$plan_file"
        exit 0
    fi

    # Confirmation
    echo -e "${YELLOW}This script will:${NC}"
    echo "  1. Create tar.gz archives using pigz (fast parallel compression)"
    echo "  2. Distribute archives across: $TRANSIT_DISK_1 and $TRANSIT_DISK_2"
    echo "  3. Verify each archive and compute md5 (sequential, one at a time)"
    echo "  4. Write full file/folder map to master manifest on each disk"
    echo ""
    echo -e "${YELLOW}Processing order: strictly sequential — one archive at a time${NC}"
    echo ""
    if ! $AUTO_YES; then
        read -p "Continue? (yes/no): " confirm
        if [[ "$confirm" != "yes" ]]; then
            log "Operation cancelled by user"
            rm -f "$plan_file"
            exit 0
        fi
    else
        log "Auto-confirmed (--yes)"
    fi

    # Initialize state dirs and master manifests
    for disk in "$TRANSIT_DISK_1" "$TRANSIT_DISK_2"; do
        mkdir -p "$disk/$STATE_DIR_NAME"
        touch "$disk/$STATE_DIR_NAME/completed.txt"
        if [ ! -f "$disk/$MASTER_MANIFEST_NAME" ]; then
            {
                echo "================================================================================"
                echo "ARCHIVE MASTER MANIFEST"
                echo "================================================================================"
                echo "Source: $SOURCE_DIR"
                echo "Created: $(date '+%Y-%m-%d %H:%M:%S')"
                echo "Transit disk: $disk"
                echo ""
                echo "This file contains the complete file/folder map for every archive"
                echo "stored on this disk. Use it to find which archive contains a given file."
                echo "================================================================================"
            } > "$disk/$MASTER_MANIFEST_NAME"
        fi
    done

    # Phase 2: Process archives sequentially
    log_color "Phase 2: Starting sequential archiving..." "$BLUE"

    local success_count=0
    local error_count=0
    local prev_key=""
    local prev_disk=""
    local prev_group=""
    local prev_batch=""
    local current_folders=()

    while IFS=$'\t' read -r disk_path group batch_num folder_path size_bytes; do
        local key="${group}:${batch_num}"
        if [[ "$key" != "$prev_key" || "$disk_path" != "$prev_disk" ]]; then
            # Process previous batch
            if [ -n "$prev_key" ] && [ ${#current_folders[@]} -gt 0 ]; then
                if create_archive "$prev_disk" "$prev_group" "$prev_batch" "${current_folders[@]}"; then
                    success_count=$((success_count + 1))
                else
                    error_count=$((error_count + 1))
                fi
            fi
            prev_key="$key"
            prev_disk="$disk_path"
            prev_group="$group"
            prev_batch="$batch_num"
            current_folders=()
        fi
        current_folders+=( "$folder_path" )
    done < <(sort -t$'\t' -k2,2V -k3,3n "$plan_file")

    # Process last batch
    if [ -n "$prev_key" ] && [ ${#current_folders[@]} -gt 0 ]; then
        if create_archive "$prev_disk" "$prev_group" "$prev_batch" "${current_folders[@]}"; then
            success_count=$((success_count + 1))
        else
            error_count=$((error_count + 1))
        fi
    fi

    rm -f "$plan_file"

    # Final summary
    echo ""
    log_color "========================================" "$BLUE"
    log_color "ARCHIVING COMPLETE" "$BLUE"
    log_color "========================================" "$BLUE"
    log "Successful: $success_count"
    log "Failed: $error_count"
    echo ""

    for disk in "$TRANSIT_DISK_1" "$TRANSIT_DISK_2"; do
        local disk_name
        disk_name=$(basename "$disk")
        local archive_count
        archive_count=$(find "$disk" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | wc -l | tr -d ' ')
        local disk_free_now
        disk_free_now=$(human_size "$(get_disk_free_bytes "$disk")")
        echo -e "  ${BLUE}$disk_name:${NC} $archive_count archives, ${disk_free_now} free"
    done

    echo ""
    log_color "Master manifest with full file maps: $MASTER_MANIFEST_NAME on each disk" "$GREEN"
    log_color "Next step: upload archives to S3 Glacier Deep Archive" "$YELLOW"

    return $error_count
}

main "$@"
