#!/usr/bin/env bash
#
# Example:
#   chmod +x download_google_verrus_sources.sh
#   ./download_google_verrus_sources.sh
#
# Include a local copy of the Salem agenda package:
#   ./download_google_verrus_sources.sh \
#       --agenda ../Agenda_-_FullPackage_City+Council_Jul27_2026.pdf
#
# Specify another output directory:
#   ./download_google_verrus_sources.sh \
#       --output-dir ./docs \
#       --agenda /path/to/Agenda_-_FullPackage_City+Council_Jul27_2026.pdf
#
# $Id$
# $HeadURL$
#
# Download and preserve publicly available source material supporting the
# Google / Alphabet / Sidewalk Infrastructure Partners / Verrus relationship
# diagram.
#
# Provenance retained for each remote source:
#
#   * Original source URL
#   * Final URL after redirects
#   * Local retrieval timestamp
#   * Local filesystem modification timestamp
#   * HTTP response status
#   * Server Date header
#   * Server Last-Modified header
#   * Server ETag
#   * Content-Type
#   * Raw wget response headers
#   * File byte count
#   * SHA-256 digest
#
# HTML pages are saved as delivered by the remote server. This preserves the
# returned source HTML but does not execute JavaScript or create a screenshot
# of the rendered page.
#

set -o errexit
set -o nounset
set -o pipefail

PROGRAM_NAME="${0##*/}"
PROGRAM_DIR="$(cd -- "$(dirname -- "$0")" && pwd -P)"

OUTPUT_DIR="${PROGRAM_DIR}/docs"
AGENDA_FILE=""

usage()
{
    cat <<EOF
Usage:
  ${PROGRAM_NAME} [options]

Options:
  --output-dir DIR    Destination directory.
                      Default: ${PROGRAM_DIR}/docs

  --agenda FILE       Copy an existing Salem City Council agenda PDF into
                      the snapshot directory and include it in the manifest.

  -h, --help          Display this help.

Examples:
  ./${PROGRAM_NAME}

  ./${PROGRAM_NAME} \\
      --agenda ../Agenda_-_FullPackage_City+Council_Jul27_2026.pdf

  ./${PROGRAM_NAME} \\
      --output-dir ./docs \\
      --agenda /path/to/Agenda_-_FullPackage_City+Council_Jul27_2026.pdf
EOF
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --output-dir)
            if [[ $# -lt 2 ]]; then
                printf 'ERROR: --output-dir requires an argument.\n' >&2
                exit 2
            fi

            OUTPUT_DIR="$2"
            shift 2
            ;;

        --agenda)
            if [[ $# -lt 2 ]]; then
                printf 'ERROR: --agenda requires an argument.\n' >&2
                exit 2
            fi

            AGENDA_FILE="$2"
            shift 2
            ;;

        -h|--help)
            usage
            exit 0
            ;;

        *)
            printf 'ERROR: Unknown argument: %s\n\n' "$1" >&2
            usage >&2
            exit 2
            ;;
    esac
done

required_programs=(
    awk
    date
    find
    sha256sum
    sort
    stat
    tee
    tr
    wget
)

for required_program in "${required_programs[@]}"; do
    if ! command -v "$required_program" >/dev/null 2>&1; then
        printf 'ERROR: Required program not found: %s\n' \
            "$required_program" >&2
        exit 1
    fi
done

mkdir -p -- "$OUTPUT_DIR"
OUTPUT_DIR="$(cd -- "$OUTPUT_DIR" && pwd -P)"

TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
RETRIEVAL_STARTED_ISO="$(date '+%Y-%m-%dT%H:%M:%S%z')"

LOG_FILE="${OUTPUT_DIR}/download_log_${TIMESTAMP}.log"
MANIFEST_FILE="${OUTPUT_DIR}/source_manifest_${TIMESTAMP}.tsv"
CHECKSUM_FILE="${OUTPUT_DIR}/SHA256SUMS_${TIMESTAMP}.txt"
FAILURE_FILE="${OUTPUT_DIR}/download_failures_${TIMESTAMP}.tsv"

: > "$LOG_FILE"

printf '%s\n' \
    $'source_url\terror_type\tdetail' \
    > "$FAILURE_FILE"

printf '%s\n' \
    $'snapshot_timestamp\tretrieved_at\tlocal_mtime\tfilename\tbytes\tsha256\tsource_url\tfinal_url\thttp_status\tserver_date\tlast_modified\tetag\tcontent_type\tdescription' \
    > "$MANIFEST_FILE"

log()
{
    printf '%s %s\n' \
        "$(date '+%Y-%m-%dT%H:%M:%S%z')" \
        "$*" \
        | tee -a "$LOG_FILE"
}

sanitize_tsv()
{
    printf '%s' "$1" | tr '\t\r\n' '   '
}

extract_last_header()
{
    local header_name="$1"
    local header_file="$2"

    awk -v wanted="$header_name" '
        BEGIN {
            IGNORECASE = 1
        }

        {
            line = $0
            sub(/^[[:space:]]+/, "", line)

            prefix = wanted ":"

            if (tolower(substr(line, 1, length(prefix))) == tolower(prefix)) {
                value = substr(line, length(prefix) + 1)
                sub(/^[[:space:]]+/, "", value)
            }
        }

        END {
            print value
        }
    ' "$header_file"
}

extract_last_http_status()
{
    local header_file="$1"

    awk '
        /^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]/ {
            line = $0
            sub(/^[[:space:]]+/, "", line)
            split(line, fields, /[[:space:]]+/)
            status = fields[2]
        }

        END {
            print status
        }
    ' "$header_file"
}

extract_final_location()
{
    local header_file="$1"

    extract_last_header 'Location' "$header_file"
}

record_failure()
{
    local source_url="$1"
    local error_type="$2"
    local detail="$3"

    printf '%s\t%s\t%s\n' \
        "$(sanitize_tsv "$source_url")" \
        "$(sanitize_tsv "$error_type")" \
        "$(sanitize_tsv "$detail")" \
        >> "$FAILURE_FILE"
}

write_manifest_record()
{
    local retrieved_at="$1"
    local local_mtime="$2"
    local output_name="$3"
    local byte_count="$4"
    local sha256="$5"
    local source_url="$6"
    local final_url="$7"
    local http_status="$8"
    local server_date="$9"
    local last_modified="${10}"
    local etag="${11}"
    local content_type="${12}"
    local description="${13}"

    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
        "$TIMESTAMP" \
        "$(sanitize_tsv "$retrieved_at")" \
        "$(sanitize_tsv "$local_mtime")" \
        "$(sanitize_tsv "$output_name")" \
        "$(sanitize_tsv "$byte_count")" \
        "$(sanitize_tsv "$sha256")" \
        "$(sanitize_tsv "$source_url")" \
        "$(sanitize_tsv "$final_url")" \
        "$(sanitize_tsv "$http_status")" \
        "$(sanitize_tsv "$server_date")" \
        "$(sanitize_tsv "$last_modified")" \
        "$(sanitize_tsv "$etag")" \
        "$(sanitize_tsv "$content_type")" \
        "$(sanitize_tsv "$description")" \
        >> "$MANIFEST_FILE"
}

download_source()
{
    local base_name="$1"
    local extension="$2"
    local url="$3"
    local description="$4"

    local output_name="${base_name}_${TIMESTAMP}.${extension}"
    local output_path="${OUTPUT_DIR}/${output_name}"
    local temporary_path="${output_path}.part"
    local header_name="${base_name}_${TIMESTAMP}.wget_headers.txt"
    local header_path="${OUTPUT_DIR}/${header_name}"

    local retrieved_at=""
    local local_mtime=""
    local byte_count=""
    local sha256=""
    local http_status=""
    local server_date=""
    local last_modified=""
    local etag=""
    local content_type=""
    local redirect_location=""
    local final_url="$url"
    local wget_status=0

    log "Downloading: ${description}"
    log "URL: ${url}"

    rm -f -- "$temporary_path"

    retrieved_at="$(date '+%Y-%m-%dT%H:%M:%S%z')"

    if wget \
        --server-response \
        --https-only \
        --no-use-server-timestamps \
        --max-redirect=20 \
        --timeout=30 \
        --read-timeout=60 \
        --tries=4 \
        --retry-connrefused \
        --waitretry=3 \
        --user-agent='Mozilla/5.0 source-preservation/1.0' \
        --output-document="$temporary_path" \
        "$url" \
        2>"$header_path"
    then
        :
    else
        wget_status=$?

        http_status="$(extract_last_http_status "$header_path")"

        log "ERROR: wget exited with status ${wget_status}: ${url}"

        record_failure \
            "$url" \
            'wget failure' \
            "wget_status=${wget_status}; http_status=${http_status}"

        rm -f -- "$temporary_path"
        return 1
    fi

    if [[ ! -s "$temporary_path" ]]; then
        log "ERROR: Downloaded file is empty: ${url}"

        record_failure \
            "$url" \
            'empty downloaded file' \
            "temporary_path=${temporary_path}"

        rm -f -- "$temporary_path"
        return 1
    fi

    mv -- "$temporary_path" "$output_path"

    http_status="$(extract_last_http_status "$header_path")"
    server_date="$(extract_last_header 'Date' "$header_path")"
    last_modified="$(extract_last_header 'Last-Modified' "$header_path")"
    etag="$(extract_last_header 'ETag' "$header_path")"
    content_type="$(extract_last_header 'Content-Type' "$header_path")"
    redirect_location="$(extract_final_location "$header_path")"

    if [[ -n "$redirect_location" ]]; then
        final_url="$redirect_location"
    fi

    local_mtime="$(stat --format='%y' "$output_path")"
    byte_count="$(stat --format='%s' "$output_path")"
    sha256="$(sha256sum "$output_path" | awk '{print $1}')"

    write_manifest_record \
        "$retrieved_at" \
        "$local_mtime" \
        "$output_name" \
        "$byte_count" \
        "$sha256" \
        "$url" \
        "$final_url" \
        "$http_status" \
        "$server_date" \
        "$last_modified" \
        "$etag" \
        "$content_type" \
        "$description"

    log "Saved: ${output_path}"
    log "Bytes: ${byte_count}"
    log "SHA-256: ${sha256}"

    if [[ -n "$server_date" ]]; then
        log "Server Date: ${server_date}"
    fi

    if [[ -n "$last_modified" ]]; then
        log "Server Last-Modified: ${last_modified}"
    fi

    return 0
}

copy_local_source()
{
    local source_path="$1"
    local base_name="$2"
    local description="$3"

    local extension=""
    local output_name=""
    local output_path=""
    local retrieved_at=""
    local local_mtime=""
    local byte_count=""
    local sha256=""

    if [[ ! -f "$source_path" ]]; then
        log "ERROR: Local source does not exist: ${source_path}"

        record_failure \
            "$source_path" \
            'local file not found' \
            'The file specified with --agenda does not exist.'

        return 1
    fi

    extension="${source_path##*.}"
    output_name="${base_name}_${TIMESTAMP}.${extension}"
    output_path="${OUTPUT_DIR}/${output_name}"
    retrieved_at="$(date '+%Y-%m-%dT%H:%M:%S%z')"

    cp -- "$source_path" "$output_path"

    local_mtime="$(stat --format='%y' "$output_path")"
    byte_count="$(stat --format='%s' "$output_path")"
    sha256="$(sha256sum "$output_path" | awk '{print $1}')"

    write_manifest_record \
        "$retrieved_at" \
        "$local_mtime" \
        "$output_name" \
        "$byte_count" \
        "$sha256" \
        "local:${source_path}" \
        "" \
        "" \
        "" \
        "" \
        "" \
        "application/pdf" \
        "$description"

    log "Copied local source: ${output_path}"
    log "Bytes: ${byte_count}"
    log "SHA-256: ${sha256}"
}

generate_checksum_file()
{
    local manifest_basename=""
    local log_basename=""
    local failure_basename=""
    local checksum_basename=""

    manifest_basename="$(basename -- "$MANIFEST_FILE")"
    log_basename="$(basename -- "$LOG_FILE")"
    failure_basename="$(basename -- "$FAILURE_FILE")"
    checksum_basename="$(basename -- "$CHECKSUM_FILE")"

    log "Generating SHA-256 checksum file."

    (
        cd -- "$OUTPUT_DIR"

        find . \
            -maxdepth 1 \
            -type f \
            \( \
                -name "*_${TIMESTAMP}.html" -o \
                -name "*_${TIMESTAMP}.pdf" -o \
                -name "*_${TIMESTAMP}.png" -o \
                -name "*_${TIMESTAMP}.txt" \
            \) \
            ! -name "$checksum_basename" \
            ! -name "$log_basename" \
            -print0 \
            | sort -z \
            | xargs -0 --no-run-if-empty sha256sum

        sha256sum \
            "$manifest_basename" \
            "$failure_basename"
    ) > "$CHECKSUM_FILE"
}



count_manifest_records()
{
    awk -F '\t' '
        NR > 1 {
            count++
        }

        END {
            print count + 0
        }
    ' "$MANIFEST_FILE"
}

count_failures()
{
    awk -F '\t' '
        NR > 1 {
            count++
        }

        END {
            print count + 0
        }
    ' "$FAILURE_FILE"
}

log "Beginning Google / Alphabet / SIP / Verrus source snapshot."
log "Output directory: ${OUTPUT_DIR}"
log "Snapshot timestamp: ${TIMESTAMP}"
log "Retrieval started: ${RETRIEVAL_STARTED_ISO}"

required_failure_count=0
optional_failure_count=0

##############################################################################
# Alphabet financial information
##############################################################################

download_source \
    'Alphabet_2025_Form_10-K' \
    'html' \
    'https://www.sec.gov/Archives/edgar/data/1652044/000165204426000018/goog-20251231.htm' \
    'Alphabet Inc. Form 10-K for fiscal year ended December 31, 2025' \
    || ((required_failure_count += 1))

download_source \
    'Alphabet_2025_Form_10-K_Filing_Index' \
    'html' \
    'https://www.sec.gov/Archives/edgar/data/1652044/000165204426000018/0001652044-26-000018-index.html' \
    'SEC filing index for Alphabet Inc. 2025 Form 10-K' \
    || ((required_failure_count += 1))

download_source \
    'Alphabet_FY2025_Results' \
    'html' \
    'https://www.sec.gov/Archives/edgar/data/1652044/000165204426000012/googexhibit991q42025.htm' \
    'Alphabet fourth-quarter and fiscal-year 2025 earnings release' \
    || ((required_failure_count += 1))

##############################################################################
# Sidewalk Infrastructure Partners
##############################################################################

download_source \
    'Sidewalk_Infrastructure_Partners_About' \
    'html' \
    'https://sidewalkinfra.com/about' \
    'SIP description of its Alphabet origins, Google relationship, and institutional investors' \
    || ((required_failure_count += 1))

##############################################################################
# Ontario Teachers' Pension Plan
##############################################################################

download_source \
    'Ontario_Teachers_SIP_Partnership_2019' \
    'html' \
    'https://www.otpp.com/en-ca/about-us/news-and-insights/2019/transforming-infrastructure-through-technology-with-a-new-partnership/' \
    'Ontario Teachers description of SIP as a Sidewalk Labs spinout and partnership' \
    || ((required_failure_count += 1))

##############################################################################
# Verrus
##############################################################################

download_source \
    'Verrus_About_Us' \
    'html' \
    'https://www.verrusdata.com/about-us' \
    'Verrus leadership, advisory board, Google personnel history, and SIP-company statement' \
    || ((required_failure_count += 1))

download_source \
    'Verrus_Data_Software_Engineer_Job_Posting' \
    'html' \
    'https://www.verrusdata.com/careers/data-software-engineer-optimization-' \
    'Verrus description of SIP investment from Alphabet, Ontario Teachers, and StepStone, and SIP formation of Verrus' \
    || ((required_failure_count += 1))

download_source \
    'Verrus_Staff_Site_Reliability_Engineer_Job_Posting' \
    'html' \
    'https://www.verrusdata.com/careers/software-engineer-sre' \
    'Verrus Staff Site Reliability Engineer posting and Mountain View operating description' \
    || ((required_failure_count += 1))

##############################################################################
# Historical Verrus page
#
# Search indexes may retain this page even when the origin server returns an
# HTTP error. Failure is recorded for provenance but does not cause the
# overall snapshot operation to fail.
##############################################################################

if ! download_source \
    'Verrus_Product_Manager_Job_Posting_Historical' \
    'html' \
    'https://www.verrusdata.com/careers/product-manager' \
    'Historical Verrus Product Manager posting describing SIP backing and formation of Verrus'
then
    ((optional_failure_count += 1))
    log "WARNING: Historical Product Manager page is unavailable; continuing."
fi

##############################################################################
# Local Salem agenda package
##############################################################################

if [[ -n "$AGENDA_FILE" ]]; then
    copy_local_source \
        "$AGENDA_FILE" \
        'Salem_City_Council_Agenda_20260727' \
        'City of Salem July 27, 2026 Council agenda package concerning the Verrus proposal' \
        || ((required_failure_count += 1))
else
    log "No --agenda file specified; Salem agenda package was not copied."
fi

##############################################################################
# Checksums and summary
##############################################################################

generate_checksum_file

manifest_record_count="$(count_manifest_records)"
recorded_failure_count="$(count_failures)"

log "Snapshot completed."
log "Files recorded in manifest: ${manifest_record_count}"
log "Required-source failures: ${required_failure_count}"
log "Optional-source failures: ${optional_failure_count}"
log "Recorded failure entries: ${recorded_failure_count}"
log "Manifest: ${MANIFEST_FILE}"
log "Checksums: ${CHECKSUM_FILE}"
log "Log: ${LOG_FILE}"

if [[ "$recorded_failure_count" -gt 0 ]]; then
    log "Failure report: ${FAILURE_FILE}"
else
    rm -f -- "$FAILURE_FILE"
    log "No failures were recorded."
fi

if [[ "$required_failure_count" -gt 0 ]]; then
    log "ERROR: One or more required sources could not be preserved."
    exit 1
fi

if [[ "$optional_failure_count" -gt 0 ]]; then
    log "Completed successfully with optional-source warnings."
else
    log "Completed successfully."
fi

exit 0