#!/bin/bash
# archive_v5.sh - NXP Mongo log retention (export -> zip -> S3 Glacier IR -> delete), replaces archive_v4.sh
# Draft Enigma 2 Nov 2026 (todo 9262), after Mongo down 1 Nov 20:31-21:37 WIB.
#
# What changed vs v4 (root cause of the 1 Nov incident):
#  1. Filter on _id (ObjectId timestamp = insert time, _id index always exists) instead of
#     $or creationTime/requestTime (COLLSCAN on every collection where one branch has no index,
#     e.g. activitylog_deposits 810M docs).
#  2. db.getCollection(name) instead of db.$collection: v4's delete failed for the 9 collections whose
#     name has '-' or '/' (apilog_*_payment-methods 18M docs each, data since 14 Oct 2024), export still
#     found data, so the 'stop when a day has no data' check never fired -> the loop walked back day by
#     day toward 2024, full-scanning ~206 collections per day, running 20h+ and overlapping the next cron.
#  3. Bounded forward loop: oldest day -> cutoff day, max MAX_DAYS per run, stops at STOP_AT (no endless walk).
#  4. Delete in batches (BATCH ids per deleteMany, SLEEP_MS pause, maxTimeMS per query), only after the
#     export count == countDocuments for that collection+day and the S3 upload succeeded.
#  5. flock -n (no double instance) + hard timeout in cron + timestamped log.
#  6. S3 key unique per run (<day>_<runts>.zip) + upload verified before delete (todo 9325): v4 re-walked the same
#     days every night and re-uploaded <day>.zip -> overwrote the earlier full archive with a partial one.
#
# Cron (server TZ = UTC), run 02:00-05:00 WIB only:
#   0 19 * * * /usr/bin/flock -n /var/lock/mongo-archive.lock /usr/bin/timeout -k 60 3h30m /var/www/scripts/archive_v5.sh
# First run: DRY_RUN=1 /var/www/scripts/archive_v5.sh  (counts only, no export/delete)

set -uo pipefail

DB="${DB:-xxx}"
PORT="${PORT:-00000}"
MONGO_URI="${MONGO_URI:-mongodb://localhost:$PORT/$DB?authSource=admin}"
S3_BUCKET="${S3_BUCKET:-s3://mongo-archive.xxx.xxx}"
REGION="${REGION:-ap-southeast-1}"
RETAIN="${RETAIN:-2 months}"          # keep data newer than today - RETAIN (v4: -2 months)
MAX_DAYS="${MAX_DAYS:-30}"             # days archived per run (backlog catches up gradually)
BATCH="${BATCH:-10000}"               # docs per deleteMany
SLEEP_MS="${SLEEP_MS:-1000}"          # pause between delete batches
MAX_TIME_MS="${MAX_TIME_MS:-60000}"   # per query
STOP_AT="${STOP_AT:-$(date -u -d 'today 22:00' +%s)}"   # 22:00 UTC = 05:00 WIB, no new day after this
DRY_RUN="${DRY_RUN:-0}"
WORK_DIR="${WORK_DIR:-/opt/mongodb-archive}"
LOG_DIR="${LOG_DIR:-/var/log/mongo-archive}"

mkdir -p "$LOG_DIR" "$WORK_DIR"
LOG_FILE="$LOG_DIR/run_$(date -u +%Y%m%d).log"
log() { echo "$(date -u '+%F %T') [$$] $*" | tee -a "$LOG_FILE"; }

# Self-lock too, in case someone runs it by hand while cron is running.
exec 9>"${LOCK_FILE:-/var/lock/mongo-archive.lock.self}"
if ! flock -n 9; then log "another archive_v5 is running, exit"; exit 0; fi

msh() { mongosh "$MONGO_URI" --quiet "$@"; }
oid() { printf '%08x0000000000000000' "$(date -u -d "$1 00:00:00" +%s)"; }   # ObjectId lower bound of a UTC day

CUTOFF=$(date -u -d "-$RETAIN" +%F)
log "start archive_v5 cutoff=$CUTOFF (archive days < cutoff) max_days=$MAX_DAYS dry=$DRY_RUN"

# Collections with ObjectId _id + their oldest doc day (uses _id index, cheap). Tab separated: name<TAB>YYYY-MM-DD
COLLS=$(msh --eval '
  db.getCollectionInfos({type: "collection"}).map(i => i.name).filter(n => !n.startsWith("system.")).forEach(n => {
    const d = db.getCollection(n).find({}, {_id: 1, creationTime: 1, requestTime: 1}).sort({_id: 1}).limit(1).maxTimeMS(5000).toArray()[0];
    if (!d) return;
    if (!(d._id instanceof ObjectId)) print("#SKIP\t" + n + " (_id not ObjectId)");
    else if (!d.creationTime && !d.requestTime) print("#SKIP\t" + n + " (no creationTime/requestTime, out of scope like v4)");
    else print(n + "\t" + d._id.getTimestamp().toISOString().slice(0, 10));
  })') || { log "ERROR listing collections"; exit 1; }
echo "$COLLS" | grep '^#SKIP' | while IFS=$'\t' read -r _ msg; do log "skip $msg"; done
COLLS=$(echo "$COLLS" | grep -v '^#SKIP')

OLDEST=$(echo "$COLLS" | cut -f2 | sort | head -1)
[[ -z "$OLDEST" || "$OLDEST" > "$CUTOFF" || "$OLDEST" == "$CUTOFF" ]] && { log "nothing older than $CUTOFF, done"; exit 0; }
log "oldest day=$OLDEST collections=$(echo "$COLLS" | wc -l)"

day="$OLDEST"; done_days=0
while [[ "$day" < "$CUTOFF" && $done_days -lt $MAX_DAYS ]]; do
  if (( $(date -u +%s) >= STOP_AT )); then log "STOP_AT reached, stop before $day"; break; fi
  next=$(date -u -d "$day +1 day" +%F)
  ymd=$(date -u -d "$day" +%Y%m%d)
  GTE=$(oid "$day"); LT=$(oid "$next")
  SRC="$WORK_DIR/$ymd"; rm -rf "$SRC"; mkdir -p "$SRC"
  log "day $day _id [$GTE, $LT)"
  declare -A EXPECTED=()

  while IFS=$'\t' read -r coll first; do
    [[ "$first" > "$day" ]] && continue                    # collection has nothing that old
    n=$(COLL="$coll" GTE="$GTE" LT="$LT" msh --eval '
      print(db.getCollection(process.env.COLL).countDocuments(
        {_id: {$gte: ObjectId(process.env.GTE), $lt: ObjectId(process.env.LT)}}, {maxTimeMS: '"$MAX_TIME_MS"'}))') \
      || { log "ERROR count $coll $day"; continue; }
    [[ "$n" =~ ^[0-9]+$ ]] || { log "ERROR count $coll $day: $n"; continue; }
    (( n == 0 )) && continue
    if [[ "$DRY_RUN" == 1 ]]; then log "dry $coll $day $n docs"; continue; fi
    safe="${coll//\//_}"                                     # '/' in name breaks the file path
    out="$SRC/${safe}_${ymd}.json"
    exp=$(mongoexport --uri="$MONGO_URI" --collection="$coll" \
          --query "{\"_id\":{\"\$gte\":{\"\$oid\":\"$GTE\"},\"\$lt\":{\"\$oid\":\"$LT\"}}}" \
          --out="$out" 2>&1); rc=$?
    echo "$exp" >> "$LOG_FILE"
    got=$(echo "$exp" | grep -oE 'exported [0-9]+ record' | grep -oE '[0-9]+' | tail -1)
    if [[ $rc -ne 0 || "$got" != "$n" ]]; then
      log "ERROR export $coll $day rc=$rc expected=$n got=${got:-?} -> not deleting this collection/day"
      rm -f "$out"; continue
    fi
    EXPECTED["$coll"]=$n
    log "exported $coll $day $n docs"
  done <<< "$COLLS"

  if [[ "$DRY_RUN" == 1 ]]; then rm -rf "$SRC"; day="$next"; done_days=$((done_days+1)); continue; fi
  if (( ${#EXPECTED[@]} == 0 )); then log "day $day empty"; rm -rf "$SRC"; day="$next"; continue; fi

  # Unique key per run: a day can be re-processed (a collection failed before) -> never overwrite an older zip
  # of the same day in S3 (v4 used ${ymd}.zip -> a re-run overwrote the full archive with a partial one).
  ZIP="$WORK_DIR/${ymd}_$(date -u +%Y%m%dT%H%M%S).zip"; rm -f "$ZIP"
  if ! zip -j "$ZIP" "$SRC"/*.json >> "$LOG_FILE" 2>&1 || [[ ! -s "$ZIP" ]]; then log "ERROR zip $day, abort"; exit 1; fi
  if ! aws s3 cp "$ZIP" "$S3_BUCKET/" --storage-class GLACIER_IR --region "$REGION" >> "$LOG_FILE" 2>&1; then
    log "ERROR upload $day, abort (nothing deleted)"; exit 1
  fi
  if ! aws s3 ls "$S3_BUCKET/$(basename "$ZIP")" --region "$REGION" >/dev/null 2>&1; then
    log "ERROR upload check $day: $(basename "$ZIP") not in bucket, abort (nothing deleted)"; exit 1
  fi
  log "uploaded $(basename "$ZIP") ($(du -h "$ZIP" | cut -f1))"

  for coll in "${!EXPECTED[@]}"; do
    res=$(COLL="$coll" GTE="$GTE" LT="$LT" msh --eval '
      const c = db.getCollection(process.env.COLL);
      const q = {_id: {$gte: ObjectId(process.env.GTE), $lt: ObjectId(process.env.LT)}};
      let total = 0;
      while (true) {
        const ids = c.find(q, {_id: 1}).sort({_id: 1}).limit('"$BATCH"').maxTimeMS('"$MAX_TIME_MS"').toArray().map(d => d._id);
        if (!ids.length) break;
        total += c.deleteMany({_id: {$in: ids}}).deletedCount;
        sleep('"$SLEEP_MS"');
      }
      print(total)') ; rc=$?
    log "deleted $coll $day ${res:-?}/${EXPECTED[$coll]} rc=$rc"
    [[ $rc -ne 0 || "${res:-}" != "${EXPECTED[$coll]}" ]] && log "WARN delete mismatch $coll $day (exported ${EXPECTED[$coll]}, deleted ${res:-?})"
  done

  rm -rf "$SRC" "$ZIP"
  log "completed $day"
  unset EXPECTED
  day="$next"; done_days=$((done_days+1))
done
log "end archive_v5 (days done=$done_days, next start=$day)"