test_models.sh

#!/bin/bash
module load python

# Save the directory where the script was started
ORIG_DIR=$(pwd)

# Create a unique random folder in /dev/shm
TMPDIR=$(mktemp -d /dev/shm/llm_run_temp_XXXXXX)
WORKDIR="$TMPDIR/llm_for_analysis"

conda activate llm_env

# Get the root of the current Git repository
SRC_DIR=$(git rev-parse --show-toplevel)

# Copy files from the Git repo root, excluding .git, results/, and .snakemake/
rsync -av \
    --exclude='.git' \
    --exclude='results/' \
    --exclude='.snakemake/' \
    --exclude='test/' \
    "$SRC_DIR/" \
    "$WORKDIR/"

chmod +x "$WORKDIR/test_stats.sh"
cd "$WORKDIR"
mkdir -p results
# source ~/.apikeys.sh

MODEL_LIST="models.txt"
OUT_NAME="${1:-test}"   # Take from first argument, default to "test"

while IFS= read -r model; do
    # Use timestamp for unique run naming
    timestamp=$(date +"%Y%m%d_%H%M%S")
    MODEL_SAFE="${model//\//_}_$timestamp"
    export MODEL_NAME="$model"
    echo "Starting model [$timestamp]: $model"

    # Create config file for this run
    cat > config.yml <<EOF
model: '$model'
name: '$MODEL_SAFE'
out_dir: '$WORKDIR/results/$MODEL_SAFE'
EOF

    # Run the test script
    bash test_stats.sh

    # Save results for this model immediately
    DEST_DIR="$SRC_DIR/$OUT_NAME/"
    mkdir -p "$DEST_DIR"
    cp -r "$WORKDIR/results/$MODEL_SAFE" "$DEST_DIR/"
    mkdir -p "$DEST_DIR/$MODEL_SAFE/snakemake_log/"
    cp -r "$WORKDIR/.snakemake/log/"* "$DEST_DIR/$MODEL_SAFE/snakemake_log/" || true
    cp stats.csv "$DEST_DIR/$MODEL_SAFE/stats.csv" || true

done < "$MODEL_LIST"

# Return to the original directory
cd "$ORIG_DIR"