From 35189e2effde6f219cba3592f55aa162c5d04ee4 Mon Sep 17 00:00:00 2001
From: oschmanf <franziska.oschmann@id.ethz.ch>
Date: Fri, 30 Jun 2023 11:13:46 +0200
Subject: [PATCH] Add bash scripts for model training on cluster

---
 euler/prepare_data.sh         | 6 ++++++
 euler/prepare_data_cluster.sh | 6 ++++++
 euler/run_cluster.sh          | 7 +++++++
 euler/train_model_cluster.sh  | 6 ++++++
 4 files changed, 25 insertions(+)
 create mode 100755 euler/prepare_data.sh
 create mode 100755 euler/prepare_data_cluster.sh
 create mode 100755 euler/run_cluster.sh
 create mode 100755 euler/train_model_cluster.sh

diff --git a/euler/prepare_data.sh b/euler/prepare_data.sh
new file mode 100755
index 0000000..3fab9c0
--- /dev/null
+++ b/euler/prepare_data.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
+source ../pp_env_tf_python310/bin/activate
+
+sbatch --mem-per-cpu=4g --time=6:00:00 --wrap "moderation_classifier --prepare_data ../data/tamedia_for_classifier_v3.csv"
diff --git a/euler/prepare_data_cluster.sh b/euler/prepare_data_cluster.sh
new file mode 100755
index 0000000..3fab9c0
--- /dev/null
+++ b/euler/prepare_data_cluster.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
+source ../pp_env_tf_python310/bin/activate
+
+sbatch --mem-per-cpu=4g --time=6:00:00 --wrap "moderation_classifier --prepare_data ../data/tamedia_for_classifier_v3.csv"
diff --git a/euler/run_cluster.sh b/euler/run_cluster.sh
new file mode 100755
index 0000000..9592c73
--- /dev/null
+++ b/euler/run_cluster.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
+source ../pp_env_tf_python310/bin/activate
+
+sbatch --mem-per-cpu=4g --wrap "moderation_classifier --split ../data/tamedia_for_classifier_v3.csv"
+sbatch --mem-per-cpu=8g --gpus=1 --gres=gpumem:12g --time=30:00:00 --wrap "moderation_classifier --text_preprocessing --train_bert ../data/tamedia_for_classifier_v3_train.csv"
diff --git a/euler/train_model_cluster.sh b/euler/train_model_cluster.sh
new file mode 100755
index 0000000..c4c13f3
--- /dev/null
+++ b/euler/train_model_cluster.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
+source ../pp_env_tf_python310/bin/activate
+
+sbatch --mem-per-cpu=12g --gpus=1 --gres=gpumem:12g --time=30:00:00 --wrap "moderation_classifier --train_bert ../data/tamedia_for_classifier_v3_preproc_train.csv"
-- 
GitLab