From 891c4b9794fdc01b7f491db4fcdb3749348848e2 Mon Sep 17 00:00:00 2001
From: Uwe Schmitt <uwe.schmitt@id.ethz.ch>
Date: Mon, 9 Sep 2019 18:03:27 +0200
Subject: [PATCH] improved / adapted euler setup scripts
---
setup_eth_euler/.gitignore | 1 +
setup_eth_euler/prepare_notebooks.sh | 50 +++++++++++++++++++
.../setup_course_accounts_on_euler.sh | 13 +++--
.../start_jupyter_nb.sh | 2 +-
4 files changed, 61 insertions(+), 5 deletions(-)
create mode 100755 setup_eth_euler/prepare_notebooks.sh
diff --git a/setup_eth_euler/.gitignore b/setup_eth_euler/.gitignore
index e892217..1bb0b78 100644
--- a/setup_eth_euler/.gitignore
+++ b/setup_eth_euler/.gitignore
@@ -1,2 +1,3 @@
printout.pdf
**/.*.sw?
+build
diff --git a/setup_eth_euler/prepare_notebooks.sh b/setup_eth_euler/prepare_notebooks.sh
new file mode 100755
index 0000000..ba1d1a6
--- /dev/null
+++ b/setup_eth_euler/prepare_notebooks.sh
@@ -0,0 +1,50 @@
+#! /bin/sh
+#
+# create_installer.sh
+# Copyright (C) 2019 Uwe Schitt <uwe.schmitt@id.ethz.ch>
+#
+# Distributed under terms of the MIT license.
+#
+
+set -e
+
+# files to include for installation:
+NOTEBOOKS=??_*.ipynb
+
+LOCAL_BUILD_FOLDER=$(pwd)/build
+rm -rf ${LOCAL_BUILD_FOLDER}
+
+# create fresh folder structure to build archive with notebooks
+rm -rf ${LOCAL_BUILD_FOLDER}
+mkdir -p ${LOCAL_BUILD_FOLDER}/course
+mkdir -p ${LOCAL_BUILD_FOLDER}/solutions
+
+cp ../custom.html ${LOCAL_BUILD_FOLDER}/course
+cp ../custom.html ${LOCAL_BUILD_FOLDER}/solutions
+
+# cp -r ../data ${LOCAL_BUILD_FOLDER}/course
+# cp -r ../data ${LOCAL_BUILD_FOLDER}/solutions
+
+cp -r ../images ${LOCAL_BUILD_FOLDER}/course
+cp -r ../images ${LOCAL_BUILD_FOLDER}/solutions
+
+# create solution notebooks, copy originals and solutions
+# to build folder
+pushd .. >/dev/null
+
+for NOTEBOOK in ${NOTEBOOKS}; do
+ echo process ${NOTEBOOK}
+ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace ${NOTEBOOK}
+
+ TARGET=${LOCAL_BUILD_FOLDER}/course/${NOTEBOOK}
+ echo ${TARGET}
+ nb-filter-cells -i $NOTEBOOK -t solution | nb-filter-cells -t TODO -o ${TARGET}
+
+ TARGET=${LOCAL_BUILD_FOLDER}/solutions/${NOTEBOOK%.ipynb}-with-solutions.ipynb
+ nb-filter-cells -i $NOTEBOOK -t TODO -o ${TARGET}
+ # cp $NOTEBOOK ${TARGET}
+done
+
+popd >/dev/null
+echo
+
diff --git a/setup_eth_euler/setup_course_accounts_on_euler.sh b/setup_eth_euler/setup_course_accounts_on_euler.sh
index 8ec649f..b40f27f 100755
--- a/setup_eth_euler/setup_course_accounts_on_euler.sh
+++ b/setup_eth_euler/setup_course_accounts_on_euler.sh
@@ -8,6 +8,11 @@
EULER=euler.ethz.ch
+if test ! -d build; then
+ echo run ./prepare_notebooks.sh first
+ exit;
+fi
+
cat credentials.txt | while read -r USER PASSWORD
do
# USER=$(echo $line | cut -d" " -f 1)
@@ -15,8 +20,8 @@ do
echo $USER
echo
# https://stackoverflow.com/questions/13800225/
- ssh -n -i id_rsa $USER@$EULER "mkdir -p mlw"
- scp -i id_rsa ../??_*.ipynb ../custom.html $USER@$EULER:mlw
- scp -i id_rsa -r ../data ../images $USER@$EULER:mlw
- scp -i id_rsa ./to_deploy_on_euler/* ./upload_to_polybox_from_euler.sh $USER@$EULER:
+ # ssh -n -i id_rsa $USER@$EULER "rm -rf *; mkdir -p mlw"
+ scp -i id_rsa -r build/course $USER@$EULER:mlw/course
+ scp -i id_rsa -r build/solutions $USER@$EULER:mlw/solutions
+ scp -i id_rsa ./to_deploy_on_euler/* $USER@$EULER:
done
diff --git a/setup_eth_euler/to_deploy_on_course_computers/start_jupyter_nb.sh b/setup_eth_euler/to_deploy_on_course_computers/start_jupyter_nb.sh
index f780f8f..5fe549f 100755
--- a/setup_eth_euler/to_deploy_on_course_computers/start_jupyter_nb.sh
+++ b/setup_eth_euler/to_deploy_on_course_computers/start_jupyter_nb.sh
@@ -85,7 +85,7 @@ EULER_INSTALL_HONE="/cluster/scratch/schmittu/mlw/"
ssh -i id_rsa $USERNAME@$CHOSTNAME "echo source /etc/bashrc > .bashrc"
# ssh -i id_rsa $USERNAME@$CHOSTNAME bsub -n $NUM_CORES -W $RUN_TIME -R "rusage[mem=$MEM_PER_CORE]" bash -l <<ENDBSUB
-ssh -i id_rsa $USERNAME@$CHOSTNAME bsub -J mlw_jupyter -R light <<ENDBSUB
+ssh -i id_rsa $USERNAME@$CHOSTNAME bsub -J mlw_jupyter -n 8 -W 12:00 -R "rusage[mem=2000]" <<ENDBSUB
set -x
export XDG_RUNTIME_DIR=
IP_REMOTE="\$(hostname -i)"
--
GitLab