Skip to content
Snippets Groups Projects
Commit 0f7bac13 authored by oschmanf's avatar oschmanf
Browse files

Update of bash scripts

parent a11e862f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 21 additions and 47 deletions
euler/train_model_cluster_germbert_alltopic_allhsprob.sh
+ 2
− 5
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,9 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "bert-base-german-cased"
--text_preprocessing
--train_bert data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'bert-base-german-cased' --text_preprocessing --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_germbert_alltopic_highhsprob.sh
+ 2
− 7
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,5 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "bert-base-german-cased"
--text_preprocessing
--hsprob '[0.7,1]'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'bert-base-german-cased' --text_preprocessing --hsprob '[0.7,1.0]' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_germbert_alltopic_lowhsprob.sh
+ 2
− 6
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "bert-base-german-cased"
--text_preprocessing
--hsprob '[0.0,0.3]'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'bert-base-german-cased' --text_preprocessing --hsprob '[0.0,0.3]' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_germbert_wissentopic_allhsprob.sh
+ 2
− 6
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "bert-base-german-cased"
--text_preprocessing
--topic 'Wissen'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'bert-base-german-cased' --text_preprocessing --topic 'Wissen' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_hsbert_alltopic_allhsprob.sh
+ 2
− 5
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,9 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
--text_preprocessing
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'deepset/bert-base-german-cased-hatespeech-GermEval18Coarse' --text_preprocessing --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_hsbert_alltopic_highhsprob.sh
+ 2
− 6
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
--text_preprocessing
--hsprob '[0.7,1]'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'deepset/bert-base-german-cased-hatespeech-GermEval18Coarse' --text_preprocessing --hsprob '[0.7,1.0]' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_hsbert_alltopic_lowhsprob.sh
+ 2
− 6
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
--text_preprocessing
--hsprob '[0.0,0.3]'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'deepset/bert-base-german-cased-hatespeech-GermEval18Coarse' --text_preprocessing --hsprob '[0.0,0.3]' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
euler/train_model_cluster_hsbert_wissentopic_allhsprob.sh
+ 2
− 6
View file @ 0f7bac13
!/bin/bash
#!/bin/bash
module load gcc/8.2.0 python_gpu/3.10.4 eth_proxy
source ../pp_env_tf_python310/bin/activate
......@@ -7,10 +7,6 @@ sbatch --mem-per-cpu=12g\
--gpus=1\
--gres=gpumem:12g\
--time=30:00:00\
--wrap "moderation_classifier --newspaper tagesanzeiger
--pretrained_model "deepset/bert-base-german-cased-hatespeech-GermEval18Coarse"
--text_preprocessing
--topic 'Wissen'
--train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
--wrap "moderation_classifier --newspaper tagesanzeiger --pretrained_model 'deepset/bert-base-german-cased-hatespeech-GermEval18Coarse' --text_preprocessing --topic 'Wissen' --train_bert ../data/tamedia_for_classifier_v4_preproc_train.csv"
moderation_classifier/train_BERT.py
+ 5
− 0
View file @ 0f7bac13
......@@ -47,6 +47,11 @@ def main(
Prepares tokenizer for mapping
"""
return tokenizer(examples["text"], truncation=True)
print(pretrained_model)
print(hsprob)
print(topic)
print(newspaper)
# Extract path
p = Path(input_data)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment