diff --git a/moderation_classifier/eval_BERT.py b/moderation_classifier/eval_BERT.py new file mode 100644 index 0000000000000000000000000000000000000000..242b874d36e5880b13a4146b339095bb662f8113 --- /dev/null +++ b/moderation_classifier/eval_BERT.py @@ -0,0 +1,19 @@ +from datasets import load_dataset +from evaluate import evaluator +from transformers import pipeline + +data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(10)) + +task_evaluator = evaluator("text-classification") + +pipe = pipeline("text-classification", model="../saved_models/20230630-103946/") + +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +import pdb; pdb.set_trace() +print(eval_results) + diff --git a/moderation_classifier/predict_BERT.py b/moderation_classifier/predict_BERT.py index f930809ba3cd3438543a02435b4a56c4940185ff..242b874d36e5880b13a4146b339095bb662f8113 100644 --- a/moderation_classifier/predict_BERT.py +++ b/moderation_classifier/predict_BERT.py @@ -1,6 +1,19 @@ +from datasets import load_dataset +from evaluate import evaluator from transformers import pipeline -text = "This was a masterpiece. Not completely faithful to the books, but enthralling from beginning to end. Might be my favorite of the three." +data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(10)) + +task_evaluator = evaluator("text-classification") + +pipe = pipeline("text-classification", model="../saved_models/20230630-103946/") + +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +import pdb; pdb.set_trace() +print(eval_results) -classifier = pipeline("sentiment-analysis", model="../saved_models/") -classifier(text)