added all

.gitattributes

@@ -0,0 +1,4 @@
+*.csv filter=lfs diff=lfs merge=lfs -text
+*bin filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+keys.py 

data/AI_checker_remade.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdba3d2700937b2fe571b5903a76d3c509386fef4b91ae86fe54fae0a3cdd919
+size 301323861

data/new_data_with_embeddings.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c437f101ec9aec5244cd6bd431f486ff8e6b9ecf66db25a915f59f970cdb6fbd
+size 305301653

data/original_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cfc33da6a7155466097cbe949c4c50b15b8cb25ac115e801fbe1ef7eb3ac61e
+size 443537732

models/tokenizer_v1/added_tokens.json

@@ -0,0 +1,7 @@
+{
+ "[CLS]": 101,
+ "[MASK]": 103,
+ "[PAD]": 0,
+ "[SEP]": 102,
+ "[UNK]": 100
+}

models/tokenizer_v1/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+ "cls_token": "[CLS]",
+ "mask_token": "[MASK]",
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "unk_token": "[UNK]"
+}

models/tokenizer_v1/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+ "added_tokens_decoder": {
+ "0": {
+ "content": "[PAD]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "100": {
+ "content": "[UNK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "101": {
+ "content": "[CLS]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "102": {
+ "content": "[SEP]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "103": {
+ "content": "[MASK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [],
+ "clean_up_tokenization_spaces": true,
+ "cls_token": "[CLS]",
+ "do_lower_case": true,
+ "mask_token": "[MASK]",
+ "model_max_length": 512,
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "strip_accents": null,
+ "tokenize_chinese_chars": true,
+ "tokenizer_class": "DistilBertTokenizer",
+ "unk_token": "[UNK]"
+}

models/trained_model_v1/config.json

@@ -0,0 +1,33 @@
+{
+ "_name_or_path": "distilbert-base-uncased",
+ "activation": "gelu",
+ "architectures": [
+ "DistilBertForSequenceClassification"
+ ],
+ "attention_dropout": 0.1,
+ "dim": 768,
+ "dropout": 0.1,
+ "hidden_dim": 3072,
+ "id2label": {
+ "0": "NEGATIVE",
+ "1": "POSITIVE"
+ },
+ "initializer_range": 0.02,
+ "label2id": {
+ "NEGATIVE": 0,
+ "POSITIVE": 1
+ },
+ "max_position_embeddings": 512,
+ "model_type": "distilbert",
+ "n_heads": 12,
+ "n_layers": 6,
+ "pad_token_id": 0,
+ "problem_type": "single_label_classification",
+ "qa_dropout": 0.1,
+ "seq_classif_dropout": 0.2,
+ "sinusoidal_pos_embds": false,
+ "tie_weights_": true,
+ "torch_dtype": "float32",
+ "transformers_version": "4.34.0",
+ "vocab_size": 30522
+}

models/trained_model_v1/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502361f57902b8792424d4c98cf37fc9dfd6a2ea41b2bf3d41346eb1ff0b658b
+size 267855978

models/trained_model_v11/added_tokens.json

@@ -0,0 +1,7 @@
+{
+ "[CLS]": 101,
+ "[MASK]": 103,
+ "[PAD]": 0,
+ "[SEP]": 102,
+ "[UNK]": 100
+}

models/trained_model_v11/config.json

@@ -0,0 +1,33 @@
+{
+ "_name_or_path": "distilbert-base-uncased",
+ "activation": "gelu",
+ "architectures": [
+ "DistilBertForSequenceClassification"
+ ],
+ "attention_dropout": 0.1,
+ "dim": 768,
+ "dropout": 0.1,
+ "hidden_dim": 3072,
+ "id2label": {
+ "0": "NEGATIVE",
+ "1": "POSITIVE"
+ },
+ "initializer_range": 0.02,
+ "label2id": {
+ "NEGATIVE": 0,
+ "POSITIVE": 1
+ },
+ "max_position_embeddings": 512,
+ "model_type": "distilbert",
+ "n_heads": 12,
+ "n_layers": 6,
+ "pad_token_id": 0,
+ "problem_type": "single_label_classification",
+ "qa_dropout": 0.1,
+ "seq_classif_dropout": 0.2,
+ "sinusoidal_pos_embds": false,
+ "tie_weights_": true,
+ "torch_dtype": "float32",
+ "transformers_version": "4.34.0",
+ "vocab_size": 30522
+}

models/trained_model_v11/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502361f57902b8792424d4c98cf37fc9dfd6a2ea41b2bf3d41346eb1ff0b658b
+size 267855978

models/trained_model_v11/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+ "cls_token": "[CLS]",
+ "mask_token": "[MASK]",
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "unk_token": "[UNK]"
+}

models/trained_model_v11/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+ "added_tokens_decoder": {
+ "0": {
+ "content": "[PAD]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "100": {
+ "content": "[UNK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "101": {
+ "content": "[CLS]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "102": {
+ "content": "[SEP]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "103": {
+ "content": "[MASK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [],
+ "clean_up_tokenization_spaces": true,
+ "cls_token": "[CLS]",
+ "do_lower_case": true,
+ "mask_token": "[MASK]",
+ "model_max_length": 512,
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "strip_accents": null,
+ "tokenize_chinese_chars": true,
+ "tokenizer_class": "DistilBertTokenizer",
+ "unk_token": "[UNK]"
+}

my_awesome_model/checkpoint-15000/added_tokens.json

@@ -0,0 +1,7 @@
+{
+ "[CLS]": 101,
+ "[MASK]": 103,
+ "[PAD]": 0,
+ "[SEP]": 102,
+ "[UNK]": 100
+}

my_awesome_model/checkpoint-15000/config.json

@@ -0,0 +1,33 @@
+{
+ "_name_or_path": "distilbert-base-uncased",
+ "activation": "gelu",
+ "architectures": [
+ "DistilBertForSequenceClassification"
+ ],
+ "attention_dropout": 0.1,
+ "dim": 768,
+ "dropout": 0.1,
+ "hidden_dim": 3072,
+ "id2label": {
+ "0": "NEGATIVE",
+ "1": "POSITIVE"
+ },
+ "initializer_range": 0.02,
+ "label2id": {
+ "NEGATIVE": 0,
+ "POSITIVE": 1
+ },
+ "max_position_embeddings": 512,
+ "model_type": "distilbert",
+ "n_heads": 12,
+ "n_layers": 6,
+ "pad_token_id": 0,
+ "problem_type": "single_label_classification",
+ "qa_dropout": 0.1,
+ "seq_classif_dropout": 0.2,
+ "sinusoidal_pos_embds": false,
+ "tie_weights_": true,
+ "torch_dtype": "float32",
+ "transformers_version": "4.34.0",
+ "vocab_size": 30522
+}

my_awesome_model/checkpoint-15000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a469da14025142fa0c4f7f87a31bab3d92a4d6153b096d099432e5ec32a1af3d
+size 535727290

my_awesome_model/checkpoint-15000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf74e93c7d98e4d97620113e2ff7b00ff69f3d4ab048b2fbe444e1f4ca08d3c
+size 267855978

my_awesome_model/checkpoint-15000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0791ab28ed85fc2b7bf19fb7882ce2711b9dea36e12032576612122561cfd5f1
+size 1064

my_awesome_model/checkpoint-15000/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+ "cls_token": "[CLS]",
+ "mask_token": "[MASK]",
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "unk_token": "[UNK]"
+}

my_awesome_model/checkpoint-15000/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+ "added_tokens_decoder": {
+ "0": {
+ "content": "[PAD]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "100": {
+ "content": "[UNK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "101": {
+ "content": "[CLS]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "102": {
+ "content": "[SEP]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "103": {
+ "content": "[MASK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [],
+ "clean_up_tokenization_spaces": true,
+ "cls_token": "[CLS]",
+ "do_lower_case": true,
+ "mask_token": "[MASK]",
+ "model_max_length": 512,
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "strip_accents": null,
+ "tokenize_chinese_chars": true,
+ "tokenizer_class": "DistilBertTokenizer",
+ "unk_token": "[UNK]"
+}

my_awesome_model/checkpoint-15000/trainer_state.json

@@ -0,0 +1,208 @@
+{
+ "best_metric": 0.07496260851621628,
+ "best_model_checkpoint": "my_awesome_model/checkpoint-15000",
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 15000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.03,
+ "learning_rate": 1.9666666666666666e-05,
+ "loss": 0.1446,
+ "step": 500
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 1.9333333333333333e-05,
+ "loss": 0.0898,
+ "step": 1000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 1.9e-05,
+ "loss": 0.0545,
+ "step": 1500
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 1.866666666666667e-05,
+ "loss": 0.0487,
+ "step": 2000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 1.8333333333333333e-05,
+ "loss": 0.0444,
+ "step": 2500
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 1.8e-05,
+ "loss": 0.0443,
+ "step": 3000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 1.7666666666666668e-05,
+ "loss": 0.0439,
+ "step": 3500
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 1.7333333333333336e-05,
+ "loss": 0.0291,
+ "step": 4000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 1.7e-05,
+ "loss": 0.0342,
+ "step": 4500
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 1.6666666666666667e-05,
+ "loss": 0.0332,
+ "step": 5000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 1.6333333333333335e-05,
+ "loss": 0.0345,
+ "step": 5500
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 1.6000000000000003e-05,
+ "loss": 0.0232,
+ "step": 6000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 1.5666666666666667e-05,
+ "loss": 0.0266,
+ "step": 6500
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 1.5333333333333334e-05,
+ "loss": 0.0276,
+ "step": 7000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 1.5000000000000002e-05,
+ "loss": 0.0303,
+ "step": 7500
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 1.4666666666666666e-05,
+ "loss": 0.0254,
+ "step": 8000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 1.4333333333333334e-05,
+ "loss": 0.0214,
+ "step": 8500
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 1.4e-05,
+ "loss": 0.0206,
+ "step": 9000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 1.3666666666666667e-05,
+ "loss": 0.0258,
+ "step": 9500
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 1.3333333333333333e-05,
+ "loss": 0.0179,
+ "step": 10000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 1.3000000000000001e-05,
+ "loss": 0.0184,
+ "step": 10500
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 1.2666666666666667e-05,
+ "loss": 0.0221,
+ "step": 11000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 1.2333333333333334e-05,
+ "loss": 0.0187,
+ "step": 11500
+ },
+ {
+ "epoch": 0.8,
+ "learning_rate": 1.2e-05,
+ "loss": 0.0174,
+ "step": 12000
+ },
+ {
+ "epoch": 0.83,
+ "learning_rate": 1.1666666666666668e-05,
+ "loss": 0.0181,
+ "step": 12500
+ },
+ {
+ "epoch": 0.87,
+ "learning_rate": 1.1333333333333334e-05,
+ "loss": 0.0176,
+ "step": 13000
+ },
+ {
+ "epoch": 0.9,
+ "learning_rate": 1.1000000000000001e-05,
+ "loss": 0.0171,
+ "step": 13500
+ },
+ {
+ "epoch": 0.93,
+ "learning_rate": 1.0666666666666667e-05,
+ "loss": 0.0149,
+ "step": 14000
+ },
+ {
+ "epoch": 0.97,
+ "learning_rate": 1.0333333333333335e-05,
+ "loss": 0.0136,
+ "step": 14500
+ },
+ {
+ "epoch": 1.0,
+ "learning_rate": 1e-05,
+ "loss": 0.0178,
+ "step": 15000
+ },
+ {
+ "epoch": 1.0,
+ "eval_accuracy": 0.9843833333333334,
+ "eval_loss": 0.07496260851621628,
+ "eval_runtime": 200.0448,
+ "eval_samples_per_second": 299.933,
+ "eval_steps_per_second": 18.746,
+ "step": 15000
+ }
+ ],
+ "logging_steps": 500,
+ "max_steps": 30000,
+ "num_train_epochs": 2,
+ "save_steps": 500,
+ "total_flos": 2.27844960589872e+16,
+ "trial_name": null,
+ "trial_params": null
+}

my_awesome_model/checkpoint-15000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c46d094ebc25b6e4ba42d7d833d3a46a5a4a4755e4e22404dd4492caa24ef3
+size 4536

my_awesome_model/checkpoint-30000/added_tokens.json

@@ -0,0 +1,7 @@
+{
+ "[CLS]": 101,
+ "[MASK]": 103,
+ "[PAD]": 0,
+ "[SEP]": 102,
+ "[UNK]": 100
+}

my_awesome_model/checkpoint-30000/config.json

@@ -0,0 +1,33 @@
+{
+ "_name_or_path": "distilbert-base-uncased",
+ "activation": "gelu",
+ "architectures": [
+ "DistilBertForSequenceClassification"
+ ],
+ "attention_dropout": 0.1,
+ "dim": 768,
+ "dropout": 0.1,
+ "hidden_dim": 3072,
+ "id2label": {
+ "0": "NEGATIVE",
+ "1": "POSITIVE"
+ },
+ "initializer_range": 0.02,
+ "label2id": {
+ "NEGATIVE": 0,
+ "POSITIVE": 1
+ },
+ "max_position_embeddings": 512,
+ "model_type": "distilbert",
+ "n_heads": 12,
+ "n_layers": 6,
+ "pad_token_id": 0,
+ "problem_type": "single_label_classification",
+ "qa_dropout": 0.1,
+ "seq_classif_dropout": 0.2,
+ "sinusoidal_pos_embds": false,
+ "tie_weights_": true,
+ "torch_dtype": "float32",
+ "transformers_version": "4.34.0",
+ "vocab_size": 30522
+}

my_awesome_model/checkpoint-30000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5ff325f5bc38ed6bb344bb939c32826b951890c550a38a3ab6ab2f77d1e5836
+size 535727290

my_awesome_model/checkpoint-30000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502361f57902b8792424d4c98cf37fc9dfd6a2ea41b2bf3d41346eb1ff0b658b
+size 267855978

my_awesome_model/checkpoint-30000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b36093e06845c6146f3175c64f0e8bdb441d4f7fc67a6962ed0b80b6725daf1
+size 1064

my_awesome_model/checkpoint-30000/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+ "cls_token": "[CLS]",
+ "mask_token": "[MASK]",
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "unk_token": "[UNK]"
+}

my_awesome_model/checkpoint-30000/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+ "added_tokens_decoder": {
+ "0": {
+ "content": "[PAD]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "100": {
+ "content": "[UNK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "101": {
+ "content": "[CLS]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "102": {
+ "content": "[SEP]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "103": {
+ "content": "[MASK]",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [],
+ "clean_up_tokenization_spaces": true,
+ "cls_token": "[CLS]",
+ "do_lower_case": true,
+ "mask_token": "[MASK]",
+ "model_max_length": 512,
+ "pad_token": "[PAD]",
+ "sep_token": "[SEP]",
+ "strip_accents": null,
+ "tokenize_chinese_chars": true,
+ "tokenizer_class": "DistilBertTokenizer",
+ "unk_token": "[UNK]"
+}

my_awesome_model/checkpoint-30000/trainer_state.json

@@ -0,0 +1,397 @@
+{
+ "best_metric": 0.03921842947602272,
+ "best_model_checkpoint": "my_awesome_model/checkpoint-30000",
+ "epoch": 2.0,
+ "eval_steps": 500,
+ "global_step": 30000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.03,
+ "learning_rate": 1.9666666666666666e-05,
+ "loss": 0.1446,
+ "step": 500
+ },
+ {
+ "epoch": 0.07,
+ "learning_rate": 1.9333333333333333e-05,
+ "loss": 0.0898,
+ "step": 1000
+ },
+ {
+ "epoch": 0.1,
+ "learning_rate": 1.9e-05,
+ "loss": 0.0545,
+ "step": 1500
+ },
+ {
+ "epoch": 0.13,
+ "learning_rate": 1.866666666666667e-05,
+ "loss": 0.0487,
+ "step": 2000
+ },
+ {
+ "epoch": 0.17,
+ "learning_rate": 1.8333333333333333e-05,
+ "loss": 0.0444,
+ "step": 2500
+ },
+ {
+ "epoch": 0.2,
+ "learning_rate": 1.8e-05,
+ "loss": 0.0443,
+ "step": 3000
+ },
+ {
+ "epoch": 0.23,
+ "learning_rate": 1.7666666666666668e-05,
+ "loss": 0.0439,
+ "step": 3500
+ },
+ {
+ "epoch": 0.27,
+ "learning_rate": 1.7333333333333336e-05,
+ "loss": 0.0291,
+ "step": 4000
+ },
+ {
+ "epoch": 0.3,
+ "learning_rate": 1.7e-05,
+ "loss": 0.0342,
+ "step": 4500
+ },
+ {
+ "epoch": 0.33,
+ "learning_rate": 1.6666666666666667e-05,
+ "loss": 0.0332,
+ "step": 5000
+ },
+ {
+ "epoch": 0.37,
+ "learning_rate": 1.6333333333333335e-05,
+ "loss": 0.0345,
+ "step": 5500
+ },
+ {
+ "epoch": 0.4,
+ "learning_rate": 1.6000000000000003e-05,
+ "loss": 0.0232,
+ "step": 6000
+ },
+ {
+ "epoch": 0.43,
+ "learning_rate": 1.5666666666666667e-05,
+ "loss": 0.0266,
+ "step": 6500
+ },
+ {
+ "epoch": 0.47,
+ "learning_rate": 1.5333333333333334e-05,
+ "loss": 0.0276,
+ "step": 7000
+ },
+ {
+ "epoch": 0.5,
+ "learning_rate": 1.5000000000000002e-05,
+ "loss": 0.0303,
+ "step": 7500
+ },
+ {
+ "epoch": 0.53,
+ "learning_rate": 1.4666666666666666e-05,
+ "loss": 0.0254,
+ "step": 8000
+ },
+ {
+ "epoch": 0.57,
+ "learning_rate": 1.4333333333333334e-05,
+ "loss": 0.0214,
+ "step": 8500
+ },
+ {
+ "epoch": 0.6,
+ "learning_rate": 1.4e-05,
+ "loss": 0.0206,
+ "step": 9000
+ },
+ {
+ "epoch": 0.63,
+ "learning_rate": 1.3666666666666667e-05,
+ "loss": 0.0258,
+ "step": 9500
+ },
+ {
+ "epoch": 0.67,
+ "learning_rate": 1.3333333333333333e-05,
+ "loss": 0.0179,
+ "step": 10000
+ },
+ {
+ "epoch": 0.7,
+ "learning_rate": 1.3000000000000001e-05,
+ "loss": 0.0184,
+ "step": 10500
+ },
+ {
+ "epoch": 0.73,
+ "learning_rate": 1.2666666666666667e-05,
+ "loss": 0.0221,
+ "step": 11000
+ },
+ {
+ "epoch": 0.77,
+ "learning_rate": 1.2333333333333334e-05,
+ "loss": 0.0187,
+ "step": 11500
+ },
+ {
+ "epoch": 0.8,
+ "learning_rate": 1.2e-05,
+ "loss": 0.0174,
+ "step": 12000
+ },
+ {
+ "epoch": 0.83,
+ "learning_rate": 1.1666666666666668e-05,
+ "loss": 0.0181,
+ "step": 12500
+ },
+ {
+ "epoch": 0.87,
+ "learning_rate": 1.1333333333333334e-05,
+ "loss": 0.0176,
+ "step": 13000
+ },
+ {
+ "epoch": 0.9,
+ "learning_rate": 1.1000000000000001e-05,
+ "loss": 0.0171,
+ "step": 13500
+ },
+ {
+ "epoch": 0.93,
+ "learning_rate": 1.0666666666666667e-05,
+ "loss": 0.0149,
+ "step": 14000
+ },
+ {
+ "epoch": 0.97,
+ "learning_rate": 1.0333333333333335e-05,
+ "loss": 0.0136,
+ "step": 14500
+ },
+ {
+ "epoch": 1.0,
+ "learning_rate": 1e-05,
+ "loss": 0.0178,
+ "step": 15000
+ },
+ {
+ "epoch": 1.0,
+ "eval_accuracy": 0.9843833333333334,
+ "eval_loss": 0.07496260851621628,
+ "eval_runtime": 200.0448,
+ "eval_samples_per_second": 299.933,
+ "eval_steps_per_second": 18.746,
+ "step": 15000
+ },
+ {
+ "epoch": 1.03,
+ "learning_rate": 9.666666666666667e-06,
+ "loss": 0.0039,
+ "step": 15500
+ },
+ {
+ "epoch": 1.07,
+ "learning_rate": 9.333333333333334e-06,
+ "loss": 0.0061,
+ "step": 16000
+ },
+ {
+ "epoch": 1.1,
+ "learning_rate": 9e-06,
+ "loss": 0.0114,
+ "step": 16500
+ },
+ {
+ "epoch": 1.13,
+ "learning_rate": 8.666666666666668e-06,
+ "loss": 0.0039,
+ "step": 17000
+ },
+ {
+ "epoch": 1.17,
+ "learning_rate": 8.333333333333334e-06,
+ "loss": 0.0075,
+ "step": 17500
+ },
+ {
+ "epoch": 1.2,
+ "learning_rate": 8.000000000000001e-06,
+ "loss": 0.0037,
+ "step": 18000
+ },
+ {
+ "epoch": 1.23,
+ "learning_rate": 7.666666666666667e-06,
+ "loss": 0.0105,
+ "step": 18500
+ },
+ {
+ "epoch": 1.27,
+ "learning_rate": 7.333333333333333e-06,
+ "loss": 0.0058,
+ "step": 19000
+ },
+ {
+ "epoch": 1.3,
+ "learning_rate": 7e-06,
+ "loss": 0.0036,
+ "step": 19500
+ },
+ {
+ "epoch": 1.33,
+ "learning_rate": 6.666666666666667e-06,
+ "loss": 0.0079,
+ "step": 20000
+ },
+ {
+ "epoch": 1.37,
+ "learning_rate": 6.333333333333333e-06,
+ "loss": 0.0091,
+ "step": 20500
+ },
+ {
+ "epoch": 1.4,
+ "learning_rate": 6e-06,
+ "loss": 0.0054,
+ "step": 21000
+ },
+ {
+ "epoch": 1.43,
+ "learning_rate": 5.666666666666667e-06,
+ "loss": 0.0013,
+ "step": 21500
+ },
+ {
+ "epoch": 1.47,
+ "learning_rate": 5.333333333333334e-06,
+ "loss": 0.0104,
+ "step": 22000
+ },
+ {
+ "epoch": 1.5,
+ "learning_rate": 5e-06,
+ "loss": 0.0037,
+ "step": 22500
+ },
+ {
+ "epoch": 1.53,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 0.0044,
+ "step": 23000
+ },
+ {
+ "epoch": 1.57,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 0.0047,
+ "step": 23500
+ },
+ {
+ "epoch": 1.6,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 0.0035,
+ "step": 24000
+ },
+ {
+ "epoch": 1.63,
+ "learning_rate": 3.6666666666666666e-06,
+ "loss": 0.0026,
+ "step": 24500
+ },
+ {
+ "epoch": 1.67,
+ "learning_rate": 3.3333333333333333e-06,
+ "loss": 0.0033,
+ "step": 25000
+ },
+ {
+ "epoch": 1.7,
+ "learning_rate": 3e-06,
+ "loss": 0.0027,
+ "step": 25500
+ },
+ {
+ "epoch": 1.73,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 0.002,
+ "step": 26000
+ },
+ {
+ "epoch": 1.77,
+ "learning_rate": 2.3333333333333336e-06,
+ "loss": 0.001,
+ "step": 26500
+ },
+ {
+ "epoch": 1.8,
+ "learning_rate": 2.0000000000000003e-06,
+ "loss": 0.0014,
+ "step": 27000
+ },
+ {
+ "epoch": 1.83,
+ "learning_rate": 1.6666666666666667e-06,
+ "loss": 0.004,
+ "step": 27500
+ },
+ {
+ "epoch": 1.87,
+ "learning_rate": 1.3333333333333334e-06,
+ "loss": 0.0016,
+ "step": 28000
+ },
+ {
+ "epoch": 1.9,
+ "learning_rate": 1.0000000000000002e-06,
+ "loss": 0.0027,
+ "step": 28500
+ },
+ {
+ "epoch": 1.93,
+ "learning_rate": 6.666666666666667e-07,
+ "loss": 0.0026,
+ "step": 29000
+ },
+ {
+ "epoch": 1.97,
+ "learning_rate": 3.3333333333333335e-07,
+ "loss": 0.0007,
+ "step": 29500
+ },
+ {
+ "epoch": 2.0,
+ "learning_rate": 0.0,
+ "loss": 0.0016,
+ "step": 30000
+ },
+ {
+ "epoch": 2.0,
+ "eval_accuracy": 0.9938666666666667,
+ "eval_loss": 0.03921842947602272,
+ "eval_runtime": 201.1467,
+ "eval_samples_per_second": 298.29,
+ "eval_steps_per_second": 18.643,
+ "step": 30000
+ }
+ ],
+ "logging_steps": 500,
+ "max_steps": 30000,
+ "num_train_epochs": 2,
+ "save_steps": 500,
+ "total_flos": 4.559380077797894e+16,
+ "trial_name": null,
+ "trial_params": null
+}

my_awesome_model/checkpoint-30000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c46d094ebc25b6e4ba42d7d833d3a46a5a4a4755e4e22404dd4492caa24ef3
+size 4536

requirements.txt

@@ -0,0 +1,11 @@
+transformers
+requests
+datasets
+torch
+pandas
+numpy
+tensorflow 
+tensorflow_hub
+tensorflow_text
+scikit-learn
+evaluate

research/01_embeddings_Similarity.ipynb

@@ -0,0 +1,1249 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os; os.chdir('..')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from keys import get_similarity_against"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "main_query= \"water intoxication\"\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sentence1= '''Water intoxication, also known as water poisoning, hyperhydration, overhydration, or water toxemia, is a potentially fatal disturbance in brain functions that results when the normal balance of electrolytes in the body is pushed outside safe limits by excessive water intake.\n",
+ "\n",
+ "Under normal circumstances, accidentally consuming too much water is exceptionally rare. Nearly all deaths related to water intoxication in normal individuals have resulted either from water-drinking contests, in which individuals attempt to consume large amounts of water, or from long bouts of exercise during which excessive amounts of fluid were consumed.[1] In addition, water cure, a method of torture in which the victim is forced to consume excessive amounts of water, can cause water intoxication.[1]\n",
+ "\n",
+ "Water, like any other substance, can be considered a poison when over-consumed in a brief period of time. Water intoxication mostly occurs when water is being consumed in a high quantity without adequate electrolyte intake.[2]\n",
+ "\n",
+ "Excess of body water may also be a result of a medical condition or improper treatment; see \"hyponatremia\" for some examples. Water is considered one of the least toxic chemical compounds, with an LD50 exceeding 90 ml/kg in rats;[3] drinking six liters in three hours has caused the death of a human.[4]'''\n",
+ "\n",
+ "\n",
+ "sentence2= '''Hyponatremia, colloquially termed aqua inebriation, or aqueous toxemia, represents a perilous derangement of cerebral functions. It ensues when the delicate equilibrium of bodily electrolytes is jolted beyond secure thresholds by an extravagant indulgence in aqueous libations.\n",
+ "\n",
+ "In ordinary circumstances, inadvertent indulgence in excessive aqueous elixir is exceedingly exceptional. Virtually all instances of aqueous inebriation-related demises in average individuals have stemmed either from aquatic imbiber duels, wherein contenders vie to imbibe copious volumes of water, or from prolonged stints of exertion accompanied by the immoderate ingestion of fluids. Additionally, aqua torment, a tormenting methodology in which the sufferer is coerced into partaking of profuse quantities of water, can precipitate aqueous inebriation.\n",
+ "\n",
+ "H2O, akin to any other substance, may be deemed venomous when extravagantly consumed within a concise temporal interval. Aqua intoxication predominantly manifests itself when an exorbitant quantum of water is ingested without commensurate electrolytic supplementation.\n",
+ "\n",
+ "A superabundance of corporeal aqueous content might also emerge as an outcome of a medical ailment or inept therapy; for illustrative instances, refer to \"hyponatremia.\" Water is reckoned as one of the least virulent chemical compounds, boasting an LD50 that surpasses 90 ml/kg in rodents. Consuming six liters in a mere three hours has led to the demise of a human.'''"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'Water intoxication, also known as water poisoning, hyperhydration, overhydration, or water toxemia, is a potentially fatal disturbance in brain functions that results when the normal balance of electrolytes in the body is pushed outside safe limits by excessive water intake.\\n\\nUnder normal circumstances, accidentally consuming too much water is exceptionally rare. Nearly all deaths related to water intoxication in normal individuals have resulted either from water-drinking contests, in which individuals attempt to consume large amounts of water, or from long bouts of exercise during which excessive amounts of fluid were consumed.[1] In addition, water cure, a method of torture in which the victim is forced to consume excessive amounts of water, can cause water intoxication.[1]\\n\\nWater, like any other substance, can be considered a poison when over-consumed in a brief period of time. Water intoxication mostly occurs when water is being consumed in a high quantity without adequate electrolyte intake.[2]\\n\\nExcess of body water may also be a result of a medical condition or improper treatment; see \"hyponatremia\" for some examples. Water is considered one of the least toxic chemical compounds, with an LD50 exceeding 90 ml/kg in rats;[3] drinking six liters in three hours has caused the death of a human.[4]'"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "sentence1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'Hyponatremia, colloquially termed aqua inebriation, or aqueous toxemia, represents a perilous derangement of cerebral functions. It ensues when the delicate equilibrium of bodily electrolytes is jolted beyond secure thresholds by an extravagant indulgence in aqueous libations.\\n\\nIn ordinary circumstances, inadvertent indulgence in excessive aqueous elixir is exceedingly exceptional. Virtually all instances of aqueous inebriation-related demises in average individuals have stemmed either from aquatic imbiber duels, wherein contenders vie to imbibe copious volumes of water, or from prolonged stints of exertion accompanied by the immoderate ingestion of fluids. Additionally, aqua torment, a tormenting methodology in which the sufferer is coerced into partaking of profuse quantities of water, can precipitate aqueous inebriation.\\n\\nH2O, akin to any other substance, may be deemed venomous when extravagantly consumed within a concise temporal interval. Aqua intoxication predominantly manifests itself when an exorbitant quantum of water is ingested without commensurate electrolytic supplementation.\\n\\nA superabundance of corporeal aqueous content might also emerge as an outcome of a medical ailment or inept therapy; for illustrative instances, refer to \"hyponatremia.\" Water is reckoned as one of the least virulent chemical compounds, boasting an LD50 that surpasses 90 ml/kg in rodents. Consuming six liters in a mere three hours has led to the demise of a human.'"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "sentence2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Method 1: `Direct Similarity of Embeddings`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "import json"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response= requests.post(url= \"https://embeddings.paperbot.ai/get-similarity-against\",\n",
+ " json={\n",
+ " \"main_entity\": main_query, \n",
+ " \"compare_with\": [sentence1, sentence2]\n",
+ " })"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[0.94, 0.9]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "json.loads(response.content.decode(\"utf-8\"))['similarity']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Method 1: `BERT Question-Answering`\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline\n",
+ "\n",
+ "model_name = \"deepset/roberta-base-squad2\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/ubuntu/SentenceStructureComparision/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "\n",
+ "# a) Get predictions\n",
+ "nlp = pipeline('question-answering', model=model_name, tokenizer=model_name)\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'score': 0.001219886471517384,\n",
+ " 'start': 252,\n",
+ " 'end': 274,\n",
+ " 'answer': 'excessive water intake'}"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "QA_input = {\n",
+ " 'question': main_query,\n",
+ " 'context': sentence1\n",
+ "}\n",
+ "res = nlp(QA_input)\n",
+ "\n",
+ "res"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'score': 2.8929189284099266e-05,\n",
+ " 'start': 958,\n",
+ " 'end': 975,\n",
+ " 'answer': 'Aqua intoxication'}"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "QA_input = {\n",
+ " 'question': main_query,\n",
+ " 'context': sentence2\n",
+ "}\n",
+ "res = nlp(QA_input)\n",
+ "\n",
+ "res"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Perplexity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at deepset/roberta-base-squad2 and are newly initialized: ['lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.bias', 'lm_head.bias']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "from transformers import AutoTokenizer, AutoModelForMaskedLM\n",
+ "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+ "model = AutoModelForMaskedLM.from_pretrained(model_name).to(\"cuda\")\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "RobertaConfig {\n",
+ " \"_name_or_path\": \"deepset/roberta-base-squad2\",\n",
+ " \"architectures\": [\n",
+ " \"RobertaForQuestionAnswering\"\n",
+ " ],\n",
+ " \"attention_probs_dropout_prob\": 0.1,\n",
+ " \"bos_token_id\": 0,\n",
+ " \"classifier_dropout\": null,\n",
+ " \"eos_token_id\": 2,\n",
+ " \"gradient_checkpointing\": false,\n",
+ " \"hidden_act\": \"gelu\",\n",
+ " \"hidden_dropout_prob\": 0.1,\n",
+ " \"hidden_size\": 768,\n",
+ " \"initializer_range\": 0.02,\n",
+ " \"intermediate_size\": 3072,\n",
+ " \"language\": \"english\",\n",
+ " \"layer_norm_eps\": 1e-05,\n",
+ " \"max_position_embeddings\": 514,\n",
+ " \"model_type\": \"roberta\",\n",
+ " \"name\": \"Roberta\",\n",
+ " \"num_attention_heads\": 12,\n",
+ " \"num_hidden_layers\": 12,\n",
+ " \"pad_token_id\": 1,\n",
+ " \"position_embedding_type\": \"absolute\",\n",
+ " \"transformers_version\": \"4.34.0\",\n",
+ " \"type_vocab_size\": 1,\n",
+ " \"use_cache\": true,\n",
+ " \"vocab_size\": 50265\n",
+ "}"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model.config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def calculate_perplexity(sentence):\n",
+ " inputs = tokenizer(sentence, return_tensors='pt').to(\"cuda\")\n",
+ " with torch.no_grad():\n",
+ " outputs = model(**inputs, labels=inputs['input_ids'])\n",
+ " print(outputs)\n",
+ " loss = outputs.loss # cross entropy loss ----> assuming our model gives us probabilities of different words\n",
+ " perplexity = torch.exp(loss)\n",
+ " return perplexity.item()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MaskedLMOutput(loss=tensor(17.8039, device='cuda:0'), logits=tensor([[[ 0.3795, 0.3104, -0.5476, ..., 0.6206, 1.2048, 0.2867],\n",
+ " [ 6.1099, 0.2996, -3.4115, ..., 0.1186, -2.1274, -0.5033],\n",
+ " [ 3.1611, 0.4855, -1.8712, ..., 1.0915, -1.1092, 0.9038],\n",
+ " ...,\n",
+ " [ 4.7616, 0.5287, -4.6328, ..., 0.6480, -2.6257, 0.2898],\n",
+ " [-1.5800, 0.9459, -4.8465, ..., -1.5705, -3.4567, -1.7680],\n",
+ " [-2.2582, 0.7316, -5.1463, ..., -2.0331, -4.5188, -0.8688]]],\n",
+ " device='cuda:0'), hidden_states=None, attentions=None)\n",
+ "Perplexity of the sentence1: 53969640.0\n",
+ "MaskedLMOutput(loss=tensor(17.1493, device='cuda:0'), logits=tensor([[[ 0.2888, 0.1490, -0.1353, ..., 0.9108, 1.6271, 0.4736],\n",
+ " [ 7.6726, 0.6179, -2.9211, ..., 1.1359, -1.1669, 0.7527],\n",
+ " [ 3.2592, 0.6674, -4.1829, ..., -1.3289, -2.9414, 0.0205],\n",
+ " ...,\n",
+ " [ 1.4004, 0.5725, -3.3266, ..., -0.8717, -3.7206, -0.4705],\n",
+ " [-1.4701, 0.8628, -5.1591, ..., -1.6357, -3.4620, -1.6146],\n",
+ " [-2.7571, 0.6047, -5.1016, ..., -1.9740, -4.2890, -1.0212]]],\n",
+ " device='cuda:0'), hidden_states=None, attentions=None)\n",
+ "Perplexity of the sentence2: 28044616.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f'Perplexity of the sentence1: {calculate_perplexity(sentence1)}')\n",
+ "print(f'Perplexity of the sentence2: {calculate_perplexity(sentence2)}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'input_ids': tensor([[ 0, 25589, 34205, 6, 67, 684, 25, 514, 15000, 6,\n",
+ " 8944, 30420, 8475, 6, 81, 30420, 8475, 6, 50, 514,\n",
+ " 46334, 23249, 6, 16, 10, 2905, 6484, 20771, 11, 2900,\n",
+ " 8047, 14, 775, 77, 5, 2340, 2394, 9, 39875, 12782,\n",
+ " 11, 5, 809, 16, 3148, 751, 1522, 4971, 30, 10079,\n",
+ " 514, 14797, 4, 50118, 50118, 17245, 2340, 4215, 6, 13636,\n",
+ " 16997, 350, 203, 514, 16, 20135, 3159, 4, 9221, 70,\n",
+ " 3257, 1330, 7, 514, 34205, 11, 2340, 2172, 33, 4596,\n",
+ " 1169, 31, 514, 12, 10232, 18957, 11997, 6, 11, 61,\n",
+ " 2172, 2120, 7, 14623, 739, 5353, 9, 514, 6, 50,\n",
+ " 31, 251, 24750, 9, 3325, 148, 61, 10079, 5353, 9,\n",
+ " 12293, 58, 13056, 31274, 134, 742, 96, 1285, 6, 514,\n",
+ " 13306, 6, 10, 5448, 9, 11809, 11, 61, 5, 1802,\n",
+ " 16, 1654, 7, 14623, 10079, 5353, 9, 514, 6, 64,\n",
+ " 1303, 514, 34205, 31274, 134, 742, 50118, 50118, 25589, 6,\n",
+ " 101, 143, 97, 6572, 6, 64, 28, 1687, 10, 17712,\n",
+ " 77, 81, 12, 10998, 28817, 11, 10, 4315, 675, 9,\n",
+ " 86, 4, 3201, 34205, 2260, 11493, 77, 514, 16, 145,\n",
+ " 13056, 11, 10, 239, 16363, 396, 9077, 39875, 859, 14797,\n",
+ " 31274, 176, 742, 50118, 50118, 9089, 19348, 9, 809, 514,\n",
+ " 189, 67, 28, 10, 898, 9, 10, 1131, 1881, 50,\n",
+ " 18418, 1416, 131, 192, 22, 33027, 261, 415, 5593, 493,\n",
+ " 113, 13, 103, 7721, 4, 3201, 16, 1687, 65, 9,\n",
+ " 5, 513, 8422, 4747, 18291, 6, 19, 41, 34744, 1096,\n",
+ " 17976, 1814, 36769, 73, 9043, 11, 24162, 131, 10975, 246,\n",
+ " 742, 4835, 411, 6474, 268, 11, 130, 722, 34, 1726,\n",
+ " 5, 744, 9, 10, 1050, 31274, 306, 742, 2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+ " 1, 1, 1, 1, 1]])}"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "encodings_sentence1= tokenizer(sentence1, return_tensors=\"pt\")\n",
+ "encodings_sentence1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "seq_len = 269\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/269 [00:00<?, ?it/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "# max_length = model.config.n_positions\n",
+ "max_length= model.config.max_position_embeddings\n",
+ "stride = 1\n",
+ "seq_len = encodings_sentence1.input_ids.size(1)\n",
+ "\n",
+ "print(f\"seq_len = {seq_len}\")\n",
+ "\n",
+ "nlls = []\n",
+ "prev_end_loc = 0\n",
+ "for begin_loc in tqdm(range(0, seq_len, stride)):\n",
+ " end_loc = min(begin_loc + max_length, seq_len)\n",
+ " trg_len = end_loc - prev_end_loc # may be different from stride on last loop\n",
+ " input_ids = encodings_sentence1.input_ids[:, begin_loc:end_loc].to(\"cuda\")\n",
+ " target_ids = input_ids.clone()\n",
+ " target_ids[:, :-trg_len] = -100\n",
+ "\n",
+ " with torch.no_grad():\n",
+ " outputs = model(input_ids, labels=target_ids)\n",
+ "\n",
+ " # loss is calculated using CrossEntropyLoss which averages over valid labels\n",
+ " # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels\n",
+ " # to the left by 1.\n",
+ " neg_log_likelihood = outputs.loss\n",
+ "\n",
+ " nlls.append(neg_log_likelihood)\n",
+ "\n",
+ " prev_end_loc = end_loc\n",
+ " if end_loc == seq_len:\n",
+ " break\n",
+ "\n",
+ "ppl = torch.exp(torch.stack(nlls).mean())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(53969640., device='cuda:0')"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ppl"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "seq_len = 324\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/324 [00:00<?, ?it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "tensor(28044616., device='cuda:0')"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "encodings_sentence2= tokenizer(sentence2, return_tensors=\"pt\")\n",
+ "encodings_sentence2\n",
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "# max_length = model.config.n_positions\n",
+ "max_length= model.config.max_position_embeddings\n",
+ "stride = 1\n",
+ "seq_len = encodings_sentence2.input_ids.size(1)\n",
+ "\n",
+ "print(f\"seq_len = {seq_len}\")\n",
+ "\n",
+ "nlls = []\n",
+ "prev_end_loc = 0\n",
+ "for begin_loc in tqdm(range(0, seq_len, stride)):\n",
+ " end_loc = min(begin_loc + max_length, seq_len)\n",
+ " trg_len = end_loc - prev_end_loc # may be different from stride on last loop\n",
+ " input_ids = encodings_sentence2.input_ids[:, begin_loc:end_loc].to(\"cuda\")\n",
+ " target_ids = input_ids.clone()\n",
+ " target_ids[:, :-trg_len] = -100\n",
+ "\n",
+ " with torch.no_grad():\n",
+ " outputs = model(input_ids, labels=target_ids)\n",
+ "\n",
+ " # loss is calculated using CrossEntropyLoss which averages over valid labels\n",
+ " # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels\n",
+ " # to the left by 1.\n",
+ " neg_log_likelihood = outputs.loss\n",
+ "\n",
+ " nlls.append(neg_log_likelihood)\n",
+ "\n",
+ " prev_end_loc = end_loc\n",
+ " if end_loc == seq_len:\n",
+ " break\n",
+ "\n",
+ "ppl = torch.exp(torch.stack(nlls).mean())\n",
+ "ppl"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "seq_len = 4\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/4 [00:00<?, ?it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "tensor(67252448., device='cuda:0')"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "encodings_sentence2= tokenizer(\"Good morning\", return_tensors=\"pt\")\n",
+ "encodings_sentence2\n",
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "# max_length = model.config.n_positions\n",
+ "max_length= model.config.max_position_embeddings\n",
+ "stride = 1\n",
+ "seq_len = encodings_sentence2.input_ids.size(1)\n",
+ "\n",
+ "print(f\"seq_len = {seq_len}\")\n",
+ "\n",
+ "nlls = []\n",
+ "prev_end_loc = 0\n",
+ "for begin_loc in tqdm(range(0, seq_len, stride)):\n",
+ " end_loc = min(begin_loc + max_length, seq_len)\n",
+ " trg_len = end_loc - prev_end_loc # may be different from stride on last loop\n",
+ " input_ids = encodings_sentence2.input_ids[:, begin_loc:end_loc].to(\"cuda\")\n",
+ " target_ids = input_ids.clone()\n",
+ " target_ids[:, :-trg_len] = -100\n",
+ "\n",
+ " with torch.no_grad():\n",
+ " outputs = model(input_ids, labels=target_ids)\n",
+ "\n",
+ " # loss is calculated using CrossEntropyLoss which averages over valid labels\n",
+ " # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels\n",
+ " # to the left by 1.\n",
+ " neg_log_likelihood = outputs.loss\n",
+ "\n",
+ " nlls.append(neg_log_likelihood)\n",
+ "\n",
+ " prev_end_loc = end_loc\n",
+ " if end_loc == seq_len:\n",
+ " break\n",
+ "\n",
+ "ppl = torch.exp(torch.stack(nlls).mean())\n",
+ "ppl"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/ubuntu/SentenceStructureComparision/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n",
+ "Downloading (…)lve/main/config.json: 100%|██████████| 666/666 [00:00<00:00, 4.11MB/s]\n",
+ "Downloading model.safetensors: 100%|██████████| 3.25G/3.25G [00:36<00:00, 89.3MB/s]\n",
+ "Downloading (…)neration_config.json: 100%|██████████| 124/124 [00:00<00:00, 75.5kB/s]\n",
+ "Downloading (…)olve/main/vocab.json: 100%|██████████| 1.04M/1.04M [00:00<00:00, 4.88MB/s]\n",
+ "Downloading (…)olve/main/merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 51.0MB/s]\n",
+ "Downloading (…)/main/tokenizer.json: 100%|██████████| 1.36M/1.36M [00:00<00:00, 70.6MB/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "from transformers import GPT2LMHeadModel, GPT2TokenizerFast\n",
+ "\n",
+ "device = \"cuda\"\n",
+ "model_id = \"gpt2-large\"\n",
+ "model = GPT2LMHeadModel.from_pretrained(model_id).to(device)\n",
+ "tokenizer = GPT2TokenizerFast.from_pretrained(model_id)\n",
+ "encodings = tokenizer(sentence1, return_tensors=\"pt\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/1 [00:00<?, ?it/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "max_length = model.config.n_positions\n",
+ "stride = 512\n",
+ "seq_len = encodings.input_ids.size(1)\n",
+ "\n",
+ "nlls = []\n",
+ "prev_end_loc = 0\n",
+ "for begin_loc in tqdm(range(0, seq_len, stride)):\n",
+ " end_loc = min(begin_loc + max_length, seq_len)\n",
+ " trg_len = end_loc - prev_end_loc # may be different from stride on last loop\n",
+ " input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)\n",
+ " target_ids = input_ids.clone()\n",
+ " target_ids[:, :-trg_len] = -100\n",
+ "\n",
+ " with torch.no_grad():\n",
+ " outputs = model(input_ids, labels=target_ids)\n",
+ "\n",
+ " # loss is calculated using CrossEntropyLoss which averages over valid labels\n",
+ " # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels\n",
+ " # to the left by 1.\n",
+ " neg_log_likelihood = outputs.loss\n",
+ "\n",
+ " nlls.append(neg_log_likelihood)\n",
+ "\n",
+ " prev_end_loc = end_loc\n",
+ " if end_loc == seq_len:\n",
+ " break\n",
+ "\n",
+ "ppl = torch.exp(torch.stack(nlls).mean())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "tensor(12.3761, device='cuda:0')"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ppl"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "seq_len = 322\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 0%| | 0/1 [00:00<?, ?it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "tensor(30.3624, device='cuda:0')"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "encodings_sentence2= tokenizer(sentence2, return_tensors=\"pt\")\n",
+ "encodings_sentence2\n",
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "# max_length = model.config.n_positions\n",
+ "max_length= model.config.max_position_embeddings\n",
+ "stride = 512\n",
+ "seq_len = encodings_sentence2.input_ids.size(1)\n",
+ "\n",
+ "print(f\"seq_len = {seq_len}\")\n",
+ "\n",
+ "nlls = []\n",
+ "prev_end_loc = 0\n",
+ "for begin_loc in tqdm(range(0, seq_len, stride)):\n",
+ " end_loc = min(begin_loc + max_length, seq_len)\n",
+ " trg_len = end_loc - prev_end_loc # may be different from stride on last loop\n",
+ " input_ids = encodings_sentence2.input_ids[:, begin_loc:end_loc].to(\"cuda\")\n",
+ " target_ids = input_ids.clone()\n",
+ " target_ids[:, :-trg_len] = -100\n",
+ "\n",
+ " with torch.no_grad():\n",
+ " outputs = model(input_ids, labels=target_ids)\n",
+ "\n",
+ " # loss is calculated using CrossEntropyLoss which averages over valid labels\n",
+ " # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels\n",
+ " # to the left by 1.\n",
+ " neg_log_likelihood = outputs.loss\n",
+ "\n",
+ " nlls.append(neg_log_likelihood)\n",
+ "\n",
+ " prev_end_loc = end_loc\n",
+ " if end_loc == seq_len:\n",
+ " break\n",
+ "\n",
+ "ppl = torch.exp(torch.stack(nlls).mean())\n",
+ "ppl"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s1= '''Imagine you’re trying to build a chatbot that helps home cooks autocomplete their grocery shopping lists based on popular flavor combinations from social media. Your goal is to let users type in what they have in their fridge, like “chicken, carrots,” then list the five or six ingredients that go best with those flavors. You’ve already scraped thousands of recipe sites for ingredient lists, and now you just need to choose the best NLP model to predict which words appear together most often. Easy, right?\n",
+ "\n",
+ "Well, not exactly. The gold standard for checking the performance of a model is extrinsic evaluation: measuring its final performance on a real-world task. In this case, that might mean letting your model generate a dataset of a thousand new recipes, then asking a few hundred data labelers to rate how tasty they sound.\n",
+ "\n",
+ "Unfortunately, you don’t have one dataset, you have one dataset for every variation of every parameter of every model you want to test. Even simple comparisons of the same basic model can lead to a combinatorial explosion: 3 different optimization functions with 5 different learning rates and 4 different batch sizes equals 120 different datasets, all with hundreds of thousands of individual data points. How can you quickly narrow down which models are the most promising to fully evaluate?\n",
+ "\n",
+ "Enter intrinsic evaluation: finding some property of a model that estimates the model’s quality independent of the specific tasks its used to perform. Specifically, enter perplexity, a metric that quantifies how uncertain a model is about the predictions it makes. Low perplexity only guarantees a model is confident, not accurate, but it often correlates well with the model’s final real-world performance, and it can be quickly calculated using just the probability distribution the model learns from the training dataset.\n",
+ "\n",
+ "In this week’s post, we’ll look at how perplexity is calculated, what it means intuitively for a model’s performance, and the pitfalls of using perplexity for comparisons across different datasets and models.\n",
+ "\n",
+ "Calculating perplexity\n",
+ "To understand how perplexity is calculated, let’s start with a very simple version of the recipe training dataset that only has four short ingredient lists:\n",
+ "\n",
+ "chicken, butter, pears\n",
+ "chicken, butter, chili\n",
+ "lemon, pears, shrimp\n",
+ "chili, shrimp, lemon\n",
+ "In machine learning terms, these sentences are a language with a vocabulary size of 6 (because there are a total of 6 unique words). A language model is just a function trained on a specific language that predicts the probability of a certain word appearing given the words that appeared around it.\n",
+ "\n",
+ "One of the simplest language models is a unigram model, which looks at words one at a time assuming they’re statistically independent. In other words, it returns the relative frequency that each word appears in the training data. Here’s a unigram model for the dataset above, which is especially simple because every word appears the same number of times:\n",
+ "\n",
+ "\n",
+ "It’s pretty obvious this isn’t a very good model. No matter which ingredients you say you have, it will just pick any new ingredient at random with equal probability, so you might as well be rolling a fair die to choose. Let’s quantify exactly how bad this is.\n",
+ "\n",
+ "We’re going to start by calculating how surprised our model is when it sees a single specific word like “chicken.” Intuitively, the more probable an event is, the less surprising it is. If you’re certain something is impossible — if its probability is 0 — then you would be infinitely surprised if it happened. Similarly, if something was guaranteed to happen with probability 1, your surprise when it happened would be 0.'''\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "# generated by gpt\n",
+ "\n",
+ "s2= '''Imagine you want to create a smart assistant for people who cook at home. This assistant should help them make shopping lists based on popular food combinations they find on social media. Your aim is to allow users to type in what ingredients they already have, like \"chicken, carrots,\" and then get suggestions for the five or six best ingredients that go well with those. To do this, you've collected lots of lists of ingredients from recipe websites. Now, your main task is to pick the best computer program that can predict which ingredients often appear together. Sounds easy, right?\n",
+ "\n",
+ "But it's not that simple. The usual way to check how well a computer program works is to see how well it does on a real task. In this case, that means having your program create a thousand new recipes and then asking a few hundred people to rate how good those recipes sound.\n",
+ "\n",
+ "Here's the problem: You don't have just one set of recipes to test your program on; you have many sets, each with different settings. Even when comparing different versions of the same program, there can be a huge number of combinations to test. For example, if you have three different ways to make the program work, five different speeds for the program to learn, and four different sizes of groups of data, you end up with 120 different sets of recipes to evaluate. Each set contains hundreds of thousands of individual data points. So, how do you quickly figure out which versions of the program are the most promising to test further?\n",
+ "\n",
+ "This is where \"intrinsic evaluation\" comes in. It means finding some quality of the program that tells you how good it is without having to do the real cooking task. In this case, it's about \"perplexity,\" which is a way to measure how uncertain the program is when it makes predictions. A low perplexity score means the program is pretty sure about its predictions, though it doesn't guarantee those predictions are correct. But it often matches up with how well the program does in real cooking tasks, and it's easy to calculate using the information the program learns from the training data.\n",
+ "\n",
+ "In this post, we'll explore how to calculate perplexity, what it indicates about the program's performance, and the problems you might face when using perplexity to compare different sets of data and programs.\n",
+ "\n",
+ "Calculating Perplexity:\n",
+ "To understand how perplexity works, let's start with a simple example. Imagine you have a small dataset of recipes with just four short lists of ingredients:\n",
+ "\n",
+ "1. Chicken, butter, pears\n",
+ "2. Chicken, butter, chili\n",
+ "3. Lemon, pears, shrimp\n",
+ "4. Chili, shrimp, lemon\n",
+ "\n",
+ "In machine learning terms, these sentences form a language with only six different words. A language model is like a computer program that has been trained on this language. It predicts the likelihood of a word appearing based on the words that came before it.\n",
+ "\n",
+ "One of the simplest language models is called a \"unigram model.\" It assumes that words are independent of each other and predicts each word's frequency based on how often it appears in the training data. Here's what a unigram model for this dataset would look like, and it's pretty basic because it assigns the same probability to every word:\n",
+ "\n",
+ "This model isn't very good. No matter what ingredients you mention, it will randomly pick a new ingredient with equal likelihood. It's like rolling a fair die to choose an ingredient. Let's measure exactly how bad it is.\n",
+ "\n",
+ "First, we'll calculate how surprised the model is when it sees a specific word like \"chicken.\" The more probable an event is, the less surprising it is. If something has zero chance of happening (probability of 0), you'd be incredibly surprised if it did. On the other hand, if something is guaranteed to happen (probability of 1), you wouldn't be surprised at all when it occurs.'''"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s1= '''Basketball is a team sport played by two teams of five players each. The primary objective is to score points by shooting the basketball through the opponent's hoop, which is mounted on a backboard 10 feet (3.048 meters) above the ground. The team with the most points at the end of the game wins. Basketball is played on a rectangular court, typically indoors, with a surface made of wood or synthetic materials. The rules and regulations are governed by various organizations, such as FIBA (International Basketball Federation) and the NBA (National Basketball Association). The following is a general outline of the basic rules of basketball:\n",
+ "\n",
+ "1. Game duration: A regulation basketball game is divided into four quarters, each lasting 12 minutes in the NBA and 10 minutes in FIBA play. College basketball in the US has two 20-minute halves. If the game is tied at the end of regulation, overtime periods are played until a winner is determined.\n",
+ "\n",
+ "2. Starting play: The game begins with a jump ball at the center of the court, where the referee throws the ball into the air, and one player from each team tries to gain possession by tapping it to a teammate.\n",
+ "\n",
+ "3. Scoring: Points are scored by shooting the ball through the hoop. A field goal made from inside the three-point arc is worth two points, while a field goal made from outside the arc is worth three points. Free throws, awarded after a foul, are worth one point each.\n",
+ "\n",
+ "4. Possession and dribbling: A player in possession of the ball must either pass it to a teammate or dribble (bounce) the ball while moving.'''"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s2= '''Political stability is the ability of a government to maintain order and authority within its borders. It is essential for economic growth, as it provides a foundation for investment and trade.\n",
+ "There are many factors that contribute to political stability, including:\n",
+ "A strong rule of law: The rule of law is the principle that everyone is subject to the same laws, regardless of their social status or political affiliation. A strong rule of law helps to prevent corruption and ensures that everyone has equal opportunity to succeed.\n",
+ "A well-functioning government: A well-functioning government is one that is able to provide essential services, such as security, education, and healthcare. It is also able to manage the economy effectively and to respond to crises.\n",
+ "A vibrant civil society: A vibrant civil society is one that is made up of active and engaged citizens. Civil society organizations can help to hold the government accountable and to promote democracy and good governance.\n",
+ "Political stability is not always easy to achieve, but it is essential for economic growth. By investing in political stability, we can create a foundation for long-term prosperity.\n",
+ "Here are some of the benefits of political stability:\n",
+ "Increased investment: Investors are more likely to invest in countries that are politically stable. This can lead to increased economic growth and job creation.\n",
+ "Improved trade: Trade between countries is easier and more efficient when there is political stability. This can lead to lower prices for consumers and increased profits for businesses.\n",
+ "Reduced poverty: Political stability can help to reduce poverty by creating a more conducive environment for economic growth.\n",
+ "Improved quality of life: Political stability can lead to improved quality of life by providing a safer and more secure environment.\n",
+ "Political stability is a key ingredient for a prosperous and successful society. By investing in political stability, we can create a better future for ourselves and our children.\n",
+ "Here are some of the challenges to political stability:\n",
+ "Economic inequality: Economic inequality can lead to social unrest and instability. This is because it can create a sense of injustice and resentment among those who are not benefiting from economic growth.\n",
+ "Corruption: Corruption can undermine the rule of law and erode public trust in government. This can lead to instability and violence.\n",
+ "Ethnic and religious conflict: Ethnic and religious conflict can be a major source of instability. This is because it can lead to violence, displacement, and economic disruption.\n",
+ "Natural disasters: Natural disasters can also be a source of instability. This is because they can displace people, damage infrastructure, and disrupt economic activity.\n",
+ "Despite the challenges, there are many things that can be done to promote political stability. These include:\n",
+ "Investing in education and healthcare: Education and healthcare can help to reduce poverty and inequality, which are two of the main causes of instability.\n",
+ "Promoting good governance: Good governance is essential for building trust between the government and the people. It can be promoted by strengthening the rule of law, fighting corruption, and ensuring transparency and accountability.\n",
+ "Resolving conflict peacefully: Conflict can be resolved peacefully through negotiation, mediation, and other means. This can help to prevent violence and instability.\n",
+ "Building resilience: Building resilience is essential for coping with shocks and stresses, such as economic downturns and natural disasters. It can be done by investing in infrastructure, social safety nets, and disaster preparedness.\n",
+ "Political stability is a complex issue, but it is essential for economic growth and prosperity. By investing in political stability, we can create a better future for ourselves and our children.'''"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 65,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s1= '''In a quiet, picturesque village nestled deep within the lush, rolling hills of the countryside, there stood a charming, centuries-old cottage, its timeworn facade adorned with colorful flowers that cascaded down from window boxes. The cottage, with its rustic charm, had seen generations come and go, witnessed countless stories unfold within its sturdy walls. Each morning, as the sun cast its golden rays upon the sleepy hamlet, the villagers would wake to the melodious chirping of birds, their cheerful songs serving as a gentle alarm clock. Life in the village was slow-paced, a stark contrast to the bustling cities with their constant noise and ceaseless activity. Time seemed to move differently here, as if the world beyond the village's borders existed in a parallel universe, always in a hurry, while the village embraced a rhythm that ebbed and flowed with the changing seasons. The villagers, bound by a strong sense of community, gathered for festivals, sharing laughter and stories around bonfires that crackled in the cool night air. Generations of families had lived in this idyllic haven, passing down stories, traditions, and the enduring spirit of the village from one age to the next, ensuring that the passage of time only deepened their connection to this place they called home.'''\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 86,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s1= \"\"\"The Mission Impossible franchise, a timeless icon in the world of espionage thrillers, has held audiences captive for decades with its electrifying fusion of high-stakes action, intricate espionage plots, and mind-boggling twists. Tom Cruise, a true embodiment of Ethan Hunt, the daring and ingenious secret agent, has forever etched his name alongside the series, leading a team of accomplished operatives on missions that, at first glance, appear insurmountably challenging, destined to thwart global threats. With every new installment, viewers are treated to a whirlwind of meticulously staged action sequences, Tom Cruise's jaw-dropping stunts – performed by the man himself, and a labyrinth of betrayals and double-crosses that keeps everyone on the edge of their seats, leaving them guessing until the very last, suspenseful moments. The franchise's enduring charm stems from its unyielding commitment to pushing cinematic action's boundaries, making sure that each mission remains an impossibly thrilling spectacle that unfolds relentlessly, offering a rollercoaster of excitement for fans of all ages.\"\"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 87,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['The Mission Impossible franchise, a timeless icon in the world of espionage thrillers, has held audiences captive for decades with its electrifying fusion of high-stakes action, intricate espionage plots, and mind-boggling twists',\n",
+ " ' Tom Cruise, a true embodiment of Ethan Hunt, the daring and ingenious secret agent, has forever etched his name alongside the series, leading a team of accomplished operatives on missions that, at first glance, appear insurmountably challenging, destined to thwart global threats',\n",
+ " \" With every new installment, viewers are treated to a whirlwind of meticulously staged action sequences, Tom Cruise's jaw-dropping stunts – performed by the man himself, and a labyrinth of betrayals and double-crosses that keeps everyone on the edge of their seats, leaving them guessing until the very last, suspenseful moments\",\n",
+ " \" The franchise's enduring charm stems from its unyielding commitment to pushing cinematic action's boundaries, making sure that each mission remains an impossibly thrilling spectacle that unfolds relentlessly, offering a rollercoaster of excitement for fans of all ages\",\n",
+ " '']"
+ ]
+ },
+ "execution_count": 87,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# number of sentences\n",
+ "# list_of_sentences1= sentence1.split('.')\n",
+ "s1= s1.replace('\\n', ' ')\n",
+ "list_of_sentences1= s1.split('.')\n",
+ "list_of_sentences1\n",
+ "\n",
+ "\n",
+ "# x= []\n",
+ "# for i in list_of_sentences1:\n",
+ "# if len(i)>10:\n",
+ "# x.append(i)\n",
+ " \n",
+ "list_of_sentences1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['Political stability is the ability of a government to maintain order and authority within its borders',\n",
+ " ' It is essential for economic growth, as it provides a foundation for investment and trade',\n",
+ " '\\nThere are many factors that contribute to political stability, including:\\nA strong rule of law: The rule of law is the principle that everyone is subject to the same laws, regardless of their social status or political affiliation',\n",
+ " ' A strong rule of law helps to prevent corruption and ensures that everyone has equal opportunity to succeed',\n",
+ " '\\nA well-functioning government: A well-functioning government is one that is able to provide essential services, such as security, education, and healthcare',\n",
+ " ' It is also able to manage the economy effectively and to respond to crises',\n",
+ " '\\nA vibrant civil society: A vibrant civil society is one that is made up of active and engaged citizens',\n",
+ " ' Civil society organizations can help to hold the government accountable and to promote democracy and good governance',\n",
+ " '\\nPolitical stability is not always easy to achieve, but it is essential for economic growth',\n",
+ " ' By investing in political stability, we can create a foundation for long-term prosperity',\n",
+ " '\\nHere are some of the benefits of political stability:\\nIncreased investment: Investors are more likely to invest in countries that are politically stable',\n",
+ " ' This can lead to increased economic growth and job creation',\n",
+ " '\\nImproved trade: Trade between countries is easier and more efficient when there is political stability',\n",
+ " ' This can lead to lower prices for consumers and increased profits for businesses',\n",
+ " '\\nReduced poverty: Political stability can help to reduce poverty by creating a more conducive environment for economic growth',\n",
+ " '\\nImproved quality of life: Political stability can lead to improved quality of life by providing a safer and more secure environment',\n",
+ " '\\nPolitical stability is a key ingredient for a prosperous and successful society',\n",
+ " ' By investing in political stability, we can create a better future for ourselves and our children',\n",
+ " '\\nHere are some of the challenges to political stability:\\nEconomic inequality: Economic inequality can lead to social unrest and instability',\n",
+ " ' This is because it can create a sense of injustice and resentment among those who are not benefiting from economic growth',\n",
+ " '\\nCorruption: Corruption can undermine the rule of law and erode public trust in government',\n",
+ " ' This can lead to instability and violence',\n",
+ " '\\nEthnic and religious conflict: Ethnic and religious conflict can be a major source of instability',\n",
+ " ' This is because it can lead to violence, displacement, and economic disruption',\n",
+ " '\\nNatural disasters: Natural disasters can also be a source of instability',\n",
+ " ' This is because they can displace people, damage infrastructure, and disrupt economic activity',\n",
+ " '\\nDespite the challenges, there are many things that can be done to promote political stability',\n",
+ " ' These include:\\nInvesting in education and healthcare: Education and healthcare can help to reduce poverty and inequality, which are two of the main causes of instability',\n",
+ " '\\nPromoting good governance: Good governance is essential for building trust between the government and the people',\n",
+ " ' It can be promoted by strengthening the rule of law, fighting corruption, and ensuring transparency and accountability',\n",
+ " '\\nResolving conflict peacefully: Conflict can be resolved peacefully through negotiation, mediation, and other means',\n",
+ " ' This can help to prevent violence and instability',\n",
+ " '\\nBuilding resilience: Building resilience is essential for coping with shocks and stresses, such as economic downturns and natural disasters',\n",
+ " ' It can be done by investing in infrastructure, social safety nets, and disaster preparedness',\n",
+ " '\\nPolitical stability is a complex issue, but it is essential for economic growth and prosperity',\n",
+ " ' By investing in political stability, we can create a better future for ourselves and our children',\n",
+ " '']"
+ ]
+ },
+ "execution_count": 61,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# list_of_sentences2= sentence2.split('.')\n",
+ "# s2= s2.replace('\\n', ' ')\n",
+ "\n",
+ "list_of_sentences2= s2.split('.')\n",
+ "list_of_sentences2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "def calculate_burst(list_of_sentences):\n",
+ " arr= []\n",
+ " for i in list_of_sentences:\n",
+ " ei= tokenizer(i, return_tensors=\"pt\")\n",
+ " arr.append(ei.input_ids.size(1))\n",
+ " print(f\"arr= {(arr)}\")\n",
+ " return np.var(np.array(arr))\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "arr= [41, 27, 24, 37, 24, 28, 42, 35, 24, 11, 23, 21, 29, 40]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "74.14285714285714"
+ ]
+ },
+ "execution_count": 78,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "calculate_burst(list_of_sentences1[:-1])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "arr= [16, 16, 44, 18, 32, 14, 21, 17, 17, 16, 27, 10, 17, 13, 21, 23, 13, 17, 24, 21, 18, 7, 18, 14, 13, 16, 17, 31, 19, 19, 19, 8, 23, 17, 17, 17]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "45.57098765432099"
+ ]
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "calculate_burst(list_of_sentences2[:-1])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Method 3: `Summarization via Language Model`\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Use a pipeline as a high-level helper\n",
+ "from transformers import pipeline\n",
+ "\n",
+ "pipe = pipeline(\"summarization\", model=\"DunnBC22/flan-t5-base-text_summarization_data\", device=\"cuda\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[{'summary_text': 'Understand water intoxication. Recognize water poisoning. Understand the dangers of excessive water intake. Understand how water can be considered a poison. Understand that water is considered one of the least toxic chemical compounds, with an LD50 exceeding 90 ml/kg in rats.'},\n",
+ " {'summary_text': 'Understand hyponatremia. Recognize the dangers of aqueous intoxication. Understand the causes of aqua inebriation. Identify the cause of venomous aquious content. Describe the effects of water on human health.'}]"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "pipe([sentence1, sentence2], min_length=50, max_length=200)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Use a pipeline as a high-level helper\n",
+ "# from transformers import pipeline\n",
+ "\n",
+ "# pipe = pipeline(\"summarization\", model=\"google/pegasus-cnn_dailymail\", device=\"cuda\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[{'summary_text': 'Cat ear headphones are popular among otakus, streamers, gamers, and anyone who wants a uniquely cute look .<n>Design is the priority, but sound quality should also be considered .'},\n",
+ " {'summary_text': 'Feline-themed audio headgear enjoys favor among aficionados of anime and gaming, as well as content creators .<n>Seek out headphones delivering crystal-clear and precise audio, and assess their suitability for both mature users and youngsters'}]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# pipe([sentence1, sentence2], min_length=50, max_length=200)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.12"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

research/02_dl_Ai_checker.ipynb

@@ -0,0 +1,1220 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os; os.chdir('..');"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from datasets import Dataset, load_dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Downloading readme: 100%|██████████| 2.63k/2.63k [00:00<00:00, 19.4MB/s]\n",
+ "Downloading data: 100%|██████████| 127M/127M [00:04<00:00, 30.3MB/s]\n",
+ "Downloading data files: 100%|██████████| 1/1 [00:04<00:00, 4.21s/it]\n",
+ "Extracting data files: 100%|██████████| 1/1 [00:02<00:00, 2.09s/it]\n",
+ "Generating train split: 150000 examples [00:04, 30015.64 examples/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "dataset_name= \"aadityaubhat/GPT-wiki-intro\"\n",
+ "\n",
+ "\n",
+ "\n",
+ "dataset= load_dataset(dataset_name)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "DatasetDict({\n",
+ " train: Dataset({\n",
+ " features: ['id', 'url', 'title', 'wiki_intro', 'generated_intro', 'title_len', 'wiki_intro_len', 'generated_intro_len', 'prompt', 'generated_text', 'prompt_tokens', 'generated_text_tokens'],\n",
+ " num_rows: 150000\n",
+ " })\n",
+ "})"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Creating CSV from Arrow format: 100%|██████████| 150/150 [00:08<00:00, 18.69ba/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "443537732"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset['train'].to_csv(\"data/original_data.csv\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>id</th>\n",
+ " <th>url</th>\n",
+ " <th>title</th>\n",
+ " <th>wiki_intro</th>\n",
+ " <th>generated_intro</th>\n",
+ " <th>title_len</th>\n",
+ " <th>wiki_intro_len</th>\n",
+ " <th>generated_intro_len</th>\n",
+ " <th>prompt</th>\n",
+ " <th>generated_text</th>\n",
+ " <th>prompt_tokens</th>\n",
+ " <th>generated_text_tokens</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>63064638</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Sexhow%20railway...</td>\n",
+ " <td>Sexhow railway station</td>\n",
+ " <td>Sexhow railway station was a railway station b...</td>\n",
+ " <td>Sexhow railway station was a railway station l...</td>\n",
+ " <td>3</td>\n",
+ " <td>174</td>\n",
+ " <td>78</td>\n",
+ " <td>200 word wikipedia style introduction on 'Sexh...</td>\n",
+ " <td>located in the town of Sexhow, on the Cumbria...</td>\n",
+ " <td>25</td>\n",
+ " <td>88</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>279621</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Eti%C3%A4inen</td>\n",
+ " <td>Etiäinen</td>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>1</td>\n",
+ " <td>187</td>\n",
+ " <td>80</td>\n",
+ " <td>200 word wikipedia style introduction on 'Etiä...</td>\n",
+ " <td>animate or inanimate, have a spirit or \"etiäi...</td>\n",
+ " <td>26</td>\n",
+ " <td>101</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>287229</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Inverse%20functi...</td>\n",
+ " <td>Inverse function theorem</td>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>3</td>\n",
+ " <td>170</td>\n",
+ " <td>59</td>\n",
+ " <td>200 word wikipedia style introduction on 'Inve...</td>\n",
+ " <td>function theorem states that for every real-v...</td>\n",
+ " <td>26</td>\n",
+ " <td>65</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>26712375</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Stepping%20on%20...</td>\n",
+ " <td>Stepping on Roses</td>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>3</td>\n",
+ " <td>335</td>\n",
+ " <td>121</td>\n",
+ " <td>200 word wikipedia style introduction on 'Step...</td>\n",
+ " <td>and illustrated by Maki Fujii. The series fol...</td>\n",
+ " <td>26</td>\n",
+ " <td>150</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>38894426</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Rob%20Bradley</td>\n",
+ " <td>Rob Bradley</td>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>2</td>\n",
+ " <td>170</td>\n",
+ " <td>136</td>\n",
+ " <td>200 word wikipedia style introduction on 'Rob ...</td>\n",
+ " <td>29, 1973) is an American former professional ...</td>\n",
+ " <td>28</td>\n",
+ " <td>162</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " id url \\\n",
+ "0 63064638 https://en.wikipedia.org/wiki/Sexhow%20railway... \n",
+ "1 279621 https://en.wikipedia.org/wiki/Eti%C3%A4inen \n",
+ "2 287229 https://en.wikipedia.org/wiki/Inverse%20functi... \n",
+ "3 26712375 https://en.wikipedia.org/wiki/Stepping%20on%20... \n",
+ "4 38894426 https://en.wikipedia.org/wiki/Rob%20Bradley \n",
+ "\n",
+ " title \\\n",
+ "0 Sexhow railway station \n",
+ "1 Etiäinen \n",
+ "2 Inverse function theorem \n",
+ "3 Stepping on Roses \n",
+ "4 Rob Bradley \n",
+ "\n",
+ " wiki_intro \\\n",
+ "0 Sexhow railway station was a railway station b... \n",
+ "1 In Finnish folklore, all places and things, an... \n",
+ "2 In mathematics, specifically differential calc... \n",
+ "3 is a Japanese shōjo manga series written and i... \n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... \n",
+ "\n",
+ " generated_intro title_len \\\n",
+ "0 Sexhow railway station was a railway station l... 3 \n",
+ "1 In Finnish folklore, all places and things, an... 1 \n",
+ "2 In mathematics, specifically differential calc... 3 \n",
+ "3 is a Japanese shōjo manga series written and i... 3 \n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... 2 \n",
+ "\n",
+ " wiki_intro_len generated_intro_len \\\n",
+ "0 174 78 \n",
+ "1 187 80 \n",
+ "2 170 59 \n",
+ "3 335 121 \n",
+ "4 170 136 \n",
+ "\n",
+ " prompt \\\n",
+ "0 200 word wikipedia style introduction on 'Sexh... \n",
+ "1 200 word wikipedia style introduction on 'Etiä... \n",
+ "2 200 word wikipedia style introduction on 'Inve... \n",
+ "3 200 word wikipedia style introduction on 'Step... \n",
+ "4 200 word wikipedia style introduction on 'Rob ... \n",
+ "\n",
+ " generated_text prompt_tokens \\\n",
+ "0 located in the town of Sexhow, on the Cumbria... 25 \n",
+ "1 animate or inanimate, have a spirit or \"etiäi... 26 \n",
+ "2 function theorem states that for every real-v... 26 \n",
+ "3 and illustrated by Maki Fujii. The series fol... 26 \n",
+ "4 29, 1973) is an American former professional ... 28 \n",
+ "\n",
+ " generated_text_tokens \n",
+ "0 88 \n",
+ "1 101 \n",
+ "2 65 \n",
+ "3 150 \n",
+ "4 162 "
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "df= pd.read_csv(\"data/original_data.csv\")\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>id</th>\n",
+ " <th>url</th>\n",
+ " <th>title</th>\n",
+ " <th>wiki_intro</th>\n",
+ " <th>generated_intro</th>\n",
+ " <th>title_len</th>\n",
+ " <th>wiki_intro_len</th>\n",
+ " <th>generated_intro_len</th>\n",
+ " <th>prompt</th>\n",
+ " <th>generated_text</th>\n",
+ " <th>prompt_tokens</th>\n",
+ " <th>generated_text_tokens</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>5</th>\n",
+ " <td>26709147</td>\n",
+ " <td>https://en.wikipedia.org/wiki/Moluccans</td>\n",
+ " <td>Moluccans</td>\n",
+ " <td>Moluccans are the Austronesian-speaking and Pa...</td>\n",
+ " <td>Moluccans are the Austronesian-speaking and Pa...</td>\n",
+ " <td>1</td>\n",
+ " <td>253</td>\n",
+ " <td>164</td>\n",
+ " <td>200 word wikipedia style introduction on 'Molu...</td>\n",
+ " <td>groups inhabiting the Maluku Islands. The ter...</td>\n",
+ " <td>33</td>\n",
+ " <td>238</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " id url title \\\n",
+ "5 26709147 https://en.wikipedia.org/wiki/Moluccans Moluccans \n",
+ "\n",
+ " wiki_intro \\\n",
+ "5 Moluccans are the Austronesian-speaking and Pa... \n",
+ "\n",
+ " generated_intro title_len \\\n",
+ "5 Moluccans are the Austronesian-speaking and Pa... 1 \n",
+ "\n",
+ " wiki_intro_len generated_intro_len \\\n",
+ "5 253 164 \n",
+ "\n",
+ " prompt \\\n",
+ "5 200 word wikipedia style introduction on 'Molu... \n",
+ "\n",
+ " generated_text prompt_tokens \\\n",
+ "5 groups inhabiting the Maluku Islands. The ter... 33 \n",
+ "\n",
+ " generated_text_tokens \n",
+ "5 238 "
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# df_0= df.head(1)\n",
+ "# df_0= df.iloc[1] # series result \n",
+ "df_0= df.iloc[[5]] # dataframe result\n",
+ "\n",
+ "df_0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "title= ['Moluccans']\n",
+ "\n",
+ "wiki_intro= ['Moluccans are the Austronesian-speaking and Papuan-speaking ethnic groups indigenous to the Maluku Islands, also called the Moluccas and historically known as the Spice Islands, which as a region has been annexed by Indonesia since the end of 1950. As such, \"Moluccans\" is used as a blanket term for the various ethnic and linguistic groups native to the islands.\\n\\t\\nThe original inhabitants of the Maluku Islands were Austronesian and Melanesian in origin. Austronesian peoples partially assimilated the native Melanesian population in terms of linguistics and other areas, around 2000\\xa0BCE due to extensive trade, making Malayo-Polynesian creole languages the lingua franca in most of the region. Later added to this were several Dutch, Chinese, Portuguese, Spanish, Arabian and English influences due to colonization, marriage with foreign traders during the Silk-route era and Middle ages, and even with European soldiers during the World Wars. A small number of German descendants was added to Moluccan population, especially in Ambon, along with arrival of Protestant Missionaries since 16th century. Moluccans are predominantly Christian like many other Melanesians, but Muslim villages are also present. Despite religious differences, all groups share strong cultural bonds and a sense of common identity, such as through Adat, Pela and Bongso traditions. Music is also a binding factor, playing an important role in the cultural identity. Moluccans historically tend to be a musically gifted people, excelling in creative areas such as singing and sports. In recognition, the Moluccan capital city of Ambon was awarded the official status of City of Music by UNESCO in 2019.']\n",
+ "\n",
+ "generated_intro= ['Moluccans are the Austronesian-speaking and Papuan-speaking ethnic groups inhabiting the Maluku Islands. The term \"Moluccan\" is an umbrella term that covers the various Austronesian and Papuan languages spoken on the islands. The largest group of Moluccans are the Tolo-speaking people.\\n\\nThe Maluku Islands are a group of volcanic islands in eastern Indonesia, located about 1,000 kilometres east of Java and 2,000 kilometres south of New Guinea. The islands comprise over 700 islands, with a total land area of approximately 245,000 square kilometres. They have a population of around 1 million people, most of whom are Muslim. The largest island group is the Tolo-speaking region, which comprises over 60% of the population.\\n\\nThe Moluccas were first explored by Europeans in 1512. Portuguese explorer João da Nova discovered Ternate and Tidore, while Dutch explorer Jacob Roggeveen discovered Halmahera and Bougainville. The Moluccas were later visited by Spanish explorer Ferdinand Magellan in 1521. British explorer James Cook visited Ternate, Tidore, Halmahera, and Bougainville during his first voyage in 1770–71.']\n",
+ "\n",
+ "prompt= [\"200 word wikipedia style introduction on 'Moluccans'\\n Moluccans are the Austronesian-speaking and Papuan-speaking ethnic\"]\n",
+ "\n",
+ "generated_text= [' groups inhabiting the Maluku Islands. The term \"Moluccan\" is an umbrella term that covers the various Austronesian and Papuan languages spoken on the islands. The largest group of Moluccans are the Tolo-speaking people.\\n\\nThe Maluku Islands are a group of volcanic islands in eastern Indonesia, located about 1,000 kilometres east of Java and 2,000 kilometres south of New Guinea. The islands comprise over 700 islands, with a total land area of approximately 245,000 square kilometres. They have a population of around 1 million people, most of whom are Muslim. The largest island group is the Tolo-speaking region, which comprises over 60% of the population.\\n\\nThe Moluccas were first explored by Europeans in 1512. Portuguese explorer João da Nova discovered Ternate and Tidore, while Dutch explorer Jacob Roggeveen discovered Halmahera and Bougainville. The Moluccas were later visited by Spanish explorer Ferdinand Magellan in 1521. British explorer James Cook visited Ternate, Tidore, Halmahera, and Bougainville during his first voyage in 1770–71.']\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f\"title= {df_0.title.values}\")\n",
+ "print()\n",
+ "print(f\"wiki_intro= {df_0.wiki_intro.values}\")\n",
+ "print()\n",
+ "print(f\"generated_intro= {df_0.generated_intro.values}\") # prompt {7 tokens} + generated_text\n",
+ "print()\n",
+ "print(f\"prompt= {df_0.prompt.values}\")\n",
+ "print()\n",
+ "print(f\"generated_text= {df_0.generated_text.values}\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station b...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149995</th>\n",
+ " <td>Randy Borum is a Professor and Coordinator of ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149996</th>\n",
+ " <td>Sa'och (, also, \"Sauch\") is an endangered, nea...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149997</th>\n",
+ " <td>Philip C. Hanawalt (born 1931) is an American ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149998</th>\n",
+ " <td>Vossius Gymnasium is a public gymnasium in Ams...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149999</th>\n",
+ " <td>Simone Stratigo (, Symeon Filippos Stratigos; ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>150000 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station b... 0\n",
+ "1 In Finnish folklore, all places and things, an... 0\n",
+ "2 In mathematics, specifically differential calc... 0\n",
+ "3 is a Japanese shōjo manga series written and i... 0\n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... 0\n",
+ "... ... ...\n",
+ "149995 Randy Borum is a Professor and Coordinator of ... 0\n",
+ "149996 Sa'och (, also, \"Sauch\") is an endangered, nea... 0\n",
+ "149997 Philip C. Hanawalt (born 1931) is an American ... 0\n",
+ "149998 Vossius Gymnasium is a public gymnasium in Ams... 0\n",
+ "149999 Simone Stratigo (, Symeon Filippos Stratigos; ... 0\n",
+ "\n",
+ "[150000 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "new_df_human= pd.DataFrame({\n",
+ " \"Paragraph\": df.wiki_intro, \n",
+ " \"AI_generated\": pd.Series([0]*len(df))\n",
+ " \n",
+ " }\n",
+ ")\n",
+ "new_df_human"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station l...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149995</th>\n",
+ " <td>Randy Borum is a Professor and Coordinator of ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149996</th>\n",
+ " <td>Sa'och (, also, \"Sauch\") is an endangered, nuc...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149997</th>\n",
+ " <td>Philip C. Hanawalt (born 1931) is an American ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149998</th>\n",
+ " <td>Vossius Gymnasium is a public gymnasium in the...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149999</th>\n",
+ " <td>Simone Stratigo (, Symeon Filippos Stratigos; ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>150000 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station l... 1\n",
+ "1 In Finnish folklore, all places and things, an... 1\n",
+ "2 In mathematics, specifically differential calc... 1\n",
+ "3 is a Japanese shōjo manga series written and i... 1\n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... 1\n",
+ "... ... ...\n",
+ "149995 Randy Borum is a Professor and Coordinator of ... 1\n",
+ "149996 Sa'och (, also, \"Sauch\") is an endangered, nuc... 1\n",
+ "149997 Philip C. Hanawalt (born 1931) is an American ... 1\n",
+ "149998 Vossius Gymnasium is a public gymnasium in the... 1\n",
+ "149999 Simone Stratigo (, Symeon Filippos Stratigos; ... 1\n",
+ "\n",
+ "[150000 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "new_df_ai= pd.DataFrame({\n",
+ " \"Paragraph\": df.generated_intro, \n",
+ " \"AI_generated\": pd.Series([1]*len(df))\n",
+ " \n",
+ " }\n",
+ ")\n",
+ "new_df_ai"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([\"In Finnish folklore, all places and things, and also human beings, have a haltija (a genius, guardian spirit) of their own. One such haltija is called etiäinen—an image, doppelgänger, or just an impression that goes ahead of a person, doing things the person in question later does. For example, people waiting at home might hear the door close or even see a shadow or a silhouette, only to realize that no one has yet arrived. Etiäinen can also refer to some kind of a feeling that something is going to happen. Sometimes it could, for example, warn of a bad year coming. In modern Finnish, the term has detached from its shamanistic origins and refers to premonition. Unlike clairvoyance, divination, and similar practices, etiäiset (plural) are spontaneous and can't be induced. Quite the opposite, they may be unwanted and cause anxiety, like ghosts. Etiäiset need not be too dramatic and may concern everyday events, although ones related to e.g. deaths are common. As these phenomena are still reported today, they can be considered a living tradition, as a way to explain the psychological experience of premonition.\"],\n",
+ " dtype=object)"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "new_df_human.iloc[[1]].Paragraph.values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array(['In Finnish folklore, all places and things, animate or inanimate, have a spirit or \"etiäinen\" that lives there. Etiäinen can manifest in many forms, but is usually described as a kind, elderly woman with white hair. She is the guardian of natural places and often helps people in need. \\n\\nEtiäinen has been a part of Finnish culture for centuries and is still widely believed in today. Folklorists study etiäinen to understand Finnish traditions and how they have changed over time.'],\n",
+ " dtype=object)"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "new_df_ai.iloc[[1]].Paragraph.values\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station l...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149995</th>\n",
+ " <td>Randy Borum is a Professor and Coordinator of ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149996</th>\n",
+ " <td>Sa'och (, also, \"Sauch\") is an endangered, nea...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149997</th>\n",
+ " <td>Philip C. Hanawalt (born 1931) is an American ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149998</th>\n",
+ " <td>Vossius Gymnasium is a public gymnasium in Ams...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149999</th>\n",
+ " <td>Simone Stratigo (, Symeon Filippos Stratigos; ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>300000 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station l... 1\n",
+ "1 In Finnish folklore, all places and things, an... 1\n",
+ "2 In mathematics, specifically differential calc... 1\n",
+ "3 is a Japanese shōjo manga series written and i... 1\n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... 1\n",
+ "... ... ...\n",
+ "149995 Randy Borum is a Professor and Coordinator of ... 0\n",
+ "149996 Sa'och (, also, \"Sauch\") is an endangered, nea... 0\n",
+ "149997 Philip C. Hanawalt (born 1931) is an American ... 0\n",
+ "149998 Vossius Gymnasium is a public gymnasium in Ams... 0\n",
+ "149999 Simone Stratigo (, Symeon Filippos Stratigos; ... 0\n",
+ "\n",
+ "[300000 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concat_df= pd.concat([new_df_ai, new_df_human])\n",
+ "concat_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station l...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station l... 1"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concat_df.iloc[[0000]]\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station b...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station b... 0"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concat_df.iloc[[150000]]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cdf_shffeled= concat_df.sample(frac=1).reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Sexhow railway station was a railway station l...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>In Finnish folklore, all places and things, an...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>In mathematics, specifically differential calc...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>is a Japanese shōjo manga series written and i...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>Robert Milner \"Rob\" Bradley, Jr. (born August ...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149995</th>\n",
+ " <td>Randy Borum is a Professor and Coordinator of ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149996</th>\n",
+ " <td>Sa'och (, also, \"Sauch\") is an endangered, nea...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149997</th>\n",
+ " <td>Philip C. Hanawalt (born 1931) is an American ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149998</th>\n",
+ " <td>Vossius Gymnasium is a public gymnasium in Ams...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>149999</th>\n",
+ " <td>Simone Stratigo (, Symeon Filippos Stratigos; ...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>300000 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Sexhow railway station was a railway station l... 1\n",
+ "1 In Finnish folklore, all places and things, an... 1\n",
+ "2 In mathematics, specifically differential calc... 1\n",
+ "3 is a Japanese shōjo manga series written and i... 1\n",
+ "4 Robert Milner \"Rob\" Bradley, Jr. (born August ... 1\n",
+ "... ... ...\n",
+ "149995 Randy Borum is a Professor and Coordinator of ... 0\n",
+ "149996 Sa'och (, also, \"Sauch\") is an endangered, nea... 0\n",
+ "149997 Philip C. Hanawalt (born 1931) is an American ... 0\n",
+ "149998 Vossius Gymnasium is a public gymnasium in Ams... 0\n",
+ "149999 Simone Stratigo (, Symeon Filippos Stratigos; ... 0\n",
+ "\n",
+ "[300000 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "concat_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>Paragraph</th>\n",
+ " <th>AI_generated</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>Bentivoglio is an Italian surname. Notable peo...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>Stephanie McCallum (born Sydney, Australia, 3...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>Abdāl lit: substitutes, but which can also be...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>The Gadget is a young adult historical novel w...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>The A-1 lifeboat was a powered lifeboat that w...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>299995</th>\n",
+ " <td>James Wrighten (b. 1745 - d. 1793) was an Amer...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>299996</th>\n",
+ " <td>The U-matrix (unified distance matrix) is a sy...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>299997</th>\n",
+ " <td>Holding Trevor is a 2007 American gay-themed p...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>299998</th>\n",
+ " <td>Jarvisfield is a heritage-listed former pastor...</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>299999</th>\n",
+ " <td>The Silver Guardian is a Chinese web series c...</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>300000 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " Paragraph AI_generated\n",
+ "0 Bentivoglio is an Italian surname. Notable peo... 0\n",
+ "1 Stephanie McCallum (born Sydney, Australia, 3... 0\n",
+ "2 Abdāl lit: substitutes, but which can also be... 1\n",
+ "3 The Gadget is a young adult historical novel w... 1\n",
+ "4 The A-1 lifeboat was a powered lifeboat that w... 0\n",
+ "... ... ...\n",
+ "299995 James Wrighten (b. 1745 - d. 1793) was an Amer... 1\n",
+ "299996 The U-matrix (unified distance matrix) is a sy... 1\n",
+ "299997 Holding Trevor is a 2007 American gay-themed p... 1\n",
+ "299998 Jarvisfield is a heritage-listed former pastor... 0\n",
+ "299999 The Silver Guardian is a Chinese web series c... 1\n",
+ "\n",
+ "[300000 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cdf_shffeled"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cdf_shffeled.to_csv(\"data/AI_checker_remade.csv\", index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.12"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}