ignore

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+9086958007

bert.py

@@ -98,8 +98,8 @@ class BertLayer(nn.Module):
         ln_layer: layer norm that takes input+sublayer(output)
         This function computes ``LayerNorm(input + Sublayer(output))``, where sublayer is a dense_layer followed by dropout.
         """
-        # todo
-        raise NotImplementedError
+        sublayer_output = dropout(dense_layer(output))
+        return ln_layer(input + sublayer_output)
 
     def forward(self, hidden_states, attention_mask):
         """
@@ -111,16 +111,26 @@ class BertLayer(nn.Module):
         3. a feed forward layer
         4. a add-norm that takes the output of feed forward layer and the input of feed forward layer
         """
-        # todo
-        # multi-head attention w/ self.self_attention
-
-        # add-norm layer
-
-        # feed forward
-
-        # another add-norm layer
-
-        raise NotImplementedError
+        attn_output = self.self_attention(hidden_states, attention_mask)
+        hidden_states = self.add_norm(
+            hidden_states,
+            attn_output,
+            self.attention_dense,
+            self.attention_dropout,
+            self.attention_layer_norm,
+        )
+        interm = self.interm_dense(hidden_states)
+        interm = self.interm_af(interm)
+        # ffn_output = self.out_dense(interm)
+        hidden_states = self.add_norm(
+            hidden_states,
+            interm,
+            # ffn_output,
+            self.out_dense,
+            self.out_dropout,
+            self.out_layer_norm,
+        )
+        return hidden_states
 
 
 class BertModel(BertPreTrainedModel):
@@ -138,7 +148,8 @@ class BertModel(BertPreTrainedModel):
 
         # embedding
         self.word_embedding = nn.Embedding(
-            config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
+            config.vocab_size,
+            config.hidden_size,  # , padding_idx=config.pad_token_id
         )
         self.pos_embedding = nn.Embedding(
             config.max_position_embeddings, config.hidden_size
@@ -169,28 +180,20 @@ class BertModel(BertPreTrainedModel):
         input_shape = input_ids.size()
         seq_length = input_shape[1]
 
-        # get word embedding from self.word_embedding
-        # todo
-        inputs_embeds = None
+        inputs_embeds = self.word_embedding(input_ids)
 
-        # get position index and position embedding from self.pos_embedding
-        pos_ids = self.position_ids[:, :seq_length]
-        pos_embeds = None
+        pos_ids = self.position_ids[:, :seq_length].expand(input_shape)
+        pos_embeds = self.pos_embedding(pos_ids)
 
-        # get token type ids, since we are not consider token type, just a placeholder
-        tk_type_ids = torch.zeros(
-            input_shape, dtype=torch.long, device=input_ids.device
-        )
+        tk_type_ids = torch.zeros_like(input_ids)
         tk_type_embeds = self.tk_type_embedding(tk_type_ids)
 
-        # add three embeddings together
         embeds = inputs_embeds + tk_type_embeds + pos_embeds
 
-        # layer norm and dropout
         embeds = self.embed_layer_norm(embeds)
         embeds = self.embed_dropout(embeds)
 
-        raise NotImplementedError
+        return embeds
 
     def encode(self, hidden_states, attention_mask):
         """

classifier.py

@@ -4,7 +4,12 @@ from types import SimpleNamespace
 import torch
 import torch.nn.functional as F
 from torch.utils.data import Dataset, DataLoader
-from sklearn.metrics import classification_report, f1_score, recall_score, accuracy_score
+from sklearn.metrics import (
+    classification_report,
+    f1_score,
+    recall_score,
+    accuracy_score,
+)
 
 # change it with respect to the original model
 from tokenizer import BertTokenizer
@@ -13,7 +18,9 @@ from optimizer import AdamW
 from tqdm import tqdm
 
 
-TQDM_DISABLE=True
+TQDM_DISABLE = True
+
+
 # fix the random seed
 def seed_everything(seed=11711):
     random.seed(seed)
@@ -24,33 +31,37 @@ def seed_everything(seed=11711):
     torch.backends.cudnn.benchmark = False
     torch.backends.cudnn.deterministic = True
 
+
 class BertSentClassifier(torch.nn.Module):
     def __init__(self, config):
         super(BertSentClassifier, self).__init__()
         self.num_labels = config.num_labels
-        self.bert = BertModel.from_pretrained('bert-base-uncased')
+        self.bert = BertModel.from_pretrained("bert-base-uncased")
 
         # pretrain mode does not require updating bert paramters.
         for param in self.bert.parameters():
-            if config.option == 'pretrain':
+            if config.option == "pretrain":
                 param.requires_grad = False
-            elif config.option == 'finetune':
+            elif config.option == "finetune":
                 param.requires_grad = True
 
-        # todo
-        raise NotImplementedError
+        self.dropout = torch.nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = torch.nn.Linear(config.hidden_size, config.num_labels)
 
     def forward(self, input_ids, attention_mask):
-        # todo
-        # the final bert contextualize embedding is the hidden state of [CLS] token (the first token)
-        raise NotImplementedError
+        outputs = self.bert(input_ids, attention_mask)
+        pooled_output = outputs["pooler_output"]
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+        return F.log_softmax(logits, dim=-1)
+
 
 # create a custom Dataset Class to be used for the dataloader
 class BertDataset(Dataset):
     def __init__(self, dataset, args):
         self.dataset = dataset
         self.p = args
-        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 
     def __len__(self):
         return len(self.dataset)
@@ -62,10 +73,12 @@ class BertDataset(Dataset):
     def pad_data(self, data):
         sents = [x[0] for x in data]
         labels = [x[1] for x in data]
-        encoding = self.tokenizer(sents, return_tensors='pt', padding=True, truncation=True)
-        token_ids = torch.LongTensor(encoding['input_ids'])
-        attention_mask = torch.LongTensor(encoding['attention_mask'])
-        token_type_ids = torch.LongTensor(encoding['token_type_ids'])
+        encoding = self.tokenizer(
+            sents, return_tensors="pt", padding=True, truncation=True
+        )
+        token_ids = torch.LongTensor(encoding["input_ids"])
+        attention_mask = torch.LongTensor(encoding["attention_mask"])
+        token_type_ids = torch.LongTensor(encoding["token_type_ids"])
         labels = torch.LongTensor(labels)
 
         return token_ids, token_type_ids, attention_mask, labels, sents
@@ -78,30 +91,34 @@ class BertDataset(Dataset):
 
         for i in range(num_batches):
             start_idx = i * self.p.batch_size
-            data = all_data[start_idx: start_idx + self.p.batch_size]
+            data = all_data[start_idx : start_idx + self.p.batch_size]
 
-            token_ids, token_type_ids, attention_mask, labels, sents = self.pad_data(data)
-            batches.append({
-                'token_ids': token_ids,
-                'token_type_ids': token_type_ids,
-                'attention_mask': attention_mask,
-                'labels': labels,
-                'sents': sents,
-            })
+            token_ids, token_type_ids, attention_mask, labels, sents = self.pad_data(
+                data
+            )
+            batches.append(
+                {
+                    "token_ids": token_ids,
+                    "token_type_ids": token_type_ids,
+                    "attention_mask": attention_mask,
+                    "labels": labels,
+                    "sents": sents,
+                }
+            )
 
         return batches
 
 
 # create the data which is a list of (sentence, label, token for the labels)
-def create_data(filename, flag='train'):
+def create_data(filename, flag="train"):
     # specify the tokenizer
-    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
     num_labels = {}
     data = []
 
-    with open(filename, 'r') as fp:
+    with open(filename, "r") as fp:
         for line in fp:
-            label, org_sent = line.split(' ||| ')
+            label, org_sent = line.split(" ||| ")
             sent = org_sent.lower().strip()
             tokens = tokenizer.tokenize("[CLS] " + sent + " [SEP]")
             label = int(label.strip())
@@ -109,20 +126,26 @@ def create_data(filename, flag='train'):
                 num_labels[label] = len(num_labels)
             data.append((sent, label, tokens))
     print(f"load {len(data)} data from {filename}")
-    if flag == 'train':
+    if flag == "train":
         return data, len(num_labels)
     else:
         return data
 
+
 # perform model evaluation in terms of the accuracy and f1 score.
 def model_eval(dataloader, model, device):
-    model.eval() # switch to eval model, will turn off randomness like dropout
+    model.eval()  # switch to eval model, will turn off randomness like dropout
     y_true = []
     y_pred = []
     sents = []
-    for step, batch in enumerate(tqdm(dataloader, desc=f'eval', disable=TQDM_DISABLE)):
-        b_ids, b_type_ids, b_mask, b_labels, b_sents = batch[0]['token_ids'], batch[0]['token_type_ids'], \
-                                                       batch[0]['attention_mask'], batch[0]['labels'], batch[0]['sents']
+    for step, batch in enumerate(tqdm(dataloader, desc=f"eval", disable=TQDM_DISABLE)):
+        b_ids, b_type_ids, b_mask, b_labels, b_sents = (
+            batch[0]["token_ids"],
+            batch[0]["token_type_ids"],
+            batch[0]["attention_mask"],
+            batch[0]["labels"],
+            batch[0]["sents"],
+        )
 
         b_ids = b_ids.to(device)
         b_mask = b_mask.to(device)
@@ -136,46 +159,58 @@ def model_eval(dataloader, model, device):
         y_pred.extend(preds)
         sents.extend(b_sents)
 
-    f1 = f1_score(y_true, y_pred, average='macro')
+    f1 = f1_score(y_true, y_pred, average="macro")
     acc = accuracy_score(y_true, y_pred)
 
     return acc, f1, y_pred, y_true, sents
 
+
 def save_model(model, optimizer, args, config, filepath):
     save_info = {
-        'model': model.state_dict(),
-        'optim': optimizer.state_dict(),
-        'args': args,
-        'model_config': config,
-        'system_rng': random.getstate(),
-        'numpy_rng': np.random.get_state(),
-        'torch_rng': torch.random.get_rng_state(),
+        "model": model.state_dict(),
+        "optim": optimizer.state_dict(),
+        "args": args,
+        "model_config": config,
+        "system_rng": random.getstate(),
+        "numpy_rng": np.random.get_state(),
+        "torch_rng": torch.random.get_rng_state(),
     }
 
     torch.save(save_info, filepath)
     print(f"save the model to {filepath}")
 
+
 def train(args):
-    device = torch.device('cuda') if args.use_gpu else torch.device('cpu')
+    device = torch.device("cuda") if args.use_gpu else torch.device("cpu")
     #### Load data
     # create the data and its corresponding datasets and dataloader
-    train_data, num_labels = create_data(args.train, 'train')
-    dev_data = create_data(args.dev, 'valid')
+    train_data, num_labels = create_data(args.train, "train")
+    dev_data = create_data(args.dev, "valid")
 
     train_dataset = BertDataset(train_data, args)
     dev_dataset = BertDataset(dev_data, args)
 
-    train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=args.batch_size,
-                                  collate_fn=train_dataset.collate_fn)
-    dev_dataloader = DataLoader(dev_dataset, shuffle=False, batch_size=args.batch_size,
-                                collate_fn=dev_dataset.collate_fn)
+    train_dataloader = DataLoader(
+        train_dataset,
+        shuffle=True,
+        batch_size=args.batch_size,
+        collate_fn=train_dataset.collate_fn,
+    )
+    dev_dataloader = DataLoader(
+        dev_dataset,
+        shuffle=False,
+        batch_size=args.batch_size,
+        collate_fn=dev_dataset.collate_fn,
+    )
 
     #### Init model
-    config = {'hidden_dropout_prob': args.hidden_dropout_prob,
-              'num_labels': num_labels,
-              'hidden_size': 768,
-              'data_dir': '.',
-              'option': args.option}
+    config = {
+        "hidden_dropout_prob": args.hidden_dropout_prob,
+        "num_labels": num_labels,
+        "hidden_size": 768,
+        "data_dir": ".",
+        "option": args.option,
+    }
 
     config = SimpleNamespace(**config)
 
@@ -193,9 +228,16 @@ def train(args):
         model.train()
         train_loss = 0
         num_batches = 0
-        for step, batch in enumerate(tqdm(train_dataloader, desc=f'train-{epoch}', disable=TQDM_DISABLE)):
-            b_ids, b_type_ids, b_mask, b_labels, b_sents = batch[0]['token_ids'], batch[0]['token_type_ids'], batch[0][
-                'attention_mask'], batch[0]['labels'], batch[0]['sents']
+        for step, batch in enumerate(
+            tqdm(train_dataloader, desc=f"train-{epoch}", disable=TQDM_DISABLE)
+        ):
+            b_ids, b_type_ids, b_mask, b_labels, b_sents = (
+                batch[0]["token_ids"],
+                batch[0]["token_type_ids"],
+                batch[0]["attention_mask"],
+                batch[0]["labels"],
+                batch[0]["sents"],
+            )
 
             b_ids = b_ids.to(device)
             b_mask = b_mask.to(device)
@@ -203,7 +245,9 @@ def train(args):
 
             optimizer.zero_grad()
             logits = model(b_ids, b_mask)
-            loss = F.nll_loss(logits, b_labels.view(-1), reduction='sum') / args.batch_size
+            loss = (
+                F.nll_loss(logits, b_labels.view(-1), reduction="sum") / args.batch_size
+            )
 
             loss.backward()
             optimizer.step()
@@ -220,36 +264,52 @@ def train(args):
             best_dev_acc = dev_acc
             save_model(model, optimizer, args, config, args.filepath)
 
-        print(f"epoch {epoch}: train loss :: {train_loss :.3f}, train acc :: {train_acc :.3f}, dev acc :: {dev_acc :.3f}")
+        print(
+            f"epoch {epoch}: train loss :: {train_loss:.3f}, train acc :: {train_acc:.3f}, dev acc :: {dev_acc:.3f}"
+        )
 
 
 def test(args):
     with torch.no_grad():
-        device = torch.device('cuda') if args.use_gpu else torch.device('cpu')
+        device = torch.device("cuda") if args.use_gpu else torch.device("cpu")
         saved = torch.load(args.filepath)
-        config = saved['model_config']
+        config = saved["model_config"]
         model = BertSentClassifier(config)
-        model.load_state_dict(saved['model'])
+        model.load_state_dict(saved["model"])
         model = model.to(device)
         print(f"load model from {args.filepath}")
-        dev_data = create_data(args.dev, 'valid')
+        dev_data = create_data(args.dev, "valid")
         dev_dataset = BertDataset(dev_data, args)
-        dev_dataloader = DataLoader(dev_dataset, shuffle=False, batch_size=args.batch_size, collate_fn=dev_dataset.collate_fn)
+        dev_dataloader = DataLoader(
+            dev_dataset,
+            shuffle=False,
+            batch_size=args.batch_size,
+            collate_fn=dev_dataset.collate_fn,
+        )
 
-        test_data = create_data(args.test, 'test')
+        test_data = create_data(args.test, "test")
         test_dataset = BertDataset(test_data, args)
-        test_dataloader = DataLoader(test_dataset, shuffle=False, batch_size=args.batch_size, collate_fn=test_dataset.collate_fn)
+        test_dataloader = DataLoader(
+            test_dataset,
+            shuffle=False,
+            batch_size=args.batch_size,
+            collate_fn=test_dataset.collate_fn,
+        )
 
-        dev_acc, dev_f1, dev_pred, dev_true, dev_sents = model_eval(dev_dataloader, model, device)
-        test_acc, test_f1, test_pred, test_true, test_sents = model_eval(test_dataloader, model, device)
+        dev_acc, dev_f1, dev_pred, dev_true, dev_sents = model_eval(
+            dev_dataloader, model, device
+        )
+        test_acc, test_f1, test_pred, test_true, test_sents = model_eval(
+            test_dataloader, model, device
+        )
 
         with open(args.dev_out, "w+") as f:
-            print(f"dev acc :: {dev_acc :.3f}")
+            print(f"dev acc :: {dev_acc:.3f}")
             for s, t, p in zip(dev_sents, dev_true, dev_pred):
                 f.write(f"{s} ||| {t} ||| {p}\n")
 
         with open(args.test_out, "w+") as f:
-            print(f"test acc :: {test_acc :.3f}")
+            print(f"test acc :: {test_acc:.3f}")
             for s, t, p in zip(test_sents, test_true, test_pred):
                 f.write(f"{s} ||| {t} ||| {p}\n")
 
@@ -261,28 +321,42 @@ def get_args():
     parser.add_argument("--test", type=str, default="data/cfimdb-test.txt")
     parser.add_argument("--seed", type=int, default=11711)
     parser.add_argument("--epochs", type=int, default=10)
-    parser.add_argument("--option", type=str,
-                        help='pretrain: the BERT parameters are frozen; finetune: BERT parameters are updated',
-                        choices=('pretrain', 'finetune'), default="pretrain")
-    parser.add_argument("--use_gpu", action='store_true')
+    parser.add_argument(
+        "--option",
+        type=str,
+        help="pretrain: the BERT parameters are frozen; finetune: BERT parameters are updated",
+        choices=("pretrain", "finetune"),
+        default="pretrain",
+    )
+    parser.add_argument("--use_gpu", action="store_true")
     parser.add_argument("--dev_out", type=str, default="cfimdb-dev-output.txt")
     parser.add_argument("--test_out", type=str, default="cfimdb-test-output.txt")
     parser.add_argument("--filepath", type=str, default=None)
 
     # hyper parameters
-    parser.add_argument("--batch_size", help='sst: 64, cfimdb: 8 can fit a 12GB GPU', type=int, default=8)
+    parser.add_argument(
+        "--batch_size",
+        help="sst: 64, cfimdb: 8 can fit a 12GB GPU",
+        type=int,
+        default=8,
+    )
     parser.add_argument("--hidden_dropout_prob", type=float, default=0.3)
-    parser.add_argument("--lr", type=float, help="learning rate, default lr for 'pretrain': 1e-3, 'finetune': 1e-5",
-                        default=1e-5)
+    parser.add_argument(
+        "--lr",
+        type=float,
+        help="learning rate, default lr for 'pretrain': 1e-3, 'finetune': 1e-5",
+        default=1e-5,
+    )
 
     args = parser.parse_args()
     print(f"args: {vars(args)}")
     return args
 
+
 if __name__ == "__main__":
     args = get_args()
     if args.filepath is None:
-        args.filepath = f'{args.option}-{args.epochs}-{args.lr}.pt' # save path
+        args.filepath = f"{args.option}-{args.epochs}-{args.lr}.pt"  # save path
     seed_everything(args.seed)  # fix the seed for reproducibility
     train(args)
     test(args)

optimizer.py

@@ -6,23 +6,33 @@ from torch.optim import Optimizer
 
 class AdamW(Optimizer):
     def __init__(
-            self,
-            params: Iterable[torch.nn.parameter.Parameter],
-            lr: float = 1e-3,
-            betas: Tuple[float, float] = (0.9, 0.999),
-            eps: float = 1e-6,
-            weight_decay: float = 0.0,
-            correct_bias: bool = True,
+        self,
+        params: Iterable[torch.nn.parameter.Parameter],
+        lr: float = 1e-3,
+        betas: Tuple[float, float] = (0.9, 0.999),
+        eps: float = 1e-6,
+        weight_decay: float = 0.0,
+        correct_bias: bool = True,
     ):
         if lr < 0.0:
             raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr))
         if not 0.0 <= betas[0] < 1.0:
-            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0]))
+            raise ValueError(
+                "Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])
+            )
         if not 0.0 <= betas[1] < 1.0:
-            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1]))
+            raise ValueError(
+                "Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])
+            )
         if not 0.0 <= eps:
             raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps))
-        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, correct_bias=correct_bias)
+        defaults = dict(
+            lr=lr,
+            betas=betas,
+            eps=eps,
+            weight_decay=weight_decay,
+            correct_bias=correct_bias,
+        )
         super().__init__(params, defaults)
 
     def step(self, closure: Callable = None):
@@ -36,25 +46,39 @@ class AdamW(Optimizer):
                     continue
                 grad = p.grad.data
                 if grad.is_sparse:
-                    raise RuntimeError("Adam does not support sparse gradients, please consider SparseAdam instead")
+                    raise RuntimeError(
+                        "Adam does not support sparse gradients, please consider SparseAdam instead"
+                    )
 
-                raise NotImplementedError()
-
-                # State should be stored in this dictionary
                 state = self.state[p]
 
-                # Access hyperparameters from the `group` dictionary
-                alpha = group["lr"]
+                lr = group["lr"]
+                beta1, beta2 = group["betas"]
+                eps = group["eps"]
+                weight_decay = group["weight_decay"]
 
-                # Update first and second moments of the gradients
+                if len(state) == 0:
+                    state["exp_avg"] = torch.zeros_like(p.data)
+                    state["exp_avg_sq"] = torch.zeros_like(p.data)
+                    state["step"] = 0
 
-                # Bias correction
-                # Please note that we are using the "efficient version" given in
-                # https://arxiv.org/abs/1412.6980
+                exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
+                state["step"] += 1
+                step = state["step"]
 
-                # Update parameters
+                bias_correction1 = 1 - beta1**step
+                bias_correction2 = 1 - beta2**step
 
-                # Add weight decay after the main gradient-based updates.
-                # Please note that the learning rate should be incorporated into this update.
+                if bias_correction1 == 0:
+                    alpha_t = lr
+                else:
+                    alpha_t = lr * (bias_correction2**0.5) / bias_correction1
+
+                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+
+                denom = exp_avg_sq.sqrt().add_(eps)
+                p.data.addcdiv_(exp_avg, denom, value=-alpha_t)
+                p.data.mul_(1 - alpha_t * weight_decay)
 
         return loss

pyproject.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 requires-python = "==3.9.*"
 dependencies = [
     "torch==1.8.0",
+    "numpy<2",
     "tqdm==4.58.0",
     "requests==2.25.1",
     "importlib-metadata==3.7.0",

run_exp.sh

@@ -1,5 +1,5 @@
 # Step 0. Change this to your campus ID
-CAMPUSID='9xx1234567'
+CAMPUSID='9086958007'
 mkdir -p $CAMPUSID
 
 # Step 1. (Optional) Any preprocessing step, e.g., downloading pre-trained word embeddings

run_exp_uv.sh

@@ -0,0 +1,48 @@
+# Step 0. Change this to your campus ID
+CAMPUSID='9086958007'
+mkdir -p $CAMPUSID
+
+# Step 1. (Optional) Any preprocessing step, e.g., downloading pre-trained word embeddings
+
+
+# Step 2. Train models on two datasets.
+##  2.1. Run experiments on SST
+PREF='sst'
+uv run classifier.py \
+    #--use_gpu \
+    --option finetune \
+    --lr 1e-5 \
+    --seed 1234 \
+    --train "data/${PREF}-train.txt" \
+    --dev "data/${PREF}-dev.txt" \
+    --test "data/${PREF}-test.txt" \
+    --dev_out "${CAMPUSID}/${PREF}-dev-output.txt" \
+    --test_out "${CAMPUSID}/${PREF}-test-output.txt" \
+    --filepath "${CAMPUSID}/${PREF}-model.pt" | tee ${CAMPUSID}/${PREF}-train-log.txt
+
+##  2.2 Run experiments on CF-IMDB
+PREF='cfimdb'
+uv run classifier.py \
+    #--use_gpu \
+    --option finetune \
+    --lr 1e-5 \
+    --seed 1234 \
+    --train "data/${PREF}-train.txt" \
+    --dev "data/${PREF}-dev.txt" \
+    --test "data/${PREF}-test.txt" \
+    --dev_out "${CAMPUSID}/${PREF}-dev-output.txt" \
+    --test_out "${CAMPUSID}/${PREF}-test-output.txt" \
+    --filepath "${CAMPUSID}/${PREF}-model.pt" | tee ${CAMPUSID}/${PREF}-train-log.txt
+
+
+
+# Step 3. Prepare submission:
+##  3.1. Copy your code to the $CAMPUSID folder
+for file in *.py; do cp $file ${CAMPUSID}/ done
+for file in *.sh; do cp $file ${CAMPUSID}/ done
+for file in *.md; do cp $file ${CAMPUSID}/ done
+for file in *.txt; do cp $file ${CAMPUSID}/ done
+
+##  3.2. Compress the $CAMPUSID folder to $CAMPUSID.zip (containing only .py/.txt/.pdf/.sh files)
+python prepare_submit.py ${CAMPUSID} ${CAMPUSID}
+##  3.3. Submit the zip file to Canvas! Congrats!

uv.lock

@@ -9,6 +9,7 @@ source = { virtual = "." }
 dependencies = [
     { name = "filelock" },
     { name = "importlib-metadata" },
+    { name = "numpy" },
     { name = "requests" },
     { name = "scikit-learn" },
     { name = "tokenizers" },
@@ -20,6 +21,7 @@ dependencies = [
 requires-dist = [
     { name = "filelock", specifier = "==3.0.12" },
     { name = "importlib-metadata", specifier = "==3.7.0" },
+    { name = "numpy", specifier = "<2" },
     { name = "requests", specifier = "==2.25.1" },
     { name = "scikit-learn" },
     { name = "tokenizers", specifier = "==0.10.1" },
@@ -86,24 +88,21 @@ wheels = [
 
 [[package]]
 name = "numpy"
-version = "2.0.2"
+version = "1.26.4"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942, upload-time = "2024-08-26T20:14:40.108Z" },
-    { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512, upload-time = "2024-08-26T20:15:00.985Z" },
-    { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976, upload-time = "2024-08-26T20:15:10.876Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494, upload-time = "2024-08-26T20:15:22.055Z" },
-    { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596, upload-time = "2024-08-26T20:15:42.452Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099, upload-time = "2024-08-26T20:16:11.048Z" },
-    { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823, upload-time = "2024-08-26T20:16:40.171Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424, upload-time = "2024-08-26T20:17:02.604Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809, upload-time = "2024-08-26T20:17:13.553Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314, upload-time = "2024-08-26T20:17:36.72Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288, upload-time = "2024-08-26T20:18:07.732Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793, upload-time = "2024-08-26T20:18:19.125Z" },
-    { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885, upload-time = "2024-08-26T20:18:47.237Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784, upload-time = "2024-08-26T20:19:11.19Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/24/ce71dc08f06534269f66e73c04f5709ee024a1afe92a7b6e1d73f158e1f8/numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c", size = 20636301, upload-time = "2024-02-05T23:59:10.976Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/8c/ab03a7c25741f9ebc92684a20125fbc9fc1b8e1e700beb9197d750fdff88/numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be", size = 13971216, upload-time = "2024-02-05T23:59:35.472Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/64/c3bcdf822269421d85fe0d64ba972003f9bb4aa9a419da64b86856c9961f/numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764", size = 14226281, upload-time = "2024-02-05T23:59:59.372Z" },
+    { url = "https://files.pythonhosted.org/packages/54/30/c2a907b9443cf42b90c17ad10c1e8fa801975f01cb9764f3f8eb8aea638b/numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", size = 18249516, upload-time = "2024-02-06T00:00:32.79Z" },
+    { url = "https://files.pythonhosted.org/packages/43/12/01a563fc44c07095996d0129b8899daf89e4742146f7044cdbdb3101c57f/numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd", size = 13882132, upload-time = "2024-02-06T00:00:58.197Z" },
+    { url = "https://files.pythonhosted.org/packages/16/ee/9df80b06680aaa23fc6c31211387e0db349e0e36d6a63ba3bd78c5acdf11/numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c", size = 18084181, upload-time = "2024-02-06T00:01:31.21Z" },
+    { url = "https://files.pythonhosted.org/packages/28/7d/4b92e2fe20b214ffca36107f1a3e75ef4c488430e64de2d9af5db3a4637d/numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6", size = 5976360, upload-time = "2024-02-06T00:01:43.013Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/42/054082bd8220bbf6f297f982f0a8f5479fcbc55c8b511d928df07b965869/numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea", size = 15814633, upload-time = "2024-02-06T00:02:16.694Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/72/3df6c1c06fc83d9cfe381cccb4be2532bbd38bf93fbc9fad087b6687f1c0/numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30", size = 20455961, upload-time = "2024-02-06T00:03:05.993Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/02/570545bac308b58ffb21adda0f4e220ba716fb658a63c151daecc3293350/numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c", size = 18061071, upload-time = "2024-02-06T00:03:41.5Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/5f/fafd8c51235f60d49f7a88e2275e13971e90555b67da52dd6416caec32fe/numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0", size = 15709730, upload-time = "2024-02-06T00:04:11.719Z" },
 ]
 
 [[package]]