course/videos/tf_predictions.ipynb

{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "This notebook regroups the code sample of the video below, which is a part of the [Hugging Face course](https://huggingface.co/course)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form" }, "outputs": [ { "data": { "text/html": [ "<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/nx10eh4CoOs?rel=0&controls=0&showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>" ], "text/plain": [ "<IPython.core.display.HTML object>" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#@title\n", "from IPython.display import HTML\n", "\n", "HTML('<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/nx10eh4CoOs?rel=0&controls=0&showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Install the Transformers and Datasets libraries to run this notebook." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "! pip install datasets transformers[sentencepiece]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Reusing dataset glue (/home/sgugger/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n" ] } ], "source": [ "from datasets import load_dataset\n", "from transformers import AutoTokenizer\n", "import numpy as np\n", "\n", "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n", "checkpoint = \"bert-base-uncased\"\n", "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n", "\n", "def tokenize_dataset(dataset):\n", " encoded = tokenizer(\n", " dataset[\"sentence1\"],\n", " dataset[\"sentence2\"],\n", " padding=True,\n", " truncation=True,\n", " return_tensors='np',\n", " )\n", " return encoded.data\n", "\n", "tokenized_datasets = {\n", " split: tokenize_dataset(raw_datasets[split]) for split in raw_datasets.keys()\n", "}\n", "train_tokens = tokenized_datasets['train']['input_ids']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n", "\n", "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "import tensorflow as tf\n", "from transformers import TFAutoModelForSequenceClassification\n", "\n", "checkpoint = 'bert-base-cased'\n", "model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n", "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from tensorflow.keras.optimizers.schedules import PolynomialDecay\n", "\n", "batch_size = 8\n", "num_epochs = 3\n", "num_train_steps = (len(train_tokens) // batch_size) * num_epochs\n", "lr_scheduler = PolynomialDecay(\n", " initial_learning_rate=5e-5,\n", " end_learning_rate=0.,\n", " decay_steps=num_train_steps\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from tensorflow.keras.optimizers import Adam\n", "\n", "opt = Adam(learning_rate=lr_scheduler)\n", "model.compile(loss=loss, optimizer=opt)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/3\n", "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n", "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f5b279ce050>> and will run it as-is.\n", "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n", "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n", "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f5b279ce050>> and will run it as-is.\n", "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n", "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n", "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n", "WARNING:tensorflow:From /home/sgugger/.pyenv/versions/3.7.9/envs/base/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:5049: calling gather (from tensorflow.python.ops.array_ops) with validate_indices is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "The `validate_indices` argument has no effect. Indices are always validated on CPU and never validated on GPU.\n", "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n", "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n", "459/459 [==============================] - ETA: 0s - loss: 0.6243WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n", "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n", "459/459 [==============================] - 49s 81ms/step - loss: 0.6243 - val_loss: 0.5969\n", "Epoch 2/3\n", "459/459 [==============================] - 36s 79ms/step - loss: 0.5407 - val_loss: 0.5179\n", "Epoch 3/3\n", "459/459 [==============================] - 36s 79ms/step - loss: 0.3405 - val_loss: 0.5317\n" ] }, { "data": { "text/plain": [ "<tensorflow.python.keras.callbacks.History at 0x7f58507d3410>" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.fit(\n", " tokenized_datasets['train'],\n", " np.array(raw_datasets['train']['label']), \n", " validation_data=(tokenized_datasets['validation'], np.array(raw_datasets['validation']['label'])),\n", " batch_size=8,\n", " epochs=3\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n", "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n" ] } ], "source": [ "preds = model.predict(tokenized_datasets['validation'])['logits']\n", "probabilities = tf.nn.softmax(preds)\n", "class_preds = np.argmax(probabilities, axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.7549019607843137, 'f1': 0.8371335504885994}" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from datasets import load_metric\n", "\n", "metric = load_metric(\"glue\", \"mrpc\")\n", "metric.compute(predictions=class_preds, references=raw_datasets['validation']['label'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "colab": { "name": "TensorFlow Predictions and metrics", "provenance": [] } }, "nbformat": 4, "nbformat_minor": 4 }

course/videos/tf_predictions.ipynb (261 lines of code) (raw):