huggingface_hub/upload_hf_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Upload_hf_model.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/huggingface/notebooks/blob/upload_hf_model/huggingface_hub/upload_hf_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mpyOZpyNbe4s",
        "outputId": "f0ebe61d-f355-4d32-8f8d-627856add2c0"
      },
      "source": [
        "!pip install transformers\n",
        "!sudo apt-get install git-lfs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (4.1.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tokenizers==0.9.4 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "git-lfs is already the newest version (2.3.4-1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 15 not upgraded.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XxYs3FFjfZgg"
      },
      "source": [
        "!git config --global user.email \"julien@huggingface.co\"\n",
        "!git config --global user.name \"Julien Chaumond\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEixzjPibn_t"
      },
      "source": [
        "!transformers-cli login"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rirFL2oEcBxo",
        "outputId": "e45a2a89-45a0-427e-baff-c18e07aa1278"
      },
      "source": [
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxKVjZBfcKdr",
        "outputId": "2da79381-12ce-486c-c7d8-96e54ea9ed23"
      },
      "source": [
        "!transformers-cli repo create policy-distilbert-7d"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-12-26 10:02:28.904892: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
            "\u001b[90mgit version 2.17.1\u001b[0m\n",
            "Error: unknown flag: --version\n",
            "\n",
            "\u001b[90mSorry, no usage text found for \"git-lfs\"\u001b[0m\n",
            "\n",
            "You are about to create \u001b[1mjulien-c/policy-distilbert-7d\u001b[0m\n",
            "Proceed? [Y/n] Y\n",
            "\n",
            "Your repo now lives at:\n",
            "  \u001b[1mhttps://huggingface.co/julien-c/policy-distilbert-7d\u001b[0m\n",
            "\n",
            "You can clone it locally with the command below, and commit/push as usual.\n",
            "\n",
            "  git clone https://huggingface.co/julien-c/policy-distilbert-7d\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5R6NQVkLcN4v",
        "outputId": "b767deb5-4966-4e82-853f-6d65300d4c5c"
      },
      "source": [
        "!git clone https://julien-c:...token...@huggingface.co/julien-c/policy-distilbert-7d"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'policy-distilbert-7d'...\n",
            "remote: Enumerating objects: 3, done.\u001b[K\n",
            "remote: Counting objects: 100% (3/3), done.\u001b[K\n",
            "remote: Compressing objects: 100% (2/2), done.\u001b[K\n",
            "remote: Total 3 (delta 0), reused 0 (delta 0)\u001b[K\n",
            "Unpacking objects: 100% (3/3), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iS_wSC-IcklY",
        "outputId": "0caf0b61-58fc-4892-bee3-6ea30fb6531f"
      },
      "source": [
        "!ls -al"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 20\n",
            "drwxr-xr-x 1 root root 4096 Dec 26 10:03 .\n",
            "drwxr-xr-x 1 root root 4096 Dec 26 09:42 ..\n",
            "drwxr-xr-x 1 root root 4096 Dec 21 17:29 .config\n",
            "drwxr-xr-x 3 root root 4096 Dec 26 10:03 policy-distilbert-7d\n",
            "drwxr-xr-x 1 root root 4096 Dec 21 17:29 sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1zYuepDccmgH",
        "outputId": "07af3a37-56c0-4215-b8d6-03ff89944f79"
      },
      "source": [
        "%cd policy-distilbert-7d"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/policy-distilbert-7d\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dm8Ffkpmc9ui",
        "outputId": "5024777a-ebdd-4f70-e67b-9c0cb6490be4"
      },
      "source": [
        "!wget https://huggingface.co/distilbert-base-uncased/resolve/main/config.json\n",
        "!wget https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-26 10:03:55--  https://huggingface.co/distilbert-base-uncased/resolve/main/config.json\n",
            "Resolving huggingface.co (huggingface.co)... 34.201.172.85\n",
            "Connecting to huggingface.co (huggingface.co)|34.201.172.85|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 442 [application/json]\n",
            "Saving to: ‘config.json’\n",
            "\n",
            "\rconfig.json           0%[                    ]       0  --.-KB/s               \rconfig.json         100%[===================>]     442  --.-KB/s    in 0s      \n",
            "\n",
            "2020-12-26 10:03:55 (56.7 MB/s) - ‘config.json’ saved [442/442]\n",
            "\n",
            "--2020-12-26 10:03:55--  https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin\n",
            "Resolving huggingface.co (huggingface.co)... 34.201.172.85\n",
            "Connecting to huggingface.co (huggingface.co)|34.201.172.85|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/distilbert-base-uncased/e60d71610916da4787c5513c81bc026d415708528295502fb3e1a6fe1485ea7c [following]\n",
            "--2020-12-26 10:03:55--  https://cdn-lfs.huggingface.co/distilbert-base-uncased/e60d71610916da4787c5513c81bc026d415708528295502fb3e1a6fe1485ea7c\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 13.226.49.106, 13.226.49.46, 13.226.49.103, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|13.226.49.106|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 267967963 (256M) [application/macbinary]\n",
            "Saving to: ‘pytorch_model.bin’\n",
            "\n",
            "pytorch_model.bin   100%[===================>] 255.55M  87.1MB/s    in 2.9s    \n",
            "\n",
            "2020-12-26 10:03:58 (87.1 MB/s) - ‘pytorch_model.bin’ saved [267967963/267967963]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-y-GHzOigVjC",
        "outputId": "febf16e7-2cda-4730-e44e-b354e9c2308a"
      },
      "source": [
        "!git lfs install"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Updated git hooks.\n",
            "Git LFS initialized.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4GEDLShhdAe8"
      },
      "source": [
        "!git add ."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "peUV0eaYdOxn",
        "outputId": "7f0ad732-3786-4f11-be2e-c16bddfbbc8f"
      },
      "source": [
        "!git commit -m \"from Google Colab\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[main 41a7c98] from Google Colab\n",
            " 2 files changed, 24 insertions(+)\n",
            " create mode 100644 config.json\n",
            " create mode 100644 pytorch_model.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FbeYiBCAdbAt",
        "outputId": "6647c7f8-5d04-4555-9a92-c59e3e9fad87"
      },
      "source": [
        "!git log"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[33mcommit 41a7c98f1285a7e5ef19095dab11f0ac71ac1406\u001b[m\u001b[33m (\u001b[m\u001b[1;36mHEAD -> \u001b[m\u001b[1;32mmain\u001b[m\u001b[33m)\u001b[m\n",
            "Author: Julien Chaumond <julien@huggingface.co>\n",
            "Date:   Sat Dec 26 10:04:20 2020 +0000\n",
            "\n",
            "    from Google Colab\n",
            "\n",
            "\u001b[33mcommit 4630180ab4844ecc97c7f187c3d5bc54a568fac0\u001b[m\u001b[33m (\u001b[m\u001b[1;31morigin/main\u001b[m\u001b[33m, \u001b[m\u001b[1;31morigin/HEAD\u001b[m\u001b[33m)\u001b[m\n",
            "Author: system <system@huggingface.co>\n",
            "Date:   Sat Dec 26 10:02:45 2020 +0000\n",
            "\n",
            "    initial commit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y08-nstXdqPH",
        "outputId": "d383d262-d6da-49c3-e7e5-7fc65255996a"
      },
      "source": [
        "!git push"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Git LFS: (1 of 1 files) 255.55 MB / 255.55 MB\n",
            "Counting objects: 4, done.\n",
            "Delta compression using up to 4 threads.\n",
            "Compressing objects: 100% (4/4), done.\n",
            "Writing objects: 100% (4/4), 712 bytes | 712.00 KiB/s, done.\n",
            "Total 4 (delta 0), reused 0 (delta 0)\n",
            "To https://huggingface.co/julien-c/policy-distilbert-7d\n",
            "   4630180..41a7c98  main -> main\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pp0W8O1gggpZ"
      },
      "source": [
        "## Check out resulting commit: https://huggingface.co/julien-c/policy-distilbert-7d/commit/41a7c98f1285a7e5ef19095dab11f0ac71ac1406"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBQPu2kjgplw"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}