{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "41296cc6-9d84-47c5-8a92-2d292f6f3c4a",
   "metadata": {
    "papermill": {
     "duration": 0.005559,
     "end_time": "2024-01-31T17:49:14.479844",
     "exception": false,
     "start_time": "2024-01-31T17:49:14.474285",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Module - Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9286e0b8-3c78-4b0f-943c-d219e9840dfe",
   "metadata": {
    "papermill": {
     "duration": 0.013698,
     "end_time": "2024-01-31T17:49:14.497101",
     "exception": false,
     "start_time": "2024-01-31T17:49:14.483403",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import to be able to import python package from src\n",
    "import sys\n",
    "sys.path.insert(0, '../src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2028eed7-b1c3-4c9e-b6a0-00433caa7d0f",
   "metadata": {
    "papermill": {
     "duration": 0.834479,
     "end_time": "2024-01-31T17:49:15.334389",
     "exception": false,
     "start_time": "2024-01-31T17:49:14.499910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from darts.datasets import EnergyDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4733b4e6-71a2-42b2-93fd-a5615b84ac1a",
   "metadata": {
    "papermill": {
     "duration": 1.661857,
     "end_time": "2024-01-31T17:49:16.997960",
     "exception": false,
     "start_time": "2024-01-31T17:49:15.336103",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ontime as on"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e24da8ab-6a83-4c2f-9ff0-c633d4693a91",
   "metadata": {
    "papermill": {
     "duration": 0.001706,
     "end_time": "2024-01-31T17:49:17.001465",
     "exception": false,
     "start_time": "2024-01-31T17:49:16.999759",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "---\n",
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e9a96d79-0423-4d79-b01d-726193216238",
   "metadata": {
    "papermill": {
     "duration": 0.070867,
     "end_time": "2024-01-31T17:49:17.073871",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.003004",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ts = EnergyDataset().load()\n",
    "ts = ts.astype(np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d4bec6b-eedb-4a88-ba68-dbeae5f0644e",
   "metadata": {
    "papermill": {
     "duration": 0.001603,
     "end_time": "2024-01-31T17:49:17.077199",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.075596",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2c873dd-8643-40cd-895b-fddd7a515c6d",
   "metadata": {
    "papermill": {
     "duration": 0.001605,
     "end_time": "2024-01-31T17:49:17.080662",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.079057",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Common Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a630af5c-687e-48e2-a6d4-5a8cb1d1ec66",
   "metadata": {
    "papermill": {
     "duration": 0.004556,
     "end_time": "2024-01-31T17:49:17.086770",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.082214",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from ontime.module import processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b508ee5-7c7e-4793-904e-45a40df354db",
   "metadata": {
    "papermill": {
     "duration": 0.001617,
     "end_time": "2024-01-31T17:49:17.089935",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.088318",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a4b12f07-8a97-403a-a554-89e166574120",
   "metadata": {
    "papermill": {
     "duration": 0.010295,
     "end_time": "2024-01-31T17:49:17.101761",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.091466",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/benjy/.cache/pypoetry/virtualenvs/ontime-2OQVvbNf-py3.10/lib/python3.10/site-packages/sklearn/utils/_array_api.py:472: RuntimeWarning: All-NaN slice encountered\n",
      "  return xp.asarray(numpy.nanmin(X, axis=axis))\n",
      "/home/benjy/.cache/pypoetry/virtualenvs/ontime-2OQVvbNf-py3.10/lib/python3.10/site-packages/sklearn/utils/_array_api.py:489: RuntimeWarning: All-NaN slice encountered\n",
      "  return xp.asarray(numpy.nanmax(X, axis=axis))\n"
     ]
    }
   ],
   "source": [
    "ts_t = processing.common.normalize(ts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42428ed1-7556-4341-9675-bad6dca0ecac",
   "metadata": {
    "papermill": {
     "duration": 0.001593,
     "end_time": "2024-01-31T17:49:17.105049",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.103456",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Train test split (for time series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8b67892d-db8c-4f12-93b6-147016da4186",
   "metadata": {
    "papermill": {
     "duration": 0.006516,
     "end_time": "2024-01-31T17:49:17.113217",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.106701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train, test = processing.common.train_test_split(ts_t, train_split=0.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498b0e13-04bc-45ee-ab1a-3996fbfd1df2",
   "metadata": {
    "papermill": {
     "duration": 0.001585,
     "end_time": "2024-01-31T17:49:17.116391",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.114806",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Split time series in windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "500e954a-82d6-4eff-bbdd-0b889c2a10f8",
   "metadata": {
    "papermill": {
     "duration": 3.177923,
     "end_time": "2024-01-31T17:49:20.295854",
     "exception": false,
     "start_time": "2024-01-31T17:49:17.117931",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_list = processing.common.split_in_windows(train, 100, 100)\n",
    "test_list = processing.common.split_in_windows(test, 100, 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4a88496-6b33-4bff-abb7-1d5ff4c81597",
   "metadata": {
    "papermill": {
     "duration": 0.001701,
     "end_time": "2024-01-31T17:49:20.299405",
     "exception": false,
     "start_time": "2024-01-31T17:49:20.297704",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Split in X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f7897c44-71ba-4752-86c6-547387245ae4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-31T17:49:20.303250Z",
     "iopub.status.busy": "2024-01-31T17:49:20.303124Z",
     "iopub.status.idle": "2024-01-31T17:49:27.551266Z",
     "shell.execute_reply": "2024-01-31T17:49:27.550811Z"
    },
    "papermill": {
     "duration": 7.251438,
     "end_time": "2024-01-31T17:49:27.552431",
     "exception": false,
     "start_time": "2024-01-31T17:49:20.300993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train, y_train = processing.common.split_inputs_from_targets(train_list, 70, 30)\n",
    "X_test, y_test = processing.common.split_inputs_from_targets(test_list, 70, 30)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9626370a-e4ba-4421-b40b-d6e7c5787beb",
   "metadata": {
    "papermill": {
     "duration": 0.001581,
     "end_time": "2024-01-31T17:49:27.555956",
     "exception": false,
     "start_time": "2024-01-31T17:49:27.554375",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Transform in generic data type "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a4ab9cfa-289d-4d8e-be40-d5d4247f5ab5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-31T17:49:27.559714Z",
     "iopub.status.busy": "2024-01-31T17:49:27.559596Z",
     "iopub.status.idle": "2024-01-31T17:49:28.917513Z",
     "shell.execute_reply": "2024-01-31T17:49:28.917038Z"
    },
    "papermill": {
     "duration": 1.36114,
     "end_time": "2024-01-31T17:49:28.918647",
     "exception": false,
     "start_time": "2024-01-31T17:49:27.557507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train = processing.common.timeseries_list_to_numpy(X_train)\n",
    "y_train = processing.common.timeseries_list_to_numpy(y_train)\n",
    "X_test = processing.common.timeseries_list_to_numpy(X_test)\n",
    "y_test = processing.common.timeseries_list_to_numpy(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1b0a2843-6d02-4b08-96f8-91712e521bf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-31T17:49:28.922874Z",
     "iopub.status.busy": "2024-01-31T17:49:28.922748Z",
     "iopub.status.idle": "2024-01-31T17:49:28.924710Z",
     "shell.execute_reply": "2024-01-31T17:49:28.924464Z"
    },
    "papermill": {
     "duration": 0.005064,
     "end_time": "2024-01-31T17:49:28.925555",
     "exception": false,
     "start_time": "2024-01-31T17:49:28.920491",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(280, 28, 70)\n",
      "(280, 28, 30)\n",
      "(70, 28, 70)\n",
      "(70, 28, 30)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 15.619503,
   "end_time": "2024-01-31T17:49:29.550784",
   "environment_variables": {},
   "exception": null,
   "input_path": "docs/user_guide/1_module/0-preprocessing/1.0-preprocessing-common.ipynb",
   "output_path": "docs/user_guide/1_module/0-preprocessing/1.0-preprocessing-common.ipynb",
   "parameters": {},
   "start_time": "2024-01-31T17:49:13.931281",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}