{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SPIRE-NEP master catalogue\n",
    "\n",
    "This notebook presents the merge of the various pristine catalogues to produce HELP mater catalogue on SPIRE-NEP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook was run with herschelhelp_internal version: \n",
      "1407877 (Mon Feb 4 12:56:29 2019 +0000)\n",
      "This notebook was executed on: \n",
      "2019-10-31 17:22:08.233223\n"
     ]
    }
   ],
   "source": [
    "from herschelhelp_internal import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n",
    "import datetime\n",
    "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/seaborn/apionly.py:9: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "#%config InlineBackend.figure_format = 'svg'\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord\n",
    "from astropy.table import Column, Table\n",
    "import numpy as np\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n",
    "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n",
    "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
    "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
    "\n",
    "try:\n",
    "    os.makedirs(OUT_DIR)\n",
    "except FileExistsError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I - Reading the prepared pristine catalogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps1 = Table.read(\"{}/PS1.fits\".format(TMP_DIR))\n",
    "seip = Table.read(\"{}/SEIP.fits\".format(TMP_DIR))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II - Merging tables\n",
    "\n",
    "We first merge the optical catalogues and then add the infrared ones: DECaLS, HSC, KIDS, PanSTARRS, UKIDSS-LAS, and VISTA-VIKING.\n",
    "\n",
    "At every step, we look at the distribution of the distances separating the sources from one catalogue to the other (within a maximum radius) to determine the best cross-matching radius."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PanSTARRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue = ps1\n",
    "master_catalogue['ps1_ra'].name = 'ra'\n",
    "master_catalogue['ps1_dec'].name = 'dec'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SEIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8XOV95/HPb+6SRpZkSb7JNgJjINxJDIGwWWiapuSyYbtNCqFN22y7JNmkSdpsL+l2093sP71tmk1IS8llc2lIaQmvLEkp5AYhNNzMNRhj1jbGyFfZlmRJo7k/+8c5IwYhWSPpjEZz5vt+vebl0czROc9g/NVPv/Oc55hzDhERCZdIowcgIiLBU7iLiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREIo16sB9fX1ucHCwUYcXEWlKjz322DHnXP982zUs3AcHB9m+fXujDi8i0pTM7MVatlNbRkQkhBTuIiIhpHAXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQmjfczWyTmd1rZjvNbIeZfXSWba42szEze9J/fLI+wxURkVrUchFTEfi4c+5xM+sEHjOz7zvnnp2x3U+cc+8IfogiIrJQ84a7c+4QcMh/Pm5mO4EBYGa4N42vP/gihVKZVDz6itdveP3mBo1IRCRYC+q5m9kgcAnw8CxvX2FmT5nZv5jZeQGMrW7ue/4on/nB85Sda/RQRETqouZwN7M08C3gY865kzPefhw4zTl3EfA54Ntz7ONGM9tuZtuHh4cXO+YlOzyW5WS2yNHxXMPGICJSTzWFu5nF8YL9G865O2a+75w76Zyb8J/fBcTNrG+W7W5xzm1zzm3r7593UbO6GZsqAPDS8UzDxiAiUk+1zJYx4EvATufcp+fYZp2/HWZ2mb/f40EONEhjGS/cXzyhcBeRcKpltsyVwHuBn5nZk/5rfwxsBnDO3Qy8C/igmRWBKeB651ZmQztXLDGeKwKwX+EuIiFVy2yZBwCbZ5ubgJuCGlQ9HR7LAtCXTnJsIkcmV6Q92bBl7UVE6qLlrlA9OOqF+4UbuwDYP6LqXUTCpwXDfQqA8zasImKwXydVRSSEWjbc+9JJ1ne1qe8uIqHUeuE+lqUjGSMejbBpdTsvjWQolVfkuV8RkUVrvXAfnaK7LQ7AaavbKZQcR05mGzwqEZFgtWS4d/nhPtDdBsBhhbuIhExLhbtzzqvc271w70x5UyAnssVGDktEJHAtFe4ns0Um86Xpyj0RixCPGhM5hbuIhEtLhXtlpkx3ewIAMyOdjCncRSR0WircD4354e5X7gCdqTjj2UKjhiQiUhctFe4H/KtTu6rCXZW7iIRR6BdVufXh/dPPf/DsEaJmpFMvf+x0Ksa+45ONGJqISN20VOU+NlVgVVuMiL28DlpnMkYmX9KFTCISKi0V7qOZPF1tiVe8VqniJ9WaEZEQaa1wnypMz3Gv6PSX+x1XuItIiLRUuE9ki9MXLlWkk7qQSUTCp2XCvVR2FMuOROyVHzmd8ir5iZymQ4pIeLRMuBdKZQCS0RnhrspdREKoZcI9X/TCPT6jck/EIiRjEfXcRSRUWi7ck7FXf+R0Msa4KncRCZHWCXe/LZOIzhLuKV2lKiLh0jLhnvMr90Qs+qr3OpMx9dxFJFRaJtwrJ1RnzpYBVe4iEj4tE+7TlftsbZlknKlCiVyxtNzDEhGpi5YJ90Jx7sq9cpXqsYn8so5JRKReWibcc/O0ZQCOjeeWdUwiIvXSMuFeOEVbprIkwbDCXURComXCPVcsY0A8aq96Lz3dllG4i0g4tEy4F0pl4rEIZnOHuyp3EQmLlgn3XLE8a0sGIBaNkIpHGFblLiIh0TLhni+WZj2ZWtGZjKstIyKh0TrhXnJzVu7gzZg5Nq6pkCISDvOGu5ltMrN7zWynme0ws4/Oso2Z2WfNbLeZPW1mr63PcBdvvsq9PRHlREbhLiLhUEvlXgQ+7px7DXA58CEzO3fGNm8FtvqPG4G/DXSUAcgXy7OuCFnRkYgxMqlwF5FwmDfcnXOHnHOP+8/HgZ3AwIzNrgW+5jwPAd1mtj7w0S5BvlQmfoq2TEcyykgmT7nslnFUIiL1saCeu5kNApcAD894awB4qerrIV79A6Ch5qvc2xMxyg7GpnS7PRFpfjWHu5mlgW8BH3POnZz59izf8qoS2MxuNLPtZrZ9eHh4YSNdonyx/Kq7MFXrSHpLAavvLiJhUFO4m1kcL9i/4Zy7Y5ZNhoBNVV9vBA7O3Mg5d4tzbptzblt/f/9ixrto+VL5VfdPrdae8C5kUt9dRMKgltkyBnwJ2Omc+/Qcm90J/Lo/a+ZyYMw5dyjAcS5J2TkKJTdP5e6F+3GFu4iEQKyGba4E3gv8zMye9F/7Y2AzgHPuZuAu4G3AbiADvC/4oS5e5UYdp54t47VlVLmLSBjMG+7OuQeYvadevY0DPhTUoIJWuTn2qWbLVNoy6rmLSBi0xBWqlXA/VeWeiEVoi0dVuYtIKLRGuJfmr9wBVnck1HMXkVBojXCvoXIH6OmIq3IXkVBoqXA/1doyAD3tCU5kdBGTiDS/lgj3XI3h3tuRUOUuIqHQEuFemQp5qiV/AXo6EpxQuItICLREuNdaua9uTzCRK5IrlpZjWCIiddMS4T5duc97QjUBwKj67iLS5Foi3HM1XMQEXs8dUGtGRJpeS4R7oVgmHjUidsoLbacrd4W7iDS7lgj3XKk878lU8C5iAoW7iDS/lgj3QrE8b78dvHnuACNaX0ZEmlxLhHuu5nCPA3B8QuEuIs2tJcK9UGNbJhaN0NUWV+UuIk2vJcI9VyyTjEVr2na1LmQSkRBoiXAvlE59/9RqPe2q3EWk+bVEuHuVe20fdXVHUj13EWl6LRHu3jz3WsNdlbuINL+WCPdcqfbKvacjwchkAe/OgSIizSn04e6cW1Dl3tuRIF8qM5nX4mEi0rxCH+6FksMx/12YKioXMp1Q311Emljow336/qk1n1D1w119dxFpYuEP98r9U2tsy1QWD9MdmUSkmbVMuNdauWvZXxEJg/CHu9+WWchsGVC4i0hzC3+4F2u7f2pFZzJGLGLquYtIU2udcK+xcjczf667wl1Emles0QOot0pbppbK/daH9wMQNePpobHpr294/eb6DVBEpA5CX7kXFjgVEqA9EWUyV6zXkERE6q5lwr3WnjtAezKmK1RFpKmFP9wrUyGjp745drWORJRMXpW7iDSv0Id7vuQwIBpZQLgnY0zlS5S1eJiINKl5w93MvmxmR83smTnev9rMxszsSf/xyeCHuXiFkrdomFnt4d6eiOKAjFozItKkaqncvwJcM882P3HOXew/PrX0YQXHC/fagx2gI+FNIsropKqINKl5w905dz9wYhnGUhcLucVeRUfSC3edVBWRZhVUz/0KM3vKzP7FzM4LaJ+ByJdczWu5V7QnvJtpazqkiDSrIC5iehw4zTk3YWZvA74NbJ1tQzO7EbgRYPPm5bkwqFAsL2gaJLxcuavnLiLNasmVu3PupHNuwn9+FxA3s745tr3FObfNObetv79/qYeuyWJ67pXKXdMhRaRZLTnczWyd+VNRzOwyf5/Hl7rfoFRmyyxEPBohEYuoLSMiTWvetoyZfRO4GugzsyHgT4E4gHPuZuBdwAfNrAhMAde7FXR36ULJ0Zla+M+wjkRUJ1RFpGnNG+7OuffM8/5NwE2BjShg+VK55hUhq7UnYmrLiEjTCv0VqovpuQN0JKNM5lS5i0hzapFwX0xbRpW7iDSv8Id7ceHz3KGy7K8qdxFpTqEO90KpTMktLtw7kjHypfL0ksEiIs0k1OGeLXiVd2IRPff2hC5kEpHmFfJw96ru2KIqdy1BICLNK+ThXqncFzcVEmBSJ1VFpAmFOtyn/HBf6KqQ4F3EBJDRSVURaULhDne/X764ee6q3EWkeYU73CuV+yLaMm2JKAaaDikiTaklwn0xPfeImT/XXZW7iDSfUId7Nr/4yh0gnYoxoXAXkSYU6nB/uS2z8J47QDoZYzxbCHJIIiLLojXCfRGzZcALd1XuItKMwh3u+cX33EHhLiLNK9Thnl3CbBmAdCpOoeR0UlVEmk6ow32qUCJiEI0svucOcGwiF+SwRETqLtzhnl/cWu4VnSmFu4g0p3CHe6G06H47vFy5D4/ngxqSiMiyCHW4ZwslYoucBglV4a7KXUSaTKjDfSpfWlJbprK+zLFxhbuINJdwh3uhRGKRc9zBOxHbnoiq5y4iTSf04b6Uyh281ozCXUSaTajDPVsoLXrpgQov3HVCVUSaSwuE+xIr95QqdxFpPqEO96VOhQToTMZ0QlVEmk64w32JFzGB15aZzJfI6I5MItJEQh3ugfTcK1ep6kImEWkioQ1355w3W2YJUyFBFzKJSHMKbbgXSo5S2S25555OxgGtLyMizSW04b6Um2NXS2vxMBFpQqEN96Wu5V4xveyveu4i0kTmTT4z+7KZHTWzZ+Z438zss2a228yeNrPXBj/MhZvKL+3+qRXRiNHdHlflLiJNpZay9ivANad4/63AVv9xI/C3Sx/W0gXVlgHoSycV7iLSVOZNPufc/cCJU2xyLfA153kI6Daz9UENcLGCDfcEw7qQSUSaSBA99wHgpaqvh/zXGipbacvEltaWAVXuItJ8ggj32dLTzbqh2Y1mtt3Mtg8PDwdw6LlVKvelToUEWNOZ4uh4Dudm/VgiIitOEOE+BGyq+nojcHC2DZ1ztzjntjnntvX39wdw6LkF2ZbZ0J0iky8xmikseV8iIsshiHC/E/h1f9bM5cCYc+5QAPtdkpdnyyz9I27saQPgwOjUkvclIrIcYvNtYGbfBK4G+sxsCPhTIA7gnLsZuAt4G7AbyADvq9dgF+Llee5L77kPdLcDMDQyxfkDXUven4hIvc0b7s6598zzvgM+FNiIAhJkz31AlbuINJnQXqE6lS8DEAsg3Hva47TFoxwYUbiLSHMIb7j7N+qIRpbeljEzBnraODCaCWBkIiL1F9pwzxZKpOLBfbyB7ja1ZUSkaYQ63NsS0cD2N9DTpraMiDSN0Ib7VKFEWzzAcO9uYyRT0O32RKQphDfc8yVSAYb79Fx3Ve8i0gTCG+5Bt2W6vXAfUt9dRJpAaMM9kw+4LaPKXUSaSGjDfSJbpDM17zVaNVvTmSIWMc2YEZGmEN5wzxWnb24dhGjEWN+dUuUuIk0htOE+ni0EWrmD5rqLSPMIZbg75/zKPehwb1flLiJNIZThnsmXKDuCr9x72jgyniVfLAe6XxGRoIUy3Cdy3oVG6YDC/daH93Prw/sZOpHBObjl/r2B7FdEpF5CGe7jWT/cA27LdLcnABjJ5APdr4hI0EIZ7pXKPei2TG+HF+66WbaIrHThDPfpyj24qZAA3e1xkrEIR05mA92viEjQQhnu41nvRtZBt2XMjLWrUhweU7iLyMoWznCvU1sGYF1XisMns3h3FxQRWZlCGe6Vtkxdwn1VimyhzCFV7yKygoUz3P3KvSPgtgx44Q7w3OGTge9bRCQooQ33VDxCPICbY8+0rqsS7uOB71tEJCihDPfxbLCLhlVLxaN0t8d57pDCXURWrpCGe/CLhlVbtyrFLlXuIrKChTLcJ3LBruU+07pVKfYMT2iNGRFZscIZ7tngV4SstrYrRbHs2DM8UbdjiIgsRTjDvQ7L/VbTjBkRWelCGe7j2WJgK0LOpi+dJBGN6KSqiKxYoQz3iVyRzjpW7tGIsXVtmh0HVbmLyMoUunCfvgtTHSt3gEsHV/PYiyM6qSoiK1Lown2qUKJUdnSm6jPPveLyM3qZKpR4emi0rscREVmM0IX7RJ1u1DHT5Wesxgwe3HO8rscREVmMmsLdzK4xs11mttvM/miW93/TzIbN7En/8dvBD7U29VwRslp3e4LXrFvFTxXuIrICzRvuZhYFPg+8FTgXeI+ZnTvLprc55y72H18MeJw1W67KHeANW3p5bP8I2UKp7scSEVmIWir3y4Ddzrm9zrk88A/AtfUd1uJN3xx7GcL9ii295ItlntivvruIrCy1hPsA8FLV10P+azP9spk9bWa3m9mmQEa3CJW7MNX7hCrApaevJmLw4J5jdT+WiMhC1BLuNstrM29D9B1g0Dl3IfAD4Kuz7sjsRjPbbmbbh4eHFzbSGo3X8UYdM61KxblgoIsH96rvLiIrSy3hPgRUV+IbgYPVGzjnjjvncv6XXwBeN9uOnHO3OOe2Oee29ff3L2a881rOtgzAFVv6ePKlUTL54rIcT0SkFrUk4KPAVjM7HTgAXA/cUL2Bma13zh3yv3wnsDPQUS5A5YRqPe7CVO3Wh/cDUCiVKZQc//O7O7lgoIsbXr+5rscVEanFvJW7c64IfBi4By+0/9E5t8PMPmVm7/Q3+4iZ7TCzp4CPAL9ZrwHPZyJXJBmLkIgtzxT+0/s66EjG+NmBsWU5nohILWoqb51zdwF3zXjtk1XPPwF8ItihLc54nddynylixvkbVvH4fi1FICIrR+iuUB3PFpdlpky1CzZ2USg5LQEsIitG6MJ9IltYtpOpFYO9HXSqNSMiK0j4wr3ON+qYTcSM8wZWsevwOJM5zZoRkcYLXbjX+0Ydc7lgoJti2fGDnUeW/dgiIjOFLtzrfaOOuZzW2053W3x6iqSISCOFLtwbVblHzLhiSy8Pv3CCZ9R7F5EGC1W4V+7CtJxTIatdOriajkSULz3wQkOOLyJSEapwzxbKlMqOdHJ5p0JWpOJRrrt0M9956iCHx7INGYOICIQs3Mdz3oqQjWjLVLzvykHKzvHVB/c1bAwiIqEK97GMF+6rGhjum1a3c8356/j7h15kNJNv2DhEpLWFKtwPjE4BsKG7raHj+OjPn8Vkrsjn793d0HGISOsKZbhv7GlsuJ+9rpNffu1GvvrTF3npRKahYxGR1tS4/kUdDI1MEYsYazpTDRtDZZ77Gf1pys7xO998gl/ZtklLAYvIsgpV5T40MsWG7jaikdluHrW8utriXHmmdyOPg/5vFCIiyyVU4X5gJNPwlky1q87qpy0e5fvPakkCEVleoQr3oZEpBhp8MrVaKh7l327tY9eRcR578USjhyMiLSQ04Z4rljg6nmNjT3ujh/IKV2zpoyMZ46/ueb7RQxGRFhKacD846l0ROrCC2jIAiViEq8/q58G9x/np7mONHo6ItIjQhPuBkZUxDXI2l52+mvVdKf7ye7twzjV6OCLSAkIT7kMj3nzyldRzr4hHI/zOm7byxP5R7t11tNHDEZEWEJpwPzA6RTRirO9q3Bz3U3n3to1sXt3OX93zPOWyqncRqa/QhPvQyBTrVqWIRVfmR4pHI3zszVt59tBJ7t5xuNHDEZGQW5lJuAhDI5kVdzJ1pmsvHmDrmjT/63u7KJTKjR6OiIRYaML9wMjUijyZWi0aMf7gmnPYMzzJX39fUyNFpH5CsbZMoVTm8MksG1fgydSK6nurXjrYw9/et4crtvTyxq39DRyViCzGXPdKXklrSIWicj88lqXsWHEXMM3l7RdsoL8zye/e9hRHx3XHJpFmkCuWeP7IOHc/c4gf7zrK3c8c5thErtHDmlMoKveXKtMgV3hbpiIRi3D9pZv5u/v38Euf/yk33XAJl2zuafSwRFasYqnMzkPjfPXBfQyP5yiVHYlYhFQswi+ev47ejiSFcpnJXJHJXJGJXIlCqcz6rhQbe9o5Z10nHcna4+7IySwP7T3ObY++xLGJHMPjOUYzBarnuUUMHtg9zCWbe3jLuWvpTDXm9p5zCUW4r+QLmOayrivFbe+/gg/f+jjvvvlBfu8tZ/EfrzydVDza6KGJNIRzjmLZkSuWyeSK7B6eYOehcR594QT/uucY49kiAFEzohEj709KuOuZ+WefRSPGBQNdXH5GL5efsZptg6tJV4V95YfH3TsOcc+OI+w+OgFAPGr0pZNs7Gnnks1J+tJJ+tIJ+tJJCqUyP35+mEdeOMGBkSnef9UZdfivsnihCPcXj2cwg/VdzRPuAM8ePMn73nA6dzwxxF/cvYub79vDVWev4a/efSHJmEJewiGTL3JgZIp8qYxhjGby7BmeYPfRCR7YfYzh8RxThRLFkmO2K0C62+KcvbaTLWvSrO9K0duRJBoxys6RK5QZzxWYzJWIRoxkLEIiFiEZixA1Y3SqwMhknv0jGV4YnuQL9+/l5h/vIWKwpjPF2lVJcsUye4cnyZfKRCPG5Wes5rptm7hiSy9PvjRKxGZfQjwVj/KOCzdw9tpOvvLTffzT9iF+44pBIitgyXEIQbiXy447nzrIttN6SMSa7xRCWyLKr77+NPYMT/DDnUf4zlMH2b7vBP/5587kV7ZtVMhL0/k/D7zAriPj7Dh4kr3HJpnMFWfdriMRpacjwZb+NOlkjFjUiEYixKNGLBqhtyPB+q7UnO2OiBltiShtiSh0zj6WtfEoa1elOGf9KgDyxTL7T2TYd3yS0UyBk9kCUfMCfe2qFGetfbl98/TQ2JzBXm3r2k7edsF6/vlnh/jrHzzPx99ydg3/leqv6cP9x88Ps/9Eht//xZXxH3SxtvSnOaOvgz3Dkzw1NMp/+/YzfOb7z/MfXjvAL79uI2ev7cRq+B9NZKGm8iV+9NxR9h2f5OjJLCXn2NDdxoauNjpTMdoSUcYyBV48kWH/iQz7j2cYGskQMaM9GaU9HqM9GSVqxp7hCV48nsEBHckYZ69N05dO0tOe4OfOWQM40sk4W9Z0sG5Vim8+8tKyftZELMKZa9KcuSYd6H7fsKWXwyezfO5HuzlrbSf/7qINge5/MZo+3L/64D7WdCb5xfPWNXooS2ZmnLkmzZb+Di7c2MUjL5zgSw+8wBd+8gIdyRhvPLOPrWvTbOhuY6C7jYEe70/16WWhnHM8c+Akt23fz/994iDjfnW9KhUjEjFGM4VZv689EWV1R4Ke9gRmXiU8nMuRHy1TKjv60gl+/jVrGOzrYLC34xWV7zXnN/+/0bmYGddetAEDfv/2pxjs7eCCjV0NHVNN4W5m1wD/G4gCX3TO/dmM95PA14DXAceB65xz+4Id6qvtOzbJfbuG+dibtzZlS2YuZsbWNZ1sXdPJRK7IzkMn2Xdskkf2neCeHYdf1Zfs7Ugw0NPGms4UfekEvekEvR3J6T9X+7/e9nQkGvJ55NXyxTIvHJtk15Fxnj88zg+fO8rweI5c0ZvlEY1EaItHOb2vnTP6vErzzLVpzuxPM9Ddtqi+biZf5GdDYzy+f5SvPbiPQ2NZYhHj/IEuXndaD5t62qf/HeWKJcamCuSLZfLFMsl4lN6OxJIKibnmhodFLBrh5ve+jnd+7gH+09e2808fuIJNqxs3PXvecDezKPB54BeAIeBRM7vTOfds1Wa/BYw45840s+uBPweuq8eAq339oReJRYwbLls5Fw4ELZ2Mcengai4dXA1Aqew4mS0wmikwmskzOuX/mSnwzIExbypYvshsa5O1J6Kcu34VW/rTbFnTQW9HkmQ8Qixi5EuOfLFMIuaFSnd7nI3+D4y57klbOVF2YHSK8WwRM28mQzoVY1UqTmcqxqo2789mP3eQLZTYfXSCo+NZjo3nKTtHd3ucrrYEPR1xutri/nZlxqYKHByd8h9ZDo5OMTqVJ1vwpuodn8xzYjI/ve9oxFjdkWDtqiRt8SjxaIRS2TFVKDE8nuPZgyeZzJdesf2qVIzOVHz676Z6KemZf/X5ojemTNU+NnSleOdFG7hoY7fXs54hGYuyprO5/84aoS+d5Au/sY3r/+4h3v7Zn/AX77qoYb+x1FK5Xwbsds7tBTCzfwCuBarD/Vrgv/vPbwduMjNzdVq8/In9I3zhJ3u5+5nDvOPCDaxZtTJXgqyHaMToafd+LYaOWbcpO0e2UGIiV2QyV2IyV2Q0k2d4Io8Z/PC5I9y2PT/r984Ui9h0GygZj5AtlDg5VeTg2NScv7rPtZ+2eJR0KkZnKsbFm7pZuyrFms4ka/w/ZwuZWhhGPGok/JkSiaj3Zzzq/eDKFcv+o0Su4P2ZLVR/PeO1YplsocSRkzleOpFh77FJ9g5PzPoDcz6JaITu9jjtCS+0z+jv4LLTV9OXTnJGfwdnre3kjP4OvvXYgVPuJ5MrcnTcm289MpVnKl8iWyi9IsirfwRXn5+JmrGlP01bIjo97zu9gDnfsjDnbejinz/yRj78zcf5wN8/xlvOXcubzlnDZaevpjedJJ2MzVkwBamWv+EBoPqsxxDw+rm2cc4VzWwM6AUCv/XQP21/id+//WlWpWJ84KotvP+qLUEfoulFzGhPxGhPxOacRTCVLzFV8FoAZeeIRoxYJEKxVKZQckzmi4z4vxGMZPIcGJ2iVHbEo0YyFuXstZ10t8Xpak/Q0x6f/nW9Mj0tWyiRLZaYqjwvlJjKez9wxrNF7ts1zLGJ3KICc7nEo94P0t6OBFedtYZ1XSm62+KkkzHMYKpQIpP3PteUXxXHokbK/82nuy1BKh6Z80T4ZK7EE/tHeWL/6LxjaU/GGEzGGOyb/Qe6rCybe9u5/QNv4DM/eJ7bHxvie88eecX7H7x6C394zTl1HUMt4T7b/5kz/0nWsg1mdiNwo//lhJntquH4c/pD/3EKfdThB8wKo88YDvqMIfCrNW73R38Of7T4w5xWy0a1hPsQsKnq643AwTm2GTKzGNAFnJi5I+fcLcAttQwsCGa23Tm3bbmO1wj6jOGgzyhBq2WKyaPAVjM73cwSwPXAnTO2uRP4Df/5u4Af1avfLiIi85u3cvd76B8G7sGbCvll59wOM/sUsN05dyfwJeDrZrYbr2K/vp6DFhGRU6vplLlz7i7grhmvfbLqeRZ4d7BDC8SytYAaSJ8xHPQZJVCm7omISPiE57JOERGZFtpwN7NrzGyXme02syXMOlqZzOzLZnbUzJ5p9Fjqxcw2mdm9ZrbTzHaY2UcbPaagmVnKzB4xs6f8z/g/Gj2mejCzqJk9YWbfbfRYWkUow71qyYS3AucC7zGzcxs7qsB9Bbim0YOosyLwcefca4DLgQ+F8O8xB7zJOXcRcDFwjZld3uAx1cNHgZ2NHkQrCWW4U7VkgnMuD1SWTAgN59xirRofAAAFNElEQVT9zHItQZg45w455x73n4/jhcNAY0cVLOeZ8L+M+49QnQgzs43A24EvNnosrSSs4T7bkgmhCoVWY2aDwCXAw40dSfD8lsWTwFHg+865sH3GzwB/AJQbPZBWEtZwr2k5BGkOZpYGvgV8zDl3stHjCZpzruScuxjv6u/LzOz8Ro8pKGb2DuCoc+6xRo+l1YQ13GtZMkGagJnF8YL9G865Oxo9nnpyzo0C9xGucylXAu80s3147dE3mdnfN3ZIrSGs4V7Lkgmywpm3nOKXgJ3OuU83ejz1YGb9ZtbtP28D3gw819hRBcc59wnn3Ebn3CDev8MfOed+rcHDagmhDHfnXBGoLJmwE/hH59yOxo4qWGb2TeBB4GwzGzKz32r0mOrgSuC9eNXek/7jbY0eVMDWA/ea2dN4Rcn3nXOaLihLpitURURCKJSVu4hIq1O4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncZdmYWcmfq77DX+L298ws4r+3zcw+e4rvHTSzG5ZvtK869pS//suKYGbX+ctZa068zErhLstpyjl3sXPuPOAXgLcBfwrgnNvunPvIKb53EGhIuPv2+Ou/1MxferounHO3Ab9dr/1L81O4S0M4544CNwIfNs/VlSrUzK6quiL1CTPrBP4MeKP/2u/61fRPzOxx//EG/3uvNrP7zOx2M3vOzL7hL2OAmV1qZj/1f2t4xMw6/RUZ/9LMHjWzp83s/bWM38y+bWaP+b+F3Fj1+oSZfcrMHgaumOOY5/nPn/SPudX/3l+rev3vKj8c/BvPPO7v44cB/jVImDnn9NBjWR7AxCyvjQBrgauB7/qvfQe40n+exruR+/T7/uvtQMp/vhXY7j+/GhjDWywugrdEw78BEsBe4FJ/u1X+fm8E/sR/LQlsB06fMcZB4JkZr632/2wDngF6/a8d8Cv+87mO+TngV6u2aQNe43/uuP/63wC/DvTjLV99evVxqz7rd2f7b62HHrEF/iwQCdpsyzP/K/BpM/sGcIdzbsgvvqvFgZvM7GKgBJxV9d4jzrkhAL9PPogX+Iecc48COH/pYDN7C3Chmb3L/94uvB8WL8wz7o+Y2S/5zzf533PcH8u3/NfPnuOYDwL/1b+JxR3Ouf9nZj8PvA541P+sbXjru18O3O+ce8HfR6hv0CLBUbhLw5jZGXhheBSvcgXAOfdnZvbPeD35h8zszbN8++8CR4CL8Cr0bNV7uarnJbz/z43Z1/Q34Hecc/csYNxX463eeIVzLmNm9wEp/+2sc65Ute9XHdM5d6vftnk7cI+Z/ba/7Vedc5+Ycax3zjFukVNSz10awsz6gZuBm5xzbsZ7W5xzP3PO/Tlem+QcYBzorNqsC68qLuOtHDnfycvngA1mdql/jE4zi+GtHPpBf914zOwsM+uYZ19dwIgf7OfgVdc1H9P/obbXOfdZvKWoLwR+CLzLzNb42642s9Pw2kpXmdnpldfnGZsIoMpdlleb3yaJ4938+uvAbOu0f8zMfg6v6n4W+Be8W7QVzewpvJuD/w3wLTN7N3AvMHmqAzvn8mZ2HfA5f930Kbzq+4t4bZvH/ROvw8C/n+dz3A18wF+mdxfw0AKPeR3wa2ZWAA4Dn3LOnTCzPwG+508PLQAfcs495J+wvcN//SjeTCORU9KSvyLzMO/+rd91zq2o29/57aH/4px7R6PHIiuP2jIi8ysBXSvtIia8315GGj0WWZlUuYuIhJAqdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCaH/D7nXmCCnEfNfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10db32ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nb_merge_dist_plot(\n",
    "    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n",
    "    SkyCoord(seip['seip_ra'], seip['seip_dec'])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given the graph above, we use 0.8 arc-second radius\n",
    "master_catalogue = merge_catalogues(master_catalogue, seip, \"seip_ra\", \"seip_dec\", radius=1.0*u.arcsec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning\n",
    "\n",
    "When we merge the catalogues, astropy masks the non-existent values (e.g. when a row comes only from a catalogue and has no counterparts in the other, the columns from the latest are masked for that row). We indicate to use NaN for masked values for floats columns, False for flag columns and -1 for ID columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in master_catalogue.colnames:\n",
    "    if \"m_\" in col or \"merr_\" in col or \"f_\" in col or \"ferr_\" in col or \"stellarity\" in col:\n",
    "        master_catalogue[col].fill_value = np.nan\n",
    "    elif \"flag\" in col:\n",
    "        master_catalogue[col].fill_value = 0\n",
    "    elif \"id\" in col:\n",
    "        master_catalogue[col].fill_value = -1\n",
    "        \n",
    "master_catalogue = master_catalogue.filled()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=10</i>\n",
       "<table id=\"table112194511592-832824\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>ps1_id</th><th>ra</th><th>dec</th><th>m_ap_gpc1_g</th><th>merr_ap_gpc1_g</th><th>m_gpc1_g</th><th>merr_gpc1_g</th><th>m_ap_gpc1_r</th><th>merr_ap_gpc1_r</th><th>m_gpc1_r</th><th>merr_gpc1_r</th><th>m_ap_gpc1_i</th><th>merr_ap_gpc1_i</th><th>m_gpc1_i</th><th>merr_gpc1_i</th><th>m_ap_gpc1_z</th><th>merr_ap_gpc1_z</th><th>m_gpc1_z</th><th>merr_gpc1_z</th><th>m_ap_gpc1_y</th><th>merr_ap_gpc1_y</th><th>m_gpc1_y</th><th>merr_gpc1_y</th><th>f_ap_gpc1_g</th><th>ferr_ap_gpc1_g</th><th>f_gpc1_g</th><th>ferr_gpc1_g</th><th>flag_gpc1_g</th><th>f_ap_gpc1_r</th><th>ferr_ap_gpc1_r</th><th>f_gpc1_r</th><th>ferr_gpc1_r</th><th>flag_gpc1_r</th><th>f_ap_gpc1_i</th><th>ferr_ap_gpc1_i</th><th>f_gpc1_i</th><th>ferr_gpc1_i</th><th>flag_gpc1_i</th><th>f_ap_gpc1_z</th><th>ferr_ap_gpc1_z</th><th>f_gpc1_z</th><th>ferr_gpc1_z</th><th>flag_gpc1_z</th><th>f_ap_gpc1_y</th><th>ferr_ap_gpc1_y</th><th>f_gpc1_y</th><th>ferr_gpc1_y</th><th>flag_gpc1_y</th><th>ps1_flag_cleaned</th><th>ps1_flag_gaia</th><th>flag_merged</th><th>ps1_flag_cleaned_1</th><th>seip_id</th><th>f_ap_irac_i1</th><th>ferr_ap_irac_i1</th><th>f_ap_irac_i2</th><th>ferr_ap_irac_i2</th><th>f_ap_irac_i3</th><th>ferr_ap_irac_i3</th><th>f_ap_irac_i4</th><th>ferr_ap_irac_i4</th><th>m_ap_irac_i1</th><th>merr_ap_irac_i1</th><th>m_ap_irac_i2</th><th>merr_ap_irac_i2</th><th>m_ap_irac_i3</th><th>merr_ap_irac_i3</th><th>m_ap_irac_i4</th><th>merr_ap_irac_i4</th><th>ps1_flag_cleaned_2</th><th>seip_flag_gaia</th></tr></thead>\n",
       "<thead><tr><th></th><th></th><th>deg</th><th>deg</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th>uJy</th><th>uJy</th><th>uJy</th><th>uJy</th><th>uJy</th><th>uJy</th><th>uJy</th><th>uJy</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>\n",
       "<tr><td>0</td><td>190582648149477540</td><td>264.814910506941</td><td>68.8223297362451</td><td>14.227999687194824</td><td>0.006335999816656113</td><td>14.305100440979004</td><td>0.0006150000263005495</td><td>13.748000144958496</td><td>3.7000001611886546e-05</td><td>13.824399948120117</td><td>0.003794000018388033</td><td>13.595100402832031</td><td>0.002245000097900629</td><td>13.663000106811523</td><td>0.0016659999964758754</td><td>13.543100357055664</td><td>0.0016680000117048621</td><td>13.602700233459473</td><td>0.001323999953456223</td><td>13.516799926757812</td><td>0.0031240000389516354</td><td>13.579899787902832</td><td>0.003249000059440732</td><td>7392.649656415478</td><td>43.14107484155039</td><td>6885.885923697371</td><td>3.900413383672567</td><td>False</td><td>11502.703942155389</td><td>0.39199214554663814</td><td>10721.116635628732</td><td>37.463903786626574</td><td>False</td><td>13242.190731341167</td><td>27.381162531005785</td><td>12439.415011830086</td><td>19.087569591075386</td><td>False</td><td>13891.84270354754</td><td>21.341826578659003</td><td>13149.822983847242</td><td>16.03553878213893</td><td>False</td><td>14232.46188062854</td><td>40.951210132452125</td><td>13428.889023883086</td><td>40.185139857741724</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>190982650740682167</td><td>265.074014206941</td><td>69.1511883262451</td><td>11.652000427246094</td><td>0.033597998321056366</td><td>11.974200248718262</td><td>0.0</td><td>13.68850040435791</td><td>0.0010930000571534038</td><td>13.253399848937988</td><td>0.0010860000038519502</td><td>10.981499671936035</td><td>0.0010860000038519502</td><td>10.861900329589844</td><td>0.0010860000038519502</td><td>11.203300476074219</td><td>0.20606200397014618</td><td>11.191200256347656</td><td>0.23958300054073334</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>79286.60628791935</td><td>2453.5161057258883</td><td>58927.75562895019</td><td>0.0</td><td>False</td><td>12150.659151094938</td><td>12.23195016284535</td><td>18140.116021857175</td><td>18.144523488558015</td><td>False</td><td>147028.02762398298</td><td>147.06375072162263</td><td>164149.6148869982</td><td>164.18949798149984</td><td>False</td><td>119861.52793638725</td><td>22748.533702067914</td><td>121204.82182154978</td><td>26745.552708616557</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>2</td><td>190572651688019436</td><td>265.16915364694097</td><td>68.8153567862451</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>13.697999954223633</td><td>0.001551999943330884</td><td>14.61620044708252</td><td>0.0026050000451505184</td><td>3.8425300121307373</td><td>0.0</td><td>3.915260076522827</td><td>0.0</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>13.068499565124512</td><td>0.08652400225400925</td><td>13.917799949645996</td><td>0.1703220009803772</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>12044.811793560197</td><td>17.217393266535648</td><td>5170.348548614671</td><td>12.405184743365526</td><td>False</td><td>105435775.12270036</td><td>0.0</td><td>98604326.11978783</td><td>0.0</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>21508.007237691858</td><td>1714.0064580676528</td><td>9837.39296628186</td><td>1543.2150527030392</td><td>False</td><td>False</td><td>0</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>3</td><td>190932646959945827</td><td>264.69593654694097</td><td>69.1125553162451</td><td>16.74720001220703</td><td>0.003360999980941415</td><td>16.809499740600586</td><td>0.003768000053241849</td><td>16.32469940185547</td><td>0.0019379999721422791</td><td>16.374000549316406</td><td>0.00508299982175231</td><td>16.164899826049805</td><td>0.0030759999062865973</td><td>16.225000381469727</td><td>0.0024649999104440212</td><td>16.118499755859375</td><td>0.005948000121861696</td><td>16.169200897216797</td><td>0.0066109998151659966</td><td>16.07990074157715</td><td>0.0039090001955628395</td><td>16.16670036315918</td><td>0.008887000381946564</td><td>726.3066074803263</td><td>2.2483513796454915</td><td>685.8041419952765</td><td>2.380053305975319</td><td>False</td><td>1071.8160083462594</td><td>1.9131529235312135</td><td>1024.2353959206953</td><td>4.795076660501607</td><td>False</td><td>1241.7668722113146</td><td>3.5180504857556243</td><td>1174.8971421433384</td><td>2.667426339356011</td><td>False</td><td>1295.9853661727427</td><td>7.099810324224828</td><td>1236.8574266584085</td><td>7.531170263265741</td><td>False</td><td>1342.887722840194</td><td>4.834828522984234</td><td>1239.7092872898284</td><td>10.147305451697394</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>4</td><td>190662646668524269</td><td>264.666786496941</td><td>68.8862480162451</td><td>17.17460060119629</td><td>0.0032889998983591795</td><td>17.252500534057617</td><td>0.0062339999713003635</td><td>16.690399169921875</td><td>0.0023690001107752323</td><td>16.75040054321289</td><td>0.0058559998869895935</td><td>16.50029945373535</td><td>0.001753999968059361</td><td>16.57080078125</td><td>0.0022430000826716423</td><td>16.42530059814453</td><td>0.004945000167936087</td><td>16.508899688720703</td><td>0.011253999546170235</td><td>16.374500274658203</td><td>0.00597799988463521</td><td>16.457000732421875</td><td>0.013656999915838242</td><td>489.95902249507265</td><td>1.4842234864457216</td><td>456.0366916335223</td><td>2.6184378029363167</td><td>False</td><td>765.3151877544151</td><td>1.66986396568775</td><td>724.1687548470541</td><td>3.905858649606748</td><td>False</td><td>911.7593349917399</td><td>1.4729414359076265</td><td>854.4362936878163</td><td>1.7651623561787408</td><td>False</td><td>976.9666994119167</td><td>4.449607990861944</td><td>904.5657181065108</td><td>9.376110086813977</td><td>False</td><td>1023.7640856620986</td><td>5.6367850304347815</td><td>948.8546866439949</td><td>11.935227285274639</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>5</td><td>190582649166892627</td><td>264.917040886941</td><td>68.8181146062451</td><td>11.59630012512207</td><td>0.0010860000038519502</td><td>11.654800415039062</td><td>0.0010860000038519502</td><td>12.5693998336792</td><td>0.0026700000744313</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>12.995599746704102</td><td>0.013261999934911728</td><td>nan</td><td>nan</td><td>12.63700008392334</td><td>0.013841000385582447</td><td>9.832220077514648</td><td>0.0010860000038519502</td><td>9.879549980163574</td><td>0.0010860000038519502</td><td>83460.30163175428</td><td>83.48057980967997</td><td>79082.39874966646</td><td>79.10161323752816</td><td>False</td><td>34059.64103816238</td><td>83.7581391395671</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>23001.708816289294</td><td>280.96019962090435</td><td>False</td><td>nan</td><td>nan</td><td>32003.68347210029</td><td>407.98399586949205</td><td>False</td><td>423752.20297073526</td><td>423.855161172422</td><td>405676.646917935</td><td>405.775213338924</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>6</td><td>190672646811690901</td><td>264.68102043694097</td><td>68.8918520762451</td><td>11.593500137329102</td><td>0.0010860000038519502</td><td>11.662099838256836</td><td>0.0010860000038519502</td><td>12.912699699401855</td><td>0.0030789999291300774</td><td>13.17770004272461</td><td>0.10782100260257721</td><td>nan</td><td>nan</td><td>13.359700202941895</td><td>0.0383090004324913</td><td>11.106200218200684</td><td>0.0010860000038519502</td><td>nan</td><td>nan</td><td>10.528200149536133</td><td>0.0010860000038519502</td><td>10.558899879455566</td><td>0.0010860000038519502</td><td>83675.81384442773</td><td>83.69614438491273</td><td>78552.50964251773</td><td>78.5715953843123</td><td>False</td><td>24826.76424756718</td><td>70.40532039347205</td><td>19450.01689518392</td><td>1931.5191969148466</td><td>False</td><td>nan</td><td>nan</td><td>16448.258352455374</td><td>580.3585931615066</td><td>False</td><td>131075.0139841171</td><td>131.1068610108258</td><td>nan</td><td>nan</td><td>False</td><td>223213.2342287855</td><td>223.26746788946687</td><td>216990.1640816585</td><td>217.0428857357784</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>7</td><td>190772651826417814</td><td>265.182559426941</td><td>68.9808885162451</td><td>18.16469955444336</td><td>0.011083000339567661</td><td>18.221799850463867</td><td>0.010095999576151371</td><td>17.324100494384766</td><td>0.003381999908015132</td><td>17.362199783325195</td><td>0.006215000059455633</td><td>16.966100692749023</td><td>0.002125999890267849</td><td>17.032100677490234</td><td>0.003083999967202544</td><td>16.787599563598633</td><td>0.0030410001054406166</td><td>16.833599090576172</td><td>0.006424999795854092</td><td>16.648799896240234</td><td>0.020871000364422798</td><td>16.741500854492188</td><td>0.031651001423597336</td><td>196.84309197018513</td><td>2.009338958751136</td><td>186.7583635489371</td><td>1.7366210604101853</td><td>False</td><td>426.9330758278138</td><td>1.3298696468355682</td><td>412.2114838454956</td><td>2.3595919389818683</td><td>False</td><td>593.689985079398</td><td>1.1625152017594358</td><td>558.6756320559264</td><td>1.5869007806745856</td><td>False</td><td>699.7778224096776</td><td>1.9599829336140604</td><td>670.7495095194851</td><td>3.9692564757863718</td><td>False</td><td>795.206720065814</td><td>15.28618063167778</td><td>730.1291003221717</td><td>21.2844677118002</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>8</td><td>190892648501256672</td><td>264.850093486941</td><td>69.0799257362451</td><td>17.720699310302734</td><td>0.0064849997870624065</td><td>17.772899627685547</td><td>0.009982000105082989</td><td>16.878700256347656</td><td>0.0033859999384731054</td><td>16.939699172973633</td><td>0.004738999996334314</td><td>16.54159927368164</td><td>0.0013849999522790313</td><td>16.61079978942871</td><td>0.002630000002682209</td><td>16.387100219726562</td><td>0.005698999855667353</td><td>16.465099334716797</td><td>0.007902000099420547</td><td>16.287599563598633</td><td>0.005286000203341246</td><td>16.37980079650879</td><td>0.007759000174701214</td><td>296.29223902992874</td><td>1.7697255545101547</td><td>282.3840410692546</td><td>2.596171625551527</td><td>False</td><td>643.457544708494</td><td>2.006700335905606</td><td>608.3035217626536</td><td>2.6551112275431206</td><td>False</td><td>877.728682546801</td><td>1.1196588804110745</td><td>823.5312509904268</td><td>1.9948558248513062</td><td>False</td><td>1011.9519977674815</td><td>5.311708556779602</td><td>941.8034263684294</td><td>6.854455747338584</td><td>False</td><td>1109.0731069723606</td><td>5.399617921263125</td><td>1018.7782894164013</td><td>7.28049860631165</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>9</td><td>190682646772120568</td><td>264.677143936941</td><td>68.8998494362451</td><td>20.582599639892578</td><td>0.02723100036382675</td><td>20.665300369262695</td><td>0.040807001292705536</td><td>19.50830078125</td><td>0.0036669999826699495</td><td>19.584800720214844</td><td>0.018195999786257744</td><td>18.668500900268555</td><td>0.010762999765574932</td><td>18.748600006103516</td><td>0.011827000416815281</td><td>18.255199432373047</td><td>0.026830000802874565</td><td>18.34980010986328</td><td>0.021203000098466873</td><td>18.232799530029297</td><td>0.05189900100231171</td><td>18.348800659179688</td><td>0.03826000168919563</td><td>21.230496171181283</td><td>0.5324752425387252</td><td>19.67341948645098</td><td>0.7394183328301067</td><td>False</td><td>57.105729983612264</td><td>0.19287070831990052</td><td>53.22059332239179</td><td>0.8919311159329748</td><td>False</td><td>123.76551117488025</td><td>1.2268985430565504</td><td>114.96350539393163</td><td>1.252305505014886</td><td>False</td><td>181.10074089746035</td><td>4.475242699272858</td><td>165.98924748409397</td><td>3.2415516912174094</td><td>False</td><td>184.87585180139567</td><td>8.837203711081298</td><td>166.142115599287</td><td>5.854642771990201</td><td>False</td><td>False</td><td>3</td><td>False</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>0</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table112194511592-832824').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table112194511592-832824').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46, 47, 50, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 71], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_catalogue[:10].show_in_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## III - Merging flags and stellarity\n",
    "\n",
    "Each pristine catalogue contains a flag indicating if the source was associated to a another nearby source that was removed during the cleaning process.  We merge these flags in a single one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag_cleaned_columns = [column for column in master_catalogue.colnames\n",
    "                        if 'flag_cleaned' in column]\n",
    "\n",
    "flag_column = np.zeros(len(master_catalogue), dtype=bool)\n",
    "for column in flag_cleaned_columns:\n",
    "    flag_column |= master_catalogue[column]\n",
    "    \n",
    "master_catalogue.add_column(Column(data=flag_column, name=\"flag_cleaned\"))\n",
    "master_catalogue.remove_columns(flag_cleaned_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each pristine catalogue contains a flag indicating the probability of a source being a Gaia object (0: not a Gaia object, 1: possibly, 2: probably, 3: definitely).  We merge these flags taking the highest value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag_gaia_columns = [column for column in master_catalogue.colnames\n",
    "                     if 'flag_gaia' in column]\n",
    "\n",
    "master_catalogue.add_column(Column(\n",
    "    data=np.max([master_catalogue[column] for column in flag_gaia_columns], axis=0),\n",
    "    name=\"flag_gaia\"\n",
    "))\n",
    "master_catalogue.remove_columns(flag_gaia_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each prisitine catalogue may contain one or several stellarity columns indicating the probability (0 to 1) of each source being a star.  We merge these columns taking the highest value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No ps1 stellarity\n",
    "#stellarity_columns = [column for column in master_catalogue.colnames\n",
    "#                      if 'stellarity' in column]\n",
    "#\n",
    "#master_catalogue.add_column(Column(\n",
    "#    data=np.nanmax([master_catalogue[column] for column in stellarity_columns], axis=0),\n",
    "#    name=\"stellarity\"\n",
    "#))\n",
    "#master_catalogue.remove_columns(stellarity_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IV - Adding E(B-V) column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue.add_column(\n",
    "    ebv(master_catalogue['ra'], master_catalogue['dec'])\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## V - Adding HELP unique identifiers and field columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue.add_column(Column(gen_help_id(master_catalogue['ra'], master_catalogue['dec']),\n",
    "                                   name=\"help_id\"))\n",
    "master_catalogue.add_column(Column(np.full(len(master_catalogue), \"SPIRE-NEP\", dtype='<U18'),\n",
    "                                   name=\"field\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK!\n"
     ]
    }
   ],
   "source": [
    "# Check that the HELP Ids are unique\n",
    "if len(master_catalogue) != len(np.unique(master_catalogue['help_id'])):\n",
    "    print(\"The HELP IDs are not unique!!!\")\n",
    "else:\n",
    "    print(\"OK!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VI - Cross-matching with spec-z catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "specz =  Table.read(\"../../dmu23/dmu23_SPIRE-NEP/data/SPIRE-NEP-specz-v2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "specz.rename_column('RA', 'ra')\n",
    "specz.rename_column('DEC', 'dec')\n",
    "specz.rename_column('OBJID', 'specz_id')\n",
    "specz.rename_column('Z_SPEC', 'z_spec')\n",
    "specz.rename_column('Z_SOURCE', 'z_source')\n",
    "specz.rename_column('Z_QUAL', 'z_qual')\n",
    "specz.rename_column('REL', 'rel')\n",
    "specz.rename_column('AGN', 'agn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Only one object so plot breaks\n",
    "#nb_merge_dist_plot(\n",
    "#    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n",
    "#    SkyCoord(specz['ra'] * u.deg, specz['dec'] * u.deg)\n",
    "#)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue = specz_merge(master_catalogue, specz, radius=1. * u.arcsec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VII - Choosing between multiple values for the same filter\n",
    "\n",
    "We only have PanSTARRS on SPIRE-NEP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VIII.a Wavelength domain coverage\n",
    "\n",
    "We add a binary `flag_optnir_obs` indicating that a source was observed in a given wavelength domain:\n",
    "\n",
    "- 1 for observation in optical;\n",
    "- 2 for observation in near-infrared;\n",
    "- 4 for observation in mid-infrared (IRAC).\n",
    "\n",
    "It's an integer binary flag, so a source observed both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.\n",
    "\n",
    "*Note 1: The observation flag is based on the creation of multi-order coverage maps from the catalogues, this may not be accurate, especially on the edges of the coverage.*\n",
    "\n",
    "*Note 2: Being on the observation coverage does not mean having fluxes in that wavelength domain. For sources observed in one domain but having no flux in it, one must take into consideration de different depths in the catalogue we are using.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps1_moc = MOC(filename=\"../../dmu0/dmu0_PanSTARRS1-3SS/data/PanSTARRS1-3SS_SPIRE-NEP_v2_MOC.fits\")\n",
    "seip_moc = MOC(filename=\"../../dmu0/dmu0_SEIP/data/SEIP_SPIRE-NEP_MOC.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "was_observed_optical = inMoc(\n",
    "    master_catalogue['ra'], master_catalogue['dec'],\n",
    "    ps1_moc) \n",
    "\n",
    "was_observed_nir = np.zeros(len(master_catalogue), dtype=bool)\n",
    "\n",
    "was_observed_mir = inMoc(\n",
    "    master_catalogue['ra'], master_catalogue['dec'],\n",
    "    seip_moc) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue.add_column(\n",
    "    Column(\n",
    "        1 * was_observed_optical + 2 * was_observed_nir + 4 * was_observed_mir,\n",
    "        name=\"flag_optnir_obs\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VIII.b Wavelength domain detection\n",
    "\n",
    "We add a binary `flag_optnir_det` indicating that a source was detected in a given wavelength domain:\n",
    "\n",
    "- 1 for detection in optical;\n",
    "- 2 for detection in near-infrared;\n",
    "- 4 for detection in mid-infrared (IRAC).\n",
    "\n",
    "It's an integer binary flag, so a source detected both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.\n",
    "\n",
    "*Note 1: We use the total flux columns to know if the source has flux, in some catalogues, we may have aperture flux and no total flux.*\n",
    "\n",
    "To get rid of artefacts (chip edges, star flares, etc.) we consider that a source is detected in one wavelength domain when it has a flux value in **at least two bands**. That means that good sources will be excluded from this flag when they are on the coverage of only one band."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SpARCS is a catalogue of sources detected in r (with fluxes measured at \n",
    "# this prior position in the other bands).  Thus, we are only using the r\n",
    "# CFHT band.\n",
    "# Check to use catalogue flags from HSC and PanSTARRS.\n",
    "nb_optical_flux = (\n",
    "    1 * ~np.isnan(master_catalogue['f_gpc1_g']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_gpc1_r']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_gpc1_i']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_gpc1_z']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_gpc1_y']) \n",
    ")\n",
    "\n",
    "nb_nir_flux = np.zeros(len(master_catalogue), dtype=float)\n",
    "\n",
    "nb_mir_flux = (\n",
    "    1 * ~np.isnan(master_catalogue['f_ap_irac_i1']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_ap_irac_i2']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_ap_irac_i3']) +\n",
    "    1 * ~np.isnan(master_catalogue['f_ap_irac_i4']) \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "has_optical_flux = nb_optical_flux >= 2\n",
    "has_nir_flux = nb_nir_flux >= 2\n",
    "has_mir_flux = nb_mir_flux >= 2\n",
    "\n",
    "master_catalogue.add_column(\n",
    "    Column(\n",
    "        1 * has_optical_flux + 2 * has_nir_flux + 4 * has_mir_flux,\n",
    "        name=\"flag_optnir_det\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IX - Cross-identification table\n",
    "\n",
    "We are producing a table associating to each HELP identifier, the identifiers of the sources in the pristine catalogue. This can be used to easily get additional information from them.\n",
    "\n",
    "For convenience, we also cross-match the master list with the SDSS catalogue and add the objID associated with each source, if any. **TODO: should we correct the astrometry with respect to Gaia positions?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 master list rows had multiple associations.\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# Addind SDSS ids\n",
    "#\n",
    "sdss = Table.read(\"../../dmu0/dmu0_SDSS/data/SDSS-DR13_SPIRE-NEP.fits\")['objID', 'ra', 'dec']\n",
    "sdss_coords = SkyCoord(sdss['ra'] * u.deg, sdss['dec'] * u.deg)\n",
    "idx_ml, d2d, _ = sdss_coords.match_to_catalog_sky(SkyCoord(master_catalogue['ra'], master_catalogue['dec']))\n",
    "idx_sdss = np.arange(len(sdss))\n",
    "\n",
    "# Limit the cross-match to 1 arcsec\n",
    "mask = d2d <= 1. * u.arcsec\n",
    "idx_ml = idx_ml[mask]\n",
    "idx_sdss = idx_sdss[mask]\n",
    "d2d = d2d[mask]\n",
    "nb_orig_matches = len(idx_ml)\n",
    "\n",
    "# In case of multiple associations of one master list object to an SDSS object, we keep only the\n",
    "# association to the nearest one.\n",
    "sort_idx = np.argsort(d2d)\n",
    "idx_ml = idx_ml[sort_idx]\n",
    "idx_sdss = idx_sdss[sort_idx]\n",
    "_, unique_idx = np.unique(idx_ml, return_index=True)\n",
    "idx_ml = idx_ml[unique_idx]\n",
    "idx_sdss = idx_sdss[unique_idx]\n",
    "print(\"{} master list rows had multiple associations.\".format(nb_orig_matches - len(idx_ml)))\n",
    "\n",
    "# Adding the ObjID to the master list\n",
    "master_catalogue.add_column(Column(data=np.full(len(master_catalogue), -1, dtype='>i8'), name=\"sdss_id\"))\n",
    "master_catalogue['sdss_id'][idx_ml] = sdss['objID'][idx_sdss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ps1_id', 'seip_id', 'help_id', 'specz_id', 'sdss_id']\n"
     ]
    }
   ],
   "source": [
    "id_names = []\n",
    "for col in master_catalogue.colnames:\n",
    "    if '_id' in col:\n",
    "        id_names += [col]\n",
    "    if '_intid' in col:\n",
    "        id_names += [col]\n",
    "        \n",
    "print(id_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue[id_names].write(\n",
    "    \"{}/master_list_cross_ident_spire-nep{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)\n",
    "id_names.remove('help_id')\n",
    "master_catalogue.remove_columns(id_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## X - Adding HEALPix index\n",
    "\n",
    "We are adding a column with a HEALPix index at order 13 associated with each source."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue.add_column(Column(\n",
    "    data=coords_to_hpidx(master_catalogue['ra'], master_catalogue['dec'], order=13),\n",
    "    name=\"hp_idx\"\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XI - Saving the catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"help_id\", \"field\", \"ra\", \"dec\", \"hp_idx\"]\n",
    "\n",
    "bands = [column[5:] for column in master_catalogue.colnames if 'f_ap' in column]\n",
    "for band in bands:\n",
    "    if band.startswith('irac'):\n",
    "        columns += [\"f_ap_{}\".format(band), \"ferr_ap_{}\".format(band),\n",
    "                \"m_ap_{}\".format(band), \"merr_ap_{}\".format(band),\n",
    "                #\"f_{}\".format(band), \"ferr_{}\".format(band),#No total fluxes from irac\n",
    "                #\"m_{}\".format(band), \"merr_{}\".format(band),\n",
    "                #\"flag_{}\".format(band)\n",
    "                   ]    \n",
    "    else:\n",
    "        columns += [\"f_ap_{}\".format(band), \"ferr_ap_{}\".format(band),\n",
    "                \"m_ap_{}\".format(band), \"merr_ap_{}\".format(band),\n",
    "                \"f_{}\".format(band), \"ferr_{}\".format(band),\n",
    "                \"m_{}\".format(band), \"merr_{}\".format(band),\n",
    "                \"flag_{}\".format(band)]  \n",
    "columns += [\"flag_cleaned\", \"flag_gaia\",  \"flag_merged\",\n",
    "            \"flag_optnir_obs\", \"flag_optnir_det\", \"ebv\", 'zspec_association_flag', 'zspec_qual', 'zspec'] #\"stellarity\", \"flag_merged\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing columns: set()\n"
     ]
    }
   ],
   "source": [
    "# We check for columns in the master catalogue that we will not save to disk.\n",
    "print(\"Missing columns: {}\".format(set(master_catalogue.colnames) - set(columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_catalogue[columns].write(\"{}/master_catalogue_spire-nep{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}