{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XMM-LSS master catalogue\n",
    "\n",
    "This notebook presents the merge of the various pristine catalogues to produce the HELP master catalogue on XMM-LSS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook was run with herschelhelp_internal version: \n",
      "0246c5d (Thu Jan 25 17:01:47 2018 +0000)\n"
     ]
    }
   ],
   "source": [
    "from herschelhelp_internal import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "#%config InlineBackend.figure_format = 'svg'\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord\n",
    "from astropy.table import Column, Table\n",
    "import numpy as np\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n",
    "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n",
    "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
    "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
    "\n",
    "try:\n",
    "    os.makedirs(OUT_DIR)\n",
    "except FileExistsError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I - Reading the prepared pristine catalogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "candels = Table.read(\"{}/CANDELS-UDS.fits\".format(TMP_DIR))           # 1.2\n",
    "#cfht_wirds = Table.read(\"{}/CFHT-WIRDS.fits\".format(TMP_DIR))     # 1.3\n",
    "#cfhtls_wide = Table.read(\"{}/CFHTLS-WIDE.fits\".format(TMP_DIR))   # 1.4a\n",
    "#cfhtls_deep = Table.read(\"{}/CFHTLS-DEEP.fits\".format(TMP_DIR))   # 1.4b\n",
    "#We no longer use CFHTLenS as it is the same raw data set as CFHTLS-WIDE\n",
    "# cfhtlens = Table.read(\"{}/CFHTLENS.fits\".format(TMP_DIR))         # 1.5\n",
    "#decals = Table.read(\"{}/DECaLS.fits\".format(TMP_DIR))             # 1.6\n",
    "#servs = Table.read(\"{}/SERVS.fits\".format(TMP_DIR))               # 1.8\n",
    "#swire = Table.read(\"{}/SWIRE.fits\".format(TMP_DIR))               # 1.7\n",
    "#hsc_wide = Table.read(\"{}/HSC-WIDE.fits\".format(TMP_DIR))         # 1.9a\n",
    "#hsc_deep = Table.read(\"{}/HSC-DEEP.fits\".format(TMP_DIR))         # 1.9b\n",
    "#hsc_udeep = Table.read(\"{}/HSC-UDEEP.fits\".format(TMP_DIR))       # 1.9c\n",
    "#ps1 = Table.read(\"{}/PS1.fits\".format(TMP_DIR))                   # 1.10\n",
    "#sxds = Table.read(\"{}/SXDS.fits\".format(TMP_DIR))                 # 1.11\n",
    "#sparcs = Table.read(\"{}/SpARCS.fits\".format(TMP_DIR))             # 1.12\n",
    "dxs = Table.read(\"{}/UKIDSS-DXS.fits\".format(TMP_DIR))            # 1.13\n",
    "uds = Table.read(\"{}/UKIDSS-UDS.fits\".format(TMP_DIR))            # 1.14\n",
    "#vipers = Table.read(\"{}/VIPERS.fits\".format(TMP_DIR))             # 1.15\n",
    "#vhs = Table.read(\"{}/VISTA-VHS.fits\".format(TMP_DIR))             # 1.16\n",
    "#video = Table.read(\"{}/VISTA-VIDEO.fits\".format(TMP_DIR))         # 1.17\n",
    "#viking = Table.read(\"{}/VISTA-VIKING.fits\".format(TMP_DIR))       # 1.18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II - Merging tables\n",
    "\n",
    "We first merge the optical catalogues and then add the infrared ones. We start with PanSTARRS because it coevrs the whole field.\n",
    "\n",
    "At every step, we look at the distribution of the distances separating the sources from one catalogue to the other (within a maximum radius) to determine the best cross-matching radius."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start with CANDELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f_acs_f606w  removed.\n",
      "ferr_acs_f606w  removed.\n",
      "f_ap_acs_f606w  removed.\n",
      "ferr_ap_acs_f606w  removed.\n",
      "f_acs_f814w  removed.\n",
      "ferr_acs_f814w  removed.\n",
      "f_ap_acs_f814w  removed.\n",
      "ferr_ap_acs_f814w  removed.\n",
      "f_wfc3_f125w  removed.\n",
      "ferr_wfc3_f125w  removed.\n",
      "f_ap_wfc3_f125w  removed.\n",
      "ferr_ap_wfc3_f125w  removed.\n",
      "f_wfc3_f160w  removed.\n",
      "ferr_wfc3_f160w  removed.\n",
      "f_ap_wfc3_f160w  removed.\n",
      "ferr_ap_wfc3_f160w  removed.\n",
      "f_candels-megacam_u  removed.\n",
      "ferr_candels-megacam_u  removed.\n",
      "f_suprime_b  removed.\n",
      "ferr_suprime_b  removed.\n",
      "f_suprime_v  removed.\n",
      "ferr_suprime_v  removed.\n",
      "f_suprime_rc  removed.\n",
      "ferr_suprime_rc  removed.\n",
      "f_suprime_ip  removed.\n",
      "ferr_suprime_ip  removed.\n",
      "f_suprime_zp  removed.\n",
      "ferr_suprime_zp  removed.\n",
      "f_hawki_k  removed.\n",
      "ferr_hawki_y  removed.\n",
      "ferr_hawki_k  removed.\n",
      "f_candels-irac_i1  removed.\n",
      "ferr_candels-irac_i1  removed.\n",
      "f_candels-irac_i2  removed.\n",
      "ferr_candels-irac_i2  removed.\n",
      "f_candels-irac_i3  removed.\n",
      "ferr_candels-irac_i3  removed.\n",
      "f_candels-irac_i4  removed.\n",
      "ferr_candels-irac_i4  removed.\n",
      "m_acs_f606w  removed.\n",
      "merr_acs_f606w  removed.\n",
      "flag_acs_f606w  removed.\n",
      "m_ap_acs_f606w  removed.\n",
      "merr_ap_acs_f606w  removed.\n",
      "m_acs_f814w  removed.\n",
      "merr_acs_f814w  removed.\n",
      "flag_acs_f814w  removed.\n",
      "m_ap_acs_f814w  removed.\n",
      "merr_ap_acs_f814w  removed.\n",
      "m_wfc3_f125w  removed.\n",
      "merr_wfc3_f125w  removed.\n",
      "flag_wfc3_f125w  removed.\n",
      "m_ap_wfc3_f125w  removed.\n",
      "merr_ap_wfc3_f125w  removed.\n",
      "m_wfc3_f160w  removed.\n",
      "merr_wfc3_f160w  removed.\n",
      "flag_wfc3_f160w  removed.\n",
      "m_ap_wfc3_f160w  removed.\n",
      "merr_ap_wfc3_f160w  removed.\n",
      "m_candels-megacam_u  removed.\n",
      "merr_candels-megacam_u  removed.\n",
      "flag_candels-megacam_u  removed.\n",
      "m_suprime_b  removed.\n",
      "merr_suprime_b  removed.\n",
      "flag_suprime_b  removed.\n",
      "m_suprime_v  removed.\n",
      "merr_suprime_v  removed.\n",
      "flag_suprime_v  removed.\n",
      "m_suprime_rc  removed.\n",
      "merr_suprime_rc  removed.\n",
      "flag_suprime_rc  removed.\n",
      "m_suprime_ip  removed.\n",
      "merr_suprime_ip  removed.\n",
      "flag_suprime_ip  removed.\n",
      "m_suprime_zp  removed.\n",
      "merr_suprime_zp  removed.\n",
      "flag_suprime_zp  removed.\n",
      "m_hawki_k  removed.\n",
      "merr_hawki_k  removed.\n",
      "flag_hawki_k  removed.\n",
      "m_candels-irac_i1  removed.\n",
      "merr_candels-irac_i1  removed.\n",
      "flag_candels-irac_i1  removed.\n",
      "m_candels-irac_i2  removed.\n",
      "merr_candels-irac_i2  removed.\n",
      "flag_candels-irac_i2  removed.\n",
      "m_candels-irac_i3  removed.\n",
      "merr_candels-irac_i3  removed.\n",
      "flag_candels-irac_i3  removed.\n",
      "m_candels-irac_i4  removed.\n",
      "merr_candels-irac_i4  removed.\n",
      "flag_candels-irac_i4  removed.\n"
     ]
    }
   ],
   "source": [
    "master_catalogue = candels\n",
    "master_catalogue['candels_ra'].name = 'ra'\n",
    "master_catalogue['candels_dec'].name = 'dec'\n",
    "del candels\n",
    "unused_bands = [ 'candels-megacam', 'candels-irac', 'hawki', 'suprime', 'wfc3', 'acs']\n",
    "for col in master_catalogue.colnames:\n",
    "    \n",
    "    for band in unused_bands:\n",
    "        if band in col:\n",
    "            master_catalogue.remove_column(col)\n",
    "            print(col, ' removed.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add DXS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HELP Warning: There weren't any cross matches. The two surveys probably don't overlap.\n"
     ]
    }
   ],
   "source": [
    "nb_merge_dist_plot(\n",
    "    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n",
    "    SkyCoord(dxs['dxs_ra'], dxs['dxs_dec'])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Given the graph above, we use 0.8 arc-second radius\n",
    "master_catalogue = merge_catalogues(master_catalogue, dxs, \"dxs_ra\", \"dxs_dec\", radius=0.8*u.arcsec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add UDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAF3CAYAAABnvQURAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8nXd99//35xztvSVrWZIt73jEK4MskoCTQAItK2F0\nAGloaAOP9tfSu6W9e9/tXXi0tDRlhEAppay2kGaAswhJbCeeibc8JEuyJWvvvb+/P3RkhHFs2b6k\n60h6PR8PPaxzzqVzfXw89Nb3+pzP15xzAgAAwNUL+F0AAADAXEGwAgAA8AjBCgAAwCMEKwAAAI8Q\nrAAAADxCsAIAAPAIwQoAAMAjBCsAAACPEKwAAAA8QrACAADwSIRfJ87IyHBFRUV+nR4AAGDK3njj\njRbnXOaljvMtWBUVFWnfvn1+nR4AAGDKzOz0VI7jUiAAAIBHCFYAAAAeIVgBAAB4hGAFAADgEYIV\nAACARwhWAAAAHiFYAQAAeIRgBQAA4JFLBisz+7aZNZnZkbd43MzsUTOrMLNDZnat92UCAACEv6ms\nWH1H0paLPH6XpNLQx4OSvn71ZQEAAMw+lwxWzrltktoucsh9kr7rxu2SlGJmC7wqEAAAYLbwoscq\nT1LNpNu1ofsAAADmlRltXjezB81sn5nta25unslTAwAATLsID57jrKSCSbfzQ/f9Gufc45Iel6QN\nGzY4D849J/xg95kpHffA5sJprgQAAFwNL1asnpb0sdC7A6+T1Omcq/fgeQEAAGaVS65YmdkPJd0q\nKcPMaiX9laRISXLOPSZpq6S7JVVI6pP0O9NVLAAAQDi7ZLByzt1/icedpIc9qwgAAGCWYvI6AACA\nRwhWAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEcIVgAAAB4hWAEAAHiEYAUAAOARghUAAIBHCFYAAAAe\nIVgBAAB4hGAFAADgEYIVAACARwhWAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEcIVgAAAB4hWAEAAHiE\nYAUAAOARghUAAIBHCFYAAAAeIVgBAAB4hGAFAADgEYIVAACARwhWAAAAHiFYAQAAeIRgBQAA4BGC\nFQAAgEcIVgAAAB4hWAEAAHiEYAUAAOARghUAAIBHCFYAAAAeIVgBAAB4hGAFAADgEYIVAACARwhW\nAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEcIVgAAAB4hWAEAAHiEYAUAAOARghUAAIBHCFYAAAAeIVgB\nAAB4hGAFAADgEYIVAACARwhWAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEemFKzMbIuZnTCzCjP73AUe\nTzazZ8zsoJkdNbPf8b5UAACA8HbJYGVmQUlflXSXpBWS7jezFecd9rCkMufcGkm3SvqSmUV5XCsA\nAEBYm8qK1SZJFc65SufckKQfSbrvvGOcpEQzM0kJktokjXhaKQAAQJibSrDKk1Qz6XZt6L7JviJp\nuaQ6SYclPeKcG/OkQgAAgFnCq+b1d0o6IClX0lpJXzGzpPMPMrMHzWyfme1rbm726NQAAADhYSrB\n6qykgkm380P3TfY7kp5w4yokVUladv4TOeced85tcM5tyMzMvNKaAQAAwtJUgtVeSaVmVhxqSP+Q\npKfPO+aMpNslycyyJS2VVOlloQAAAOEu4lIHOOdGzOzTkp6XFJT0befcUTN7KPT4Y5L+r6TvmNlh\nSSbpT51zLdNYNwAAQNi5ZLCSJOfcVklbz7vvsUmf10l6h7elAQAAzC5MXgcAAPAIwQoAAMAjBCsA\nAACPEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8AjBCgAAwCMEKwAAAI8QrAAA\nADxCsAIAAPAIwQoAAMAjBCsAAACPEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA\n8AjBCgAAwCMEKwAAAI8QrAAAADxCsAIAAPAIwQoAAMAjBCsAAACPEKwAAAA8QrACAADwCMEKAADA\nIwQrAAAAjxCsAAAAPEKwAgAA8AjBCgAAwCMEKwAAAI8QrAAAADxCsAIAAPAIwQoAAMAjBCsAAACP\nEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8AjBCgAAwCMEKwAAAI8QrAAAADxC\nsAIAAPAIwQoAAMAjBCsAAACPEKwAAAA8QrACAADwCMEKAADAI1MKVma2xcxOmFmFmX3uLY651cwO\nmNlRM3vV2zIBAADCX8SlDjCzoKSvSrpTUq2kvWb2tHOubNIxKZK+JmmLc+6MmWVNV8EAAADhaior\nVpskVTjnKp1zQ5J+JOm+8455QNITzrkzkuSca/K2TAAAgPA3lWCVJ6lm0u3a0H2TLZGUamavmNkb\nZvaxCz2RmT1oZvvMbF9zc/OVVQwAABCmvGpej5C0XtI9kt4p6fNmtuT8g5xzjzvnNjjnNmRmZnp0\nagAAgPBwyR4rSWclFUy6nR+6b7JaSa3OuV5JvWa2TdIaSSc9qRIAAGAWmMqK1V5JpWZWbGZRkj4k\n6enzjnlK0tvMLMLM4iRtlnTM21IBAADC2yVXrJxzI2b2aUnPSwpK+rZz7qiZPRR6/DHn3DEze07S\nIUljkr7lnDsynYUDAACEm6lcCpRzbqukrefd99h5t/9e0t97VxoAAMDswuR1AAAAjxCsAAAAPEKw\nAgAA8AjBCgAAwCMEKwAAAI8QrAAAADxCsAIAAPAIwQoAAMAjUxoQivDwg91npnTcA5sLp7kSAABw\nIaxYAQAAeIRgBQAA4BGCFQAAgEcIVgAAAB4hWAEAAHiEYAUAAOARghUAAIBHCFYAAAAeIVgBAAB4\nhGAFAADgEYIVAACARwhWAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEcIVgAAAB4hWAEAAHiEYAUAAOAR\nghUAAIBHCFYAAAAeIVgBAAB4hGAFAADgEYIVAACARwhWAAAAHiFYAQAAeIRgBQAA4BGCFQAAgEcI\nVgAAAB4hWAEAAHiEYAUAAOARghUAAIBHCFYAAAAeIVgBAAB4hGAFAADgEYIVAACARwhWAAAAHiFY\nzQLOOf38WKMqmnr8LgUAAFwEwWoW2H+mQ7843qTv7z6tlu5Bv8sBAABvgWAV5noGR/Szw/XKS4lV\nMGD6wZ4zGhoZ87ssAABwAQSrMPfs4XoNjYzpfevz9YENBWrsGtBTB87KOed3aQAA4DwEqzBW3tSt\n/TUdunlJhrKTYrQkO1G3LcvS/poO7Tvd7nd5AADgPASrMDU0MqanDtQpPT5Kty7NOnf/25dlaXFW\ngp45WKe6jn4fKwQAAOcjWIWpl080qa13SO9dl6fI4C//mAJm+sCGAsVFBfWDPWfUPzTqY5UAAGAy\nglUYqu/s1/byZq0vTFVJZsKvPZ4QHaH7NxWqo29Izx1t8KFCAABwIVMKVma2xcxOmFmFmX3uIsdt\nNLMRM3ufdyXOL845Pbn/rGIjg7prVc5bHrcwPV6r8pJVVt+lMRrZAQAIC5cMVmYWlPRVSXdJWiHp\nfjNb8RbHfVHSC14XOZ80dA2opr1fty/PVlx0xEWPXZKVqN7BETV0DsxQdQAA4GKmsmK1SVKFc67S\nOTck6UeS7rvAcX8g6SeSmjysb96paumVJC3NTrzksYuzxi8TMpEdAIDwMJVglSepZtLt2tB955hZ\nnqT3Svq6d6XNT1UtvUqJi1RqfNQlj02KjVR2UrTKm7pnoDIAAHApXjWvf1nSnzrnLjoS3MweNLN9\nZravubnZo1PPHc45VbX0qjg9fspfU5qVqOrWPqaxAwAQBqYSrM5KKph0Oz9032QbJP3IzKolvU/S\n18zsPec/kXPucefcBufchszMzCssee5q6h5U39CoijOmHqwWZyVodMypurV3GisDAABTMZVgtVdS\nqZkVm1mUpA9JenryAc65YudckXOuSNKPJf2+c+5Jz6ud4yb6qy4nWBVnxCsiYCpv5HIgAAB+u/jb\nziQ550bM7NOSnpcUlPRt59xRM3so9Phj01zjvFHV0qukmAilTaG/akJkMKCijHiV08AOAIDvLhms\nJMk5t1XS1vPuu2Cgcs799tWXNf9M9FctyoyXmV3W15ZmJejZIw3q7B9WcmzkNFUIAAAuhcnrYaKl\nZ0g9gyMqzvj1SeuX8suxC1wOBADATwSrMHEl/VUTcpJilBAdweVAAAB8RrAKE1UtPUqIjlBGwtT7\nqyaYmUqzElTR1MP2NgAA+IhgFQbOza/KuPz+qgmLsxLUNzSq+g62twEAwC8EqzDQ1jukroGRK7oM\nOGGiz4op7AAA+IdgFQaupr9qQmJMpBYkx9BnBQCAjwhWYaCqpVdxUUFlJUZf1fMszkrQmdY+9Q6O\neFQZAAC4HASrMFDVenX9VRNKsxI16px2V7V6VBkAALgcBCuf1bb3qaNv+KouA05YmB6nyKBp28kW\nDyoDAACXi2Dls92VbZKurr9qQmQwoKL0eO08xYoVAAB+IFj5bHdVq2Ijg8pOivHk+QrS4lTe1E2f\nFQAAPiBY+Wx3VZuK0uMUuMr+qgn5qbEac9KRs52ePB8AAJg6gpWPGjoHdLq1z5PLgBPyU+MkSYdq\nCVYAAMw0gpWPDtV2SJIK0+I8e86E6AjlpcTqYOi5AQDAzInwu4D5rKy+S2ZSTnKsp8+7Oj+ZFSsA\nwFv6we4zUzrugc2F01zJ3MOKlY+O1XepOD1eURHe/jGszk/RmbY+tfcOefq8AADg4lix8lFZfZdW\n56d4/rxr8pMlSYfOduqWJZmePz8AIHxNdTUK04MVK5909g+rpq1fKxYkef7cqyaCVQ19VgAAzCSC\nlU+O13dJ0rQEq6SYSJVkxusgfVYAAMwogpVPjk0Eq1zvg5UkrclPOfeuQwAAMDMIVj4pq+9SenyU\nshKjp+X5V+cnq6l7UA2dA9Py/AAA4NfRvO6TsvourchNknk0cf18E03xB2s7lJOcMy3nAABcPZrN\n5xaClQ+GR8d0srFHv31D0bSdY2VukiICpkO1HXrnSoIVAIS7MefU1jOk9v4h9Q+Nqm9oVH1DI+ob\nGlUwYMpOjFF2UowyE6M9H9MD7xCsfFDZ3KuhkbFpaVyfEBMZ1JLsRAaFAkAYcs6poqlHB2s79cyh\nOtV19Ku+c0BDI2O/dmx0REAjY06jY06SZJJS46O0MC1Oty/PVlp81AxXj4shWPmgrH487ExX4/qE\nNQXJ+tmhejnnpu2SIwDg0pxzKm/q0a7KVu2qbNXuyja1hoY4RwZNC5JjdW1hinKTY5WeEK24qKDi\nooKKjQoqIhDQ6JhTW++QGrsGzn0cqevU4bOdunlJpm5ZkqnIIKtY4YBg5YOyui5FRQRU4uHmyxey\nOj9FP9xTo2qPN3oGAFxac/egtpc3a9vJZu2oaFFLz3iQyk2O0S1LMnVdSbquXZiiXZVtClzih99g\nwJSZGK3MxGityhufVdjRN6RnjzToF8eb9OaZdt29aoFWTmPvLqaGYOWDY/XdWpqdqIhp/uli9cSg\n0NoOghUAeOhCDeejY05n2vp0srFbJxu7VR96V3Z6fJTeVpqhGxdl6LqSdBWkxf5K+NlT1X5FNaTE\nRen+TYXa3Nyjnx6q1w/2nFFpVoIe2FSo6Mjglf3GcNUIVjPMOaey+i7duTx72s+1JDtRMZEBHazp\n1H1r86b9fAAw3/QNjai8sUfHG7p0srFH/cOjCpi0MD1e71iRrdLsRC1IjlHATCNjTjsqWjyvoSQz\nQQ/ftli7q1q19XC9/nNfjT5y3cJLroJhehCsZlhj16Daeoemvb9KkiKDAa3MTWZQKAB46Exrn3ZU\ntOhYfZdOt/ZqzEnxUUEtX5CopTlJKs1KUMwMrxgFA6YbFmXIzPTMwTo9d6RBd1+zYEZrwDiC1Qyb\n7onr51udn6wf7jmjkdGxab/0CABzkXNOh2o79WJZo14sa9SJxm5JUnZStG4uzdSyBUnKT40NixWi\n60vS1dw9qB0VLcpMjNbGojS/S5p3CFYzrCwUrJblJM7I+dbkp+jfXqtWeVOPlk/jeAcAmEuGRsa0\ns7JVL5Y16OdlTWroGlDApI1FafqLe5arf2hU6QnTs3PG1brnmgVq7RnUUwfOKi0+SosyE/wuaV4h\nWM2wsrouFabFKTEmckbON7mBnWAFAG+trXdIr5xo0kvHm7TtRLO6B0cUGxnUzUsy9P+tWKq3L8tS\namhmVDhPSw8GTPdvKtRjr57SD3af0aduXaSMMA2BcxHBaoaV1XdN62DQ8xWlxysxJkIHazv1wY0z\ndloACHvOOZ1o7NYvjjfpF8fGRxaMOSkxOkJLcxK1YkGSFmUlKDIY0ODImJ490uB3yVMWExnUx64v\n0tdeqdC/v16th29bPON9X/MVwWoG9Q6OqLq1V+9dN3Pv0AsETKvzaWAHAElq7x3S9ooWbTvZrO3l\nzWrsGpQkrcpL0qffXqrbl2Xp8NnOsOiXulpp8VH68OaF+ub2Sm0vb9adK9jebCYQrGbQ8YZuOacZ\nvyS3Oj9F39xWqYHhUX5iATCv9AyOaG91m3adatXOylYdPtsp56Tk2Ei9rTRDt5Rm6uYlmcpJjjn3\nNUfrunys2FvFGfG6Ji9ZOypadF1J+oy1ocxnBKsZVDbD7wicsCY/RSNj4/Ozri1MndFzA8BM6uwf\n1ptn2rW3qk27Klt1qLZTI2NOkUHTuoJUPXJ7qW5ekqk1+SkKBmb/qtRU3LkiW0frOvXyiWbduybX\n73LmPILVDDpW36Xk2EjlTvrJaCasKQg1sNd0EKwAzBnOOdW29+tfflGh0629Ot3ap8auATlJAZPy\nUmJ14+IMLcpMUGFanKIixkfOHK/v1vH6bn+Ln0EZCdFavzBNe6va9LbFGWzaPM0IVjOorK5Lyxck\nzvg+TjlJMcpKjNbB2s4ZPS8AeKl/aFSHaju0v6ZDb55u15tnOtTSM94jFR0RUGFanFblZWlherwK\nUn8ZpCC9fVmW9p9p10vHGvX+DQV+lzOnEaxmyOiY0/GGLj2waeGMn9vMtKYgRQdpYAcwS4yNOVW1\n9mr/mQ7tP9OuAzUdOt7QrdExJ0kqSo/TzaUZWrcwVY2dA8pOipk3l/auRHJspG5YlK7t5S26qfRX\ne8rgLYLVDKlq6dXA8NiM91dNWJOfrBfLGtXZP6zkWJoXAYSXjr4hHajpGA9SNR06cKZdXQMjksZX\nowpSx4NUQWqc8tPilBD9y29fuSmxfpU9q9y8JFN7qtv0QlmDPnZ9kd/lzFkEqxkysZXN8gUzM3H9\nfGsKUiRJh2s79bbSDF9qAABpfDXqyy+Vq6a1T2fa+nS6re/cJT2TlJ0Uo6U5iSpIjVNBWpwyE6Pn\nxPgDv8VFRejm0ky9UNao0629Wpge73dJcxLBaoYcretSZNBUmuVPsFqdNx6sDtZ2EKwAzKi+oREd\nqOnQvup27Tvdrv1n2tUdWo2KiwqqMC1O1xamqCAtTvkpsYpmLMy0uWFRhl4/1arnjzbokzeVzHjP\n73xAsJohZfVdKs1K9K2ZMjkuUsUZ8TpYQ58VgOnV2jOovdXt2lvdpr3VbTpa16XRMSczaUlWot69\nJldDI2MqTItTenwU39xnUFREQG9flqWnD9apornHtx/25zKC1Qwpq+vSrUszfa1hTX6ydlW2+VoD\ngLmnsWtAuypbtbuqTS+WNaq5e/yyXkTAlJ8ap5sWZ2hherwK0+IUG8VqlN82FKXqpWON2l3ZRrCa\nBgSrGdDUNaCWnkGt9KlxfcLq/BQ9eaBOjV3j76ABgMsxsfFw18Cwqpp7VdnSq6qWHrX0DEkabzJf\nmB6nawtSVJQRr7yUWEUEGXkQbiICAa1fmKodFS3qGhhWEtPYPUWwmgFHJyauz/BWNuebaGA/WNOh\nd6xkzygAU9PRN6RdlW16+uBZnWruPbciFR0RUHFGvDYWpakkI0ELUmJoMp8lNhSlaVt5i9483a5b\nl2b5Xc6cQrCaAWWhfaeW+7xitTI3SREB08FaghWAtzYwPKo3TrdrR0WLdpS36Ejd+P56kUFTcUa8\n1hemqiQzXrkpsQSpWSojIVolmfHaW92mm5dk8ufoIYLVDCir71JhWpzvy60xkUEtzUnUwRomsAP4\nJeecTjR2a9vJZm0vb9He6jYNDI8pImBaV5iiR24v1Q2LMnS8oUsRAS7tzRWbitL0o701OtXUo9Js\neq28QrCaAWV1Xb5fBpywOj9FPztUp7ExpwBTioE5b6Iv6nw9gyOqaOpRRVO3atv71RS6vFealaD7\nNxXqbYsztLkk/VcGcVY09cxIzZgZKxYkKS4qqD3VbQQrDxGsplnP4IiqW3v13nV5fpciSVpbkKwf\n7jmj6tZelWQm+F0OgBkyOuZ0pq1P5Y3dKm/qUV1Hv5yk2MigFmcl6KbSDC3OSjy3M0Nj16CePlDn\nb9GYVhHBgNYXpuq1Uy3qHhhWIk3sniBYTbMTDV1yzv/G9QkTDeyHajsJVsAcV9PWp91VrSpv7NGp\n5h4NjowpYFJBWpxuX56tJdkJ9EnNcxuL0rS9YryJ/Raa2D1BsJpmR0ON6yvzwiNYLc5MUGxkUAdq\nOvSeMFlFA+CNnsER7a5s1baTzdpW3qKqll5JUkpspFbnJ6s0K1GLMhOYJYVzMhKjVZIRr72n23UT\nTeyeIFhNs7K6LqXGRSonTOZGRQQDuiYvWYdqmcAOzHaDI6N683SHdp5q0WunWnWwpkMjY06xkUFd\nV5Kmj12/UF39I8pIYLo53trG4jT9594aVTb3anEWVzKu1pSClZltkfTPkoKSvuWc+8J5j39Y0p9q\nfP/Mbkmfcs4d9LjWWamsvksrcpPC6j+11fnJ+u6u0xoeHVMkw/uAWaN3cET7z3Sc2yrmzTPtGhge\nv7y3Oj9Fv3dLiW5clKH1RamKjhhflXqr5nVgwspJTewEq6t3yWBlZkFJX5V0p6RaSXvN7GnnXNmk\nw6ok3eKcazezuyQ9LmnzdBQ8m4yMjul4Q7d+6/qFfpfyK9YUpGhoR5VONHRrVV6y3+UAuADnnGrb\n+7W/pkMHznTojdNtOjJpz72cpBitK0zV4swEFWfEKya0cXF1a5+qW/t8rh6zSUQwoGsLU/U6Teye\nmMqK1SZJFc65Skkysx9Juk/SuWDlnHt90vG7JOV7WeRsdaq5V0MjY1qZG17hZe3EBPbaDoIVECba\ne4d06GynDtV06EDoo7X3l1vFrMlP0UO3lGhjUZquXZiqnx6s97lizCUbi9K0o6JFB2o6dFOpv/va\nznZTCVZ5kmom3a7VxVejPi7p2aspaq4oqx8fxLnC54nr58tPjVVqXKQO1nTow5vDazUNmMsmLssN\nj47pbHu/atr7VNver9r2PrX3DZ87blFmvG5dmqW1hSlaV5CipTmJXLbHtMpMjFZBaqwOEqyumqfN\n62Z2m8aD1dve4vEHJT0oSYWFhV6eOiyV1XUpOiKgkox4v0v5FWamNQUpOlTLBHZgJtS292lvdZue\nOnBWte39qu/s15gbfywlNlJ5qbHaVJyu/NRYPXJHqe+7NGB+Wp2fop8drldT94CyEsPjDVez0VSC\n1VlJBZNu54fu+xVmtlrStyTd5ZxrvdATOece13j/lTZs2OAuu9pZ5mhdl5blJIbl7u5r8lO07WS5\negZHfmWyMoCr45xTVUuv9lS1aU9Vm3ZXtelsR78kKSoioPzUWN1UmqnCtDjlp8b+Wj8Ll/jgl2vy\nk7X1cL0O1XbqjuUEqys1le+oeyWVmlmxxgPVhyQ9MPkAMyuU9ISkjzrnTnpe5SzknFNZfZfuWhWe\nmx1vKk7TmJP2VrfpNobCAVfl8W2VOtXcE9oipked/eOX9eKjI1ScHqdrC1NUlBGv7KQY5gQhbCXF\nRKo4M14Hazp0+7KssHo3+2xyyWDlnBsxs09Lel7j4xa+7Zw7amYPhR5/TNJfSkqX9LXQH8SIc27D\n9JUd/uo7B9TRNxw2E9fPt35hqqKCAe081UqwAi7TyOiY3jzToVdONOnVk83nBgHHRAa0KDNBty7N\nVElGAvOjMOuszU/RE/vPqq5jQHmpsX6XMytN6RqQc26rpK3n3ffYpM8/IekT3pY2u038RxtujesT\nYiKDWluYop2nLnjVFsB5mroG9MrJZr16olnbypvVPTCiYMC0vjBVd67I1uLMBOWlsj0MZreVucl6\n6kCdDtZ2EKyuEM0106Ssrktm0rKc8AxWknTDonQ9+lK5OvuGlRxHsyww2cjomPbXjK9KvXy8WWX1\n4z8sZSVG6+5VC3Tr0kzdWJqhpJhIhnBizoiNCmpJdoIO1XZoS5i2soQ7gtU0KavvVHF6vOLDuDH8\n+pJ0ffnn5dpd1ap3rOQfEHC2o1/bTzZre3mLtpc3q2tgRAGTCtPi9c4V2VqSk6icpBiZmdr7hmk0\nx5y0uiBFxxq6Vd3a63cps1L4ftef5crqu7Q6P8XvMi5qbWGKoiMC2llJsML81DUwrD2VbdpR0aJt\n5c2qbB7/RpKTFKMtq3IUDATGNy5n02LMI8tzkhQZNB2qYSTPlSBYTYPO/mHVtPXrQxvDe1ZXdERQ\nG4vS6LPCvNE/NKp9p9v0+qlWPXOwTmfb++UkRQZNxRnxuueaBVqclaCsxGiazjFvRUUEtHxBko7U\ndbKn7BUgWE2DY6FejJVh2rg+2fWL0vX3z59Qa8+g0hOi/S4H8FTP4Ij2VY/PktpT1aZDtR0aHnWK\nCJjyUmJ169IsLcqMV0FaHN88gEnW5o8Pkd5R3qLblvHO8ctBsJoGR86G51Y2F3JdSbokaXdVm+6+\nZoHP1QBXp7NvWHur27Snuk27K1vPbVocETBdk5+s331bsa4rSdemojQ9daDO73KBsLU4O0GxkUE9\ndeAsweoyEaymwRun25WfGjsrtgRYnZ+s+KigXj/VQrDCrNPWO6Q9Va3aVTm+KnW8oUvOScGAhSac\nZ6g4I16FaXGKjhjvk6rvGCBUAZcQEQhoVV6SXihrVP/QKH2Gl4Fg5THnnPZWt+um0gy/S5mSyGBA\nG4vps8Ls8K/bq1TZ0qPKll5VNfeqoWtA0niP1MK0eN2+LEvFGQnKT43l0h5wlVbnp2hvdbteOt6o\nd63O9bvtpli2AAAZZ0lEQVScWYNg5bHTrX1q6RnUhqJUv0uZsutL0vXKiWY1dQ0oKyn8V9kwfwyO\njOqN0+16raJFOypadaim41yz+cK0eN25IlslGfHKS41VRIAgBXipOCNe2UnReupAHcHqMhCsPLa3\nuk2StLEozedKpu6GReOrazsrW3Xf2jyfq8F8NDFg0zmn1p4hnWzq1snGblW19Gp41ClgUkFqnG5b\nlqVFmQkqSCNIAdMtYKZ3r87Vd3eeZpD0ZSBYeWxfdbuSYyO1ODPB71KmbEVukpJiIrTzFMEKM69v\naETH67t0onE8TLX3jW9gnJEQpQ0L07Q4K0HFGfGKiaTHA5hp963N07d2VOm5o/X6YJiPEAoXBCuP\n7T3dpg0LUxUIzJ4ZOMGAaVNxunZW0meFmXG6tVcvH2/SL040a1dlq4ZGxhQZNC3KTNBNpZlakp2o\ntPgov8sE5r1VeUkqyYjXUwfqCFZTRLDyUGvPoCqbe/X+9QV+l3LZbliUrp8fa9TZjn7lpbDxJrw1\nODKqvVXtevlEk14+0XRuwnlJRrw+snmhnJyK0+MVQcM5EFbMTPeuzdU/v1Suxq4BZdOHe0kEKw/t\nO90uSdo4ixrXJ1y/aHye1c5TrXrf+nyfq8FcUNPWp23lzXrlRLNeq2hR39CooiICur4kXR+9bqFu\nW5qloox4SWITYyCM3bsmV1/+ebmeOVinT9xU4nc5YY9g5aF91W2KigjomvxkX+uYyjepBzb/6pLu\n0uxEpcZFEqxwxQaGR7W7qk2vnmjWqyebdCq0KpUSF6lr8pK1NCdRJRkJiooYX5V6/VSrXmfMBxD2\nSjITdE1esp4mWE0JwcpDe6vbtSY/+dwgwtkkEDBdvyhdO0+1yDnHPmm4JOecjjd0a3t5s7aXt2h3\nVZuGRsYUHRHQdSXp+vDmheoeGFFGQhR/n4BZ7r61ufqbnx1TZXOPSmbRm7P8QLDySP/QqI6c7dQn\nb569af76knRtPdygypZeLeIfDi6gqWtA28tbtKOiRS+WNapncESSlJUYrY0LU1WanajijPhzwzl5\nJx8wN7xrda7+dusxPXWgTp+9c4nf5YQ1gpVHDtR0aGTMzcr+qgl3rMjWXz59VM8crNNn7uAfDsZH\nIeyuatP2ky3aUdGsk409kqS0+CiVZMarNCtRi7MSlBzLfBtgLstJjtF1xel6+mCdPnNHKavQF0Gw\n8si+6jaZSesLZ89g0PMtSI7V9SXpenL/WT1yO/9w5qOxMacvvXhSFY3dKm/q0em2vnObGBelx2vL\nyhwtzkpQTnKMAvz9AOaV+9bm6nNPHNbhs51anZ/idzlhi2Dlkb2n27U0O3HWT6Z9z7o8/cmPD2l/\nTYeuLZy9q2+YupaeQW0vb9arJ8Z7pVp7hyRJC5JjdMOidC3OSlBRejx77wHz3F2rFujzTx3RUwfq\nCFYXQbDywOiY05un2/WedbN/L6W7VuXo808e0ZP7zxKs5qjRMacDNe16+XizXj3ZrMNnOyVJ6fFR\nuqk0Q5HBgBZnJSgxZnb/kADAW8lxkbp1aZaeOVin/3X3cgVn0SDsmUSw8sCx+i71DI7Mqv0B30pi\nTKTuXJGtZw7W6S/uWXHurfGY3Tr7h/V3W4/peMP4tjF9Q6Pj+++lxenOFdlakpWoBSlc3gNwcfet\nzdWLZY3aXdmqGxZn+F1OWCJYeWBfaOPlDXMgWEnSe9fl6aeH6rXtZLPuWJHtdzm4QjVtfXqxrFEv\nljVqT3WbRsec4qKCWpKdqGU5iSrNSlRsFO/aAzB1ty/LVnxUUP+z/yzB6i0QrDyw93S7cpNj5sxW\nMDcvyVRafJT+58BZgtUs4pzT0bouvVDWqBeONuh4Q7ckaUl2gn7v5hKNjjkVpMWxKgXgisVGBXXP\n6gX66aF6ff7dK5REy8CvIVhdJeec9lW3aXNxut+leCYyGNC7Vy/QD/fWqGtgmH84YWxszGl/TYf+\n6cWTOlrXqfa+YZmkhelxumtVjlYsSFJ6QrTfZQKYQz68eaH+a1+tntx/Vh+7vsjvcsIOweoq1bb3\nq7FrcFbPr7qQ96zL07/vPK3nDjfoAxtn36bSc9nomNPe6jY9e7hezx1tUGPXoIJmWpQVr9uWZmnZ\ngiQlRPNPG8D0WJ2frFV5Sfr+rjP66HULGc1zHv73vUp751h/1YS1BSkqzojXE/trCVY+mtj3cXTM\nqbq1V0fOdupo3fibJSICpiXZibplSaaW5SQx5RzAjDAzfWTzQn3uicN643T7nPv+d7UIVldpR0WL\nkmIitCQ70e9SPGVmes/aPP3Tz0/qbEf/nOkfm01GRsdU0dQTClOd6h0aVWTQtDQ7UatCmxrPxn0p\nAcx+716Tq7/92TF9b9dpgtV5CFZXYWB4VC8ebdSWVTlzcp7He9bl6p9+flJPH6jTp25d5Hc588Lw\n6Jh2VbZq6+F6PX+0UW29Q4oMmpblJI2HqexERmAA8F18dITee22efrSnRn/57iGlxUf5XVLYIFhd\nhVdONKl7cET3rp39g0EvZGF6vNYvTNX/7K/VQ7eUcB19mgyOjOq1ihY9e7hBLx5rVEffsOKjgnr7\n8mwlRo+vhhKmAISbD29eqO/uPK0fv1GjB2/mh+8JBKur8PTBOmUkROn6krnzjsDzvWddnj7/5BEd\nqu3UmgK2MPBK39CItp1s1nNHGvTSsfGAnhgdoTtWZOuuVTm6eUmmYiKD53qsACDcLM1J1MaiVH1/\n9xl94m0lCszBKzdXgmB1hXoGR/TSsSZ9cGOBIubwHmr3rsnVl144oS88e1w/+ORmVq2uQmffsF46\n3qjnjzbo1ZPNGhgeU0pcpO66Jkd3rVqgGxan0zMFYFb5yHUL9ciPDui1Uy26qTTT73LCAsHqCr1Y\n1qDBkTHdu2ZuXgackBwbqT+6c4k+/9RRPX+0QVtWLfC7pFmlsWvg3MDOnadaNTLmlJMUow9uGA/k\nRenxCgZM9Z0D+skbZ/0uFwAuy5ZVOUqLj9L3dp0mWIUQrK7Q0wfqlJcSOy82Kr5/U6G+t+uM/nbr\nMd26NIu39V/Coy+Vq6yuS0frOlXT3i9pfIPjGxala2VusvJSY5l+DmBOiI4I6v3r8/WtHVVq6BxQ\nTnKM3yX5jmB1Bdp7h7S9vEUfv6l4XlxTjggG9JfvXqEPf2u3/nVHlR6+bbHfJYUV55zK6rv0/JEG\nPX+0UScax7eSyU2J0R3Ls7UyN0lZidFcRgUwJz2wuVDf2Fap/9xbo0fuKPW7HN8RrK7A1iP1Ghlz\nevfquX0ZcLIbF2fonSuz9dWXK/S+9fnKTprfP5WMjTm9eaZdzx1p0PNlDapp61fAxgfF3nPNAq1Y\nkKRU3n4MYB5YmB6vm5dk6j92ndYnby5WXNT8jhbz+3d/hZ4+UKeSzHitzE3yu5QZ9ed3r9Ad//iq\nvvjccf3jB9b6Xc6M+4+dp1XZ0qOjdV06Vtel7sERBQOmxZkJeu+6PC1nKxkA89Qjty/Wb359p/51\ne5X+4Pb5vWrFd4HL1NA5oD3VbXrk9tJ5d2mnMD1OH7+pWF9/5ZQ+et1CrZsH/WX9Q6PaVt6s5480\naOuReg0MjykqGNCSnEStzE3S0uxEes4AzHvrF6bpHSuy9Y1tlXpgc+G83vydYHWZfnqoTs6Nj/Of\njx6+bbF+/Eat/vqZMj3xqRvmZI9ZZ/+wXj7epOePNuiVE83qHx5VcmykVixI1srcJC3OSlDkHB6x\nAQBX4k+2LNM7/ulV/csvKvS/713pdzm+IVhdpmcO1mllbpIWZSb4XYovEqIj9KdblumP//ugvrGt\ncs5sddPUNaDnzxuLkJUYrfetz9eWVTnaVJym/95X63eZABC2Fmcl6IMbC/T93af1uzcWqzA9zu+S\nfEGwugzVLb06WNupP7trmd+l+Oo31uXpF8cb9cXnjmvMuVn5LkHnnMqbevRiWaN+fqxR+890SPrl\nWIQVC5KUnxangJlOt/bpdGufzxUDQPj7zB1L9D/7z+ofXjihR+9f53c5viBYXYZnDtZJkt41By4D\nTnWrlAc2F/7afYGA6dEPrVNU8KD+/vkTGhwe1WfvXBL2PWfDo2PaW92ml4416efHGs+FpdX5yYxF\nAAAPZCfF6BNvK9FXXq7QJ28q0TX5yX6XNOMIVlPU2Tes77xerRsWpSsvJdbvcnwXEQzoSx9Yq+iI\noB79RYUGRsb0Z3ctC7tQ8tirp1Te2K3jDd2qaOrR4MiYggHTosx43bc2V8tykpQcG+l3mQAwZ/ze\nLSX6/u7T+uJzx/W9T2z2u5wZR7Caoi+9eELtfUP683uW+11K2AgGTH/3G9coOjKgx7dVanB4VH/1\n7pW+NrQPDI9qX3W7dlS0aEdFs46c7ZIkJcVE6Jq8ZC3NSdTizARF804+AJgWiTGR+oO3l+r//LRM\n20426+Yl82urG4LVFBw526nv7Tqtj11fpJW5829Z82ICAdNf37tSMZFBPb6tUsfqu/X7ty3SLUsy\nZ2T1anBkVEfOdmpPVbteq2jR3uo2DY6MKTJoWleQqnesyNbSnETlJMWE3WoaAMxVH76uUP/2epX+\n39Zj2lySNq82mCdYXcLYmNPnnzqitPgoffbOJX6XE5bMTH921zIVpsXpqy9X6Lf/ba9WLEjSQ7cu\n0t2rchTh0WgC55yaugd15Gyn9p1u177qNh2s7dTQyJgkaVlOoj5y3UK9bXGGNhWnKT46Ysq9ZAAA\n70RHBPVX71qpT3x3n/7XE0f0D+9fPW9+uCVYXcKP36jV/jMd+tL719CLcxFmpo9ct1Af2FCgJw+c\n1WOvntIf/nC//iEtTveuydXSnEQtyU5UcUa8oiIuHrSGR8fU0Dmghq4BnW7t07H6Lh2r79Lxhm61\n9Q5JkoJmyk2J0aaiNBWlx6kwPf7c1PP6zgE9daBu2n/PAIC3dseKbH3mjlJ9+eflWpaTqE/eXOJ3\nSTOCYHURHX1D+sJzx7VhYap+49o8v8uZFaIiAvrAhgK979p8vVDWqG9ur9TXXz2l0TEnSYoImEoy\n45WdFCPnJCc3/quT+oZGVN85oOaeQTn3y+eMjghoaU6i7lyereULErUiN1lldV2XDGgAAH/94dtL\ndbKxW3/37DEtzkrQbcuy/C5p2hGsLuJLL5xUR9+Q/s99m+fNEqZXAgHTllU52rIqR4Mjo6ps7tXJ\nxm6daOjWycZutfYOyTS+0jX+q5QUG6llOUlq7hlUSmykkmMjlRIXpfSEKAUmvf4VTT2EKgCYBQIB\n0z+8f42qW/r0hz/cr/95+AYtzkr0u6xpRbB6C4drO/W93af1W9cXacU822x5squZdzUhOiKo5QuS\ntHzB1F5H+qIAYO6Ii4rQN39rg+77yg59/N/36amHb1RKXJTfZU0bgtUF1Lb36bP/dUDp8dE0rE8R\nYQgA8FbyUmL12EfW6/5v7tKnvvemHv/YeiXGzM2+Za6nnGdfdZvu+8prauwa0KMfWkvDOgAAHthQ\nlKYv/MZq7a5q1V3/vF17q9v8LmlaEKwm+a99Nbr/m7uUFBupJx++UTcszvC7JAAA5ozfXJ+v/37o\negXM9IFv7NQXnzt+bmTOXEGwkjQ65vQ3Py3Tn/z4kDYXp+vJ379RizIT/C4LAIA5Z/3CNG195CZ9\nYH2Bvv7KKb33a6+pvLHb77I8M6VgZWZbzOyEmVWY2ecu8LiZ2aOhxw+Z2bXel+q9/qFRPXOwTvc/\nvkvf2lGl376hSN/5nY1KjuPyHwAA0yUhOkJffN9qfeOj61XfOaB7Ht2h3//+G3ruSIMGhkf9Lu+q\nXLJ53cyCkr4q6U5JtZL2mtnTzrmySYfdJak09LFZ0tdDv4adkdExvX6qVU8eOKvnjzSod2hUOUkx\n+uJvXqMPbnzrd7YBAABvvXNljq4tTNVXX67QTw/VaevhBiXGRGjLyhzduzZXawpSlDTLmtyn8q7A\nTZIqnHOVkmRmP5J0n6TJweo+Sd91zjlJu8wsxcwWOOfqPa94inoHR3SwtkM1bX0609anM239OtPW\np6rmHnUNjCgxJkLvWp2r+9blanNxuoI+bhwMAMB8lZkYrf9970r9xT3LtbOyVU8dqNOzRxr032/U\nSpIyEqK1KDNeJZkJWpQZr4yEaCXGRCgxJlJJseO/psZFKi4qPAYdTKWKPEk1k27X6tdXoy50TJ4k\n34JVVUuvHvjmbknj077zUmNVmBand63J1U2LM3TbsizFRM6fTSEBAAhnEcGAbirN1E2lmfqb96zS\nzlOtOtnYrVPNPTrV3Ktnj9Sro2/4gl97/6ZC/d1vXDPDFV/YjMY7M3tQ0oOhmz1mdmKmzn1q0ud/\nN1MnnboMSS1+FzHH8Jp6j9fUe7ym3uL19NiH/S5gir4Q+phmC6dy0FSC1VlJBZNu54fuu9xj5Jx7\nXNLjUylsPjGzfc65DX7XMZfwmnqP19R7vKbe4vVEOJjKuwL3Sio1s2Izi5L0IUlPn3fM05I+Fnp3\n4HWSOv3srwIAAPDDJVesnHMjZvZpSc9LCkr6tnPuqJk9FHr8MUlbJd0tqUJSn6Tfmb6SAQAAwtOU\neqycc1s1Hp4m3/fYpM+dpIe9LW1e4fKo93hNvcdr6j1eU2/xesJ3Np6JAAAAcLXY0gYAAMAjBCuf\nXWq7IFweM/u2mTWZ2RG/a5kLzKzAzF42szIzO2pmj/hd02xnZjFmtsfMDoZe07/2u6a5wMyCZrbf\nzH7qdy2Y3whWPpq0XdBdklZIut/MVvhb1az3HUlb/C5iDhmR9EfOuRWSrpP0MH9Hr9qgpLc759ZI\nWitpS+jd1Lg6j0g65ncRAMHKX+e2C3LODUma2C4IV8g5t01Sm991zBXOuXrn3Juhz7s1/o0rz9+q\nZjc3rid0MzL0QbPrVTCzfEn3SPqW37UABCt/vdVWQEDYMbMiSesk7fa3ktkvdNnqgKQmSS8653hN\nr86XJf2JpDG/CwEIVgAuycwSJP1E0mecc11+1zPbOedGnXNrNb5LxSYzW+V3TbOVmb1LUpNz7g2/\nawEkgpXfprQVEOAnM4vUeKj6vnPuCb/rmUuccx2SXhZ9gVfjRkn3mlm1xtsp3m5m3/O3JMxnBCt/\nTWW7IMA3ZmaS/lXSMefcP/pdz1xgZplmlhL6PFbSnZKO+1vV7OWc+zPnXL5zrkjj/4f+wjn3EZ/L\nwjxGsPKRc25E0sR2Qcck/Zdz7qi/Vc1uZvZDSTslLTWzWjP7uN81zXI3SvqoxlcBDoQ+7va7qFlu\ngaSXzeyQxn+4etE5x4gAYI5g8joAAIBHWLECAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8AjB\nCgAAwCMEKwCXZGajoRlWR83soJn9kZkFQo9tMLNHL/K1RWb2wMxV+2vn7g/tyxcWzOyDZlZhZsyu\nAuYgghWAqeh3zq11zq3U+KTwuyT9lSQ55/Y55/7wIl9bJMmXYBVyKrQv35SZWXC6inHO/aekT0zX\n8wPwF8EKwGVxzjVJelDSp23crROrL2Z2y6QJ7fvNLFHSFyTdFLrvs6FVpO1m9mbo44bQ195qZq+Y\n2Y/N7LiZfT+0pY7MbKOZvR5aLdtjZolmFjSzvzezvWZ2yMx+byr1m9mTZvZGaPXtwUn395jZl8zs\noKTr3+KcK0OfHwidszT0tR+ZdP83JoKZmW0J/R4PmtlLHv4xAAhTEX4XAGD2cc5VhsJD1nkP/bGk\nh51zr5lZgqQBSZ+T9MfOuXdJkpnFSbrTOTcQCiY/lLQh9PXrJK2UVCfpNUk3mtkeSf8p6YPOub1m\nliSpX9LHJXU65zaaWbSk18zsBedc1SXK/13nXFton769ZvYT51yrpHhJu51zfxTau/P4Bc75kKR/\nds59P3RM0MyWS/qgpBudc8Nm9jVJHzazZyV9U9LNzrkqM0u77BcawKxDsALgpdck/aOZfV/SE865\n2tCi02SRkr5iZmsljUpaMumxPc65WkkK9UUVSeqUVO+c2ytJzrmu0OPvkLTazN4X+tpkSaWSLhWs\n/tDM3hv6vCD0Na2hWn4Sun/pW5xzp6Q/N7P80O+v3Mxul7Re4yFNkmIlNUm6TtK2iaDnnGu7RF0A\n5gCCFYDLZmYlGg8iTZKWT9zvnPuCmf1M0t0aX0F65wW+/LOSGiWt0Xg7wsCkxwYnfT6qi/8fZZL+\nwDn3/GXUfaukOyRd75zrM7NXJMWEHh5wzo1e7Oudcz8ws92S7pG0NXT50ST9u3Puz84717unWheA\nuYMeKwCXxcwyJT0m6SvuvF3czWyRc+6wc+6LkvZKWiapW1LipMOSNb4aNCbpo5Iu1Sh+QtICM9sY\nOkeimUVIel7Sp8wsMnT/EjOLv8RzJUtqD4WqZRpfVZryOUOBstI596ikpyStlvSSpPeZWVbo2DQz\nWyhpl6Sbzax44v5L1AZgDmDFCsBUxIYuzUVKGpH0H5L+8QLHfcbMbpM0JumopGdDn4+GmsK/I+lr\nkn5iZh+T9Jyk3oud2Dk3ZGYflPQvob6ofo2vOn1L45cK3ww1uTdLes8lfh/PSXrIzI5pPDztusxz\nfkDSR81sWFKDpP8X6tf6C0kv2PgIimGN95ntCjXHPxG6v0nj76gEMIfZeT9wAsCcYWZFkn7qnFvl\ncym/InRJ8lxDP4C5g0uBAOayUUnJFmYDQjW+atfudy0AvMeKFQAAgEdYsQIAAPAIwQoAAMAjBCsA\nAACPEKwAAAA8QrACAADwyP8PVguoclQhBwsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb14b554a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nb_merge_dist_plot(\n",
    "    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n",
    "    SkyCoord(uds['uds_ra'], uds['uds_dec'])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given the graph above, we use 0.6 arc-second radius\n",
    "master_catalogue = merge_catalogues(master_catalogue, uds, \"uds_ra\", \"uds_dec\", radius=0.6*u.arcsec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning\n",
    "\n",
    "When we merge the catalogues, astropy masks the non-existent values (e.g. when a row comes only from a catalogue and has no counterparts in the other, the columns from the latest are masked for that row). We indicate to use NaN for masked values for floats columns, False for flag columns and -1 for ID columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for col in master_catalogue.colnames:\n",
    "    if \"m_\" in col or \"merr_\" in col or \"f_\" in col or \"ferr_\" in col or \"stellarity\" in col:\n",
    "        master_catalogue[col] = master_catalogue[col].astype(float)\n",
    "        master_catalogue[col].fill_value = np.nan\n",
    "    elif \"flag\" in col:\n",
    "        master_catalogue[col].fill_value = 0\n",
    "    elif \"id\" in col:\n",
    "        master_catalogue[col].fill_value = -1\n",
    "        \n",
    "master_catalogue = master_catalogue.filled()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Since this is not the final merged catalogue. We rename column names to make them unique\n",
    "master_catalogue['ra'].name = 'ukidss_ra'\n",
    "master_catalogue['dec'].name = 'ukidss_dec'\n",
    "master_catalogue['flag_merged'].name = 'ukidss_flag_merged'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=10&gt;\n",
       "<table id=\"table140578618877656-746900\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>candels_id</th><th>ukidss_ra</th><th>ukidss_dec</th><th>candels_stellarity</th><th>f_candels-ukidss_j</th><th>ferr_candels-ukidss_j</th><th>f_candels-ukidss_h</th><th>ferr_candels-ukidss_h</th><th>f_candels-ukidss_k</th><th>ferr_candels-ukidss_k</th><th>m_candels-ukidss_j</th><th>merr_candels-ukidss_j</th><th>flag_candels-ukidss_j</th><th>m_candels-ukidss_h</th><th>merr_candels-ukidss_h</th><th>flag_candels-ukidss_h</th><th>m_candels-ukidss_k</th><th>merr_candels-ukidss_k</th><th>flag_candels-ukidss_k</th><th>candels_flag_cleaned</th><th>candels_flag_gaia</th><th>ukidss_flag_merged</th><th>dxs_id</th><th>m_ap_ukidss_j</th><th>merr_ap_ukidss_j</th><th>m_ukidss_j</th><th>merr_ukidss_j</th><th>m_ap_ukidss_k</th><th>merr_ap_ukidss_k</th><th>m_ukidss_k</th><th>merr_ukidss_k</th><th>dxs_stellarity</th><th>f_ap_ukidss_j</th><th>ferr_ap_ukidss_j</th><th>f_ukidss_j</th><th>ferr_ukidss_j</th><th>flag_ukidss_j</th><th>f_ap_ukidss_k</th><th>ferr_ap_ukidss_k</th><th>f_ukidss_k</th><th>ferr_ukidss_k</th><th>flag_ukidss_k</th><th>dxs_flag_cleaned</th><th>dxs_flag_gaia</th><th>uds_id</th><th>m_ap_uds_j</th><th>merr_ap_uds_j</th><th>m_uds_j</th><th>merr_uds_j</th><th>m_ap_uds_h</th><th>merr_ap_uds_h</th><th>m_uds_h</th><th>merr_uds_h</th><th>m_ap_uds_k</th><th>merr_ap_uds_k</th><th>m_uds_k</th><th>merr_uds_k</th><th>uds_stellarity</th><th>f_ap_uds_j</th><th>ferr_ap_uds_j</th><th>f_uds_j</th><th>ferr_uds_j</th><th>flag_uds_j</th><th>f_ap_uds_h</th><th>ferr_ap_uds_h</th><th>f_uds_h</th><th>ferr_uds_h</th><th>flag_uds_h</th><th>f_ap_uds_k</th><th>ferr_ap_uds_k</th><th>f_uds_k</th><th>ferr_uds_k</th><th>flag_uds_k</th><th>uds_flag_cleaned</th><th>uds_flag_gaia</th></tr></thead>\n",
       "<thead><tr><th></th><th></th><th>deg</th><th>deg</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>\n",
       "<tr><td>0</td><td>21539</td><td>34.3795257693</td><td>-5.15683775973</td><td>0.019999999553</td><td>2.8747363</td><td>0.0452537</td><td>3.9708605</td><td>0.0940876</td><td>5.3420701</td><td>0.0563719</td><td>22.7535049676</td><td>0.0170915087026</td><td>False</td><td>22.4027884246</td><td>0.0257259890492</td><td>False</td><td>22.0807260434</td><td>0.011457171399</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>20563</td><td>34.3530670693</td><td>-5.16239645973</td><td>0.0</td><td>1.66879</td><td>0.0438989</td><td>2.0225565</td><td>0.0923951</td><td>2.1947441</td><td>0.0546349</td><td>23.3439957784</td><td>0.0285611880938</td><td>False</td><td>23.1352483439</td><td>0.0495989631005</td><td>False</td><td>23.0465402745</td><td>0.0270277928864</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>2</td><td>27097</td><td>34.3747176693</td><td>-5.14531865973</td><td>0.0</td><td>1.1605744</td><td>0.0502763</td><td>1.51387</td><td>0.1043705</td><td>1.8444729</td><td>0.0626992</td><td>23.7383175334</td><td>0.0470342953897</td><td>False</td><td>23.4497785431</td><td>0.0748537394616</td><td>False</td><td>23.2353193032</td><td>0.036907450063</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>3</td><td>14826</td><td>34.2856193693</td><td>-5.19395515973</td><td>0.0</td><td>1.007157</td><td>0.0422275</td><td>1.353574</td><td>0.0888748</td><td>1.4224256</td><td>0.0519606</td><td>23.8922570612</td><td>0.0455221237468</td><td>False</td><td>23.5712949908</td><td>0.0712887422857</td><td>False</td><td>23.5174261003</td><td>0.039661480109</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>4</td><td>14122</td><td>34.3560180693</td><td>-5.19795715973</td><td>0.019999999553</td><td>4.7127042</td><td>0.0448448</td><td>6.0105577</td><td>0.0941283</td><td>6.4281211</td><td>0.0565669</td><td>22.2168245463</td><td>0.0103315677982</td><td>False</td><td>21.9527130734</td><td>0.0170031648149</td><td>False</td><td>21.8797898753</td><td>0.00955438305618</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>5</td><td>1265</td><td>34.3627124693</td><td>-5.26953255973</td><td>0.019999999553</td><td>0.6432481</td><td>0.0444648</td><td>0.7652047</td><td>0.0936547</td><td>0.9115449</td><td>0.0536024</td><td>24.3790537198</td><td>0.0750519794731</td><td>False</td><td>24.1905559278</td><td>0.132885094061</td><td>False</td><td>24.0005548361</td><td>0.0638455289936</td><td>False</td><td>False</td><td>0</td><td>True</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>6</td><td>13273</td><td>34.3243668693</td><td>-5.20170115973</td><td>0.0</td><td>0.3408017</td><td>0.0415202</td><td>0.2728979</td><td>0.0877595</td><td>0.3974123</td><td>0.0509603</td><td>25.0687456188</td><td>0.13227628961</td><td>False</td><td>25.3099995157</td><td>0.349155000685</td><td>False</td><td>24.9018967388</td><td>0.139224283484</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>7</td><td>9036</td><td>34.2784354693</td><td>-5.22480465973</td><td>0.0</td><td>2.2024794</td><td>0.0400135</td><td>3.2462478</td><td>0.0842658</td><td>3.284873</td><td>0.049583</td><td>23.0427203622</td><td>0.0197250905634</td><td>False</td><td>22.6215458292</td><td>0.0281834399342</td><td>False</td><td>22.6087035413</td><td>0.016388474757</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>8</td><td>30535</td><td>34.3719116693</td><td>-5.21773125973</td><td>0.0</td><td>0.0406514</td><td>0.0429687</td><td>0.1786621</td><td>0.0905867</td><td>0.1386089</td><td>0.0536402</td><td>27.3773112327</td><td>1.14762771421</td><td>False</td><td>25.769918914</td><td>0.550498733976</td><td>False</td><td>26.0455222076</td><td>0.420168597907</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "<tr><td>9</td><td>32778</td><td>34.2738293693</td><td>-5.17910785973</td><td>0.360000014305</td><td>0.161819</td><td>0.042651</td><td>0.0782598</td><td>0.0901892</td><td>0.1113557</td><td>0.0524762</td><td>25.8774262175</td><td>0.286169948332</td><td>False</td><td>26.6661531658</td><td>1.25123856333</td><td>False</td><td>26.2832188692</td><td>0.51165149362</td><td>False</td><td>False</td><td>0</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table140578618877656-746900').dataTable()\");\n",
       "    $('#table140578618877656-746900').dataTable({\n",
       "        \"order\": [],\n",
       "        \"iDisplayLength\": 50,\n",
       "        \"aLengthMenu\": [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        \"pagingType\": \"full_numbers\"\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_catalogue[:10].show_in_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## V - Adding unique identifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_catalogue.add_column(Column(data=(np.char.array(master_catalogue['dxs_id'].astype(str)) \n",
    "                                    +  np.char.array(master_catalogue['uds_id'].astype(str) )\n",
    "                                        +  np.char.array(master_catalogue['candels_id'].astype(str) )), \n",
    "                              name=\"ukidss_intid\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['candels_id', 'dxs_id', 'uds_id', 'ukidss_intid']\n"
     ]
    }
   ],
   "source": [
    "id_names = []\n",
    "for col in master_catalogue.colnames:\n",
    "    if '_id' in col:\n",
    "        id_names += [col]\n",
    "    if '_intid' in col:\n",
    "        id_names += [col]\n",
    "        \n",
    "print(id_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VII - Choosing between multiple values for the same filter\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VII.  UKIDSS DXS and UDS\n",
    "\n",
    "There is no overlap between UDS and DXS. We choose the UDS fluxes over CANDELS-UDS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for col in master_catalogue.colnames:\n",
    "    if 'uds_h' in col:\n",
    "        master_catalogue[col].name = col.replace('uds_h', 'ukidss_h')\n",
    "\n",
    "        \n",
    "\n",
    "has_uds_j =     ~np.isnan(master_catalogue['f_uds_j'])\n",
    "has_uds_h =     ~np.isnan(master_catalogue['f_ukidss_h'])\n",
    "has_uds_k =     ~np.isnan(master_catalogue['f_uds_k'])\n",
    "has_ukidss_k =  ~np.isnan(master_catalogue['f_ukidss_k'])\n",
    "has_ukidss_j =  ~np.isnan(master_catalogue['f_ukidss_j'])\n",
    "has_candels_j =     ~np.isnan(master_catalogue['f_candels-ukidss_j'])\n",
    "has_candels_h =     ~np.isnan(master_catalogue['f_candels-ukidss_h'])\n",
    "has_candels_k =     ~np.isnan(master_catalogue['f_candels-ukidss_k'])\n",
    "\n",
    "use_candels_j = has_candels_j & ~has_uds_j & ~has_ukidss_j \n",
    "use_candels_h = has_candels_h & ~has_uds_h \n",
    "use_candels_k = has_candels_k & ~has_uds_k & ~has_ukidss_k \n",
    "\n",
    "master_catalogue['f_ukidss_j'][has_uds_j] = master_catalogue['f_uds_j'][has_uds_j]\n",
    "master_catalogue['ferr_ukidss_j'][has_uds_j] = master_catalogue['ferr_uds_j'][has_uds_j]\n",
    "master_catalogue['m_ukidss_j'][has_uds_j] = master_catalogue['m_uds_j'][has_uds_j]\n",
    "master_catalogue['merr_ukidss_j'][has_uds_j] = master_catalogue['merr_uds_j'][has_uds_j]\n",
    "master_catalogue['flag_ukidss_j'][has_uds_j] = master_catalogue['flag_uds_j'][has_uds_j]\n",
    "\n",
    "master_catalogue['f_ukidss_j'][use_candels_j] = master_catalogue['f_candels-ukidss_j'][use_candels_j]\n",
    "master_catalogue['ferr_ukidss_j'][use_candels_j] = master_catalogue['ferr_candels-ukidss_j'][use_candels_j]\n",
    "master_catalogue['m_ukidss_j'][use_candels_j] = master_catalogue['m_candels-ukidss_j'][use_candels_j]\n",
    "master_catalogue['merr_ukidss_j'][use_candels_j] = master_catalogue['merr_candels-ukidss_j'][use_candels_j]\n",
    "master_catalogue['flag_ukidss_j'][use_candels_j] = master_catalogue['flag_candels-ukidss_j'][use_candels_j]\n",
    "\n",
    "\n",
    "master_catalogue['f_ukidss_h'][use_candels_h] = master_catalogue['f_candels-ukidss_h'][use_candels_h]\n",
    "master_catalogue['ferr_ukidss_h'][use_candels_h] = master_catalogue['ferr_candels-ukidss_h'][use_candels_h]\n",
    "master_catalogue['m_ukidss_h'][use_candels_h] = master_catalogue['m_candels-ukidss_h'][use_candels_h]\n",
    "master_catalogue['merr_ukidss_h'][use_candels_h] = master_catalogue['merr_candels-ukidss_h'][use_candels_h]\n",
    "master_catalogue['flag_ukidss_h'][use_candels_h] = master_catalogue['flag_candels-ukidss_h'][use_candels_h]\n",
    "\n",
    "\n",
    "master_catalogue['f_ukidss_k'][has_uds_k] = master_catalogue['f_uds_k'][has_uds_k]\n",
    "master_catalogue['ferr_ukidss_k'][has_uds_k] = master_catalogue['ferr_uds_k'][has_uds_k]\n",
    "master_catalogue['m_ukidss_k'][has_uds_k] = master_catalogue['m_uds_k'][has_uds_k]\n",
    "master_catalogue['merr_ukidss_k'][has_uds_k] = master_catalogue['merr_uds_k'][has_uds_k]\n",
    "master_catalogue['flag_ukidss_k'][has_uds_k] = master_catalogue['flag_uds_k'][has_uds_k]\n",
    "\n",
    "master_catalogue['f_ukidss_k'][use_candels_k] = master_catalogue['f_candels-ukidss_k'][use_candels_k]\n",
    "master_catalogue['ferr_ukidss_k'][use_candels_k] = master_catalogue['ferr_candels-ukidss_k'][use_candels_k]\n",
    "master_catalogue['m_ukidss_k'][use_candels_k] = master_catalogue['m_candels-ukidss_k'][use_candels_k]\n",
    "master_catalogue['merr_ukidss_k'][use_candels_k] = master_catalogue['merr_candels-ukidss_k'][use_candels_k]\n",
    "master_catalogue['flag_ukidss_k'][use_candels_k] = master_catalogue['flag_candels-ukidss_k'][use_candels_k]\n",
    "\n",
    "\n",
    "\n",
    "has_ap_uds_k =  ~np.isnan(master_catalogue['f_ap_uds_k'])\n",
    "has_ap_uds_j =  ~np.isnan(master_catalogue['f_ap_uds_j'])\n",
    "has_ap_ukidss_k =  ~np.isnan(master_catalogue['f_ap_ukidss_k'])\n",
    "has_ap_ukidss_j =  ~np.isnan(master_catalogue['f_ap_ukidss_j'])\n",
    "\n",
    "master_catalogue['f_ap_ukidss_k'][has_ap_uds_k] = master_catalogue['f_ap_uds_k'][has_ap_uds_k]\n",
    "master_catalogue['ferr_ap_ukidss_k'][has_ap_uds_k] = master_catalogue['ferr_ap_uds_k'][has_ap_uds_k]\n",
    "master_catalogue['m_ap_ukidss_k'][has_ap_uds_k] = master_catalogue['m_ap_uds_k'][has_ap_uds_k]\n",
    "master_catalogue['merr_ap_ukidss_k'][has_ap_uds_k] = master_catalogue['merr_ap_uds_k'][has_ap_uds_k]\n",
    "\n",
    "master_catalogue['f_ap_ukidss_j'][has_ap_uds_j] = master_catalogue['f_ap_uds_j'][has_ap_uds_j]\n",
    "master_catalogue['ferr_ap_ukidss_j'][has_ap_uds_j] = master_catalogue['ferr_ap_uds_j'][has_ap_uds_j]\n",
    "master_catalogue['m_ap_ukidss_j'][has_ap_uds_j] = master_catalogue['m_ap_uds_j'][has_ap_uds_j]\n",
    "master_catalogue['merr_ap_ukidss_j'][has_ap_uds_j] = master_catalogue['merr_ap_uds_j'][has_ap_uds_j]\n",
    "\n",
    "master_catalogue.remove_columns(['f_candels-ukidss_j','ferr_candels-ukidss_j',\n",
    "                                 'm_candels-ukidss_j','merr_candels-ukidss_j','flag_candels-ukidss_j',\n",
    "                                 'f_candels-ukidss_h','ferr_candels-ukidss_h',\n",
    "                                 'm_candels-ukidss_h','merr_candels-ukidss_h','flag_candels-ukidss_h',\n",
    "                                 'f_candels-ukidss_k','ferr_candels-ukidss_k',\n",
    "                                 'm_candels-ukidss_k','merr_candels-ukidss_k','flag_candels-ukidss_k',\n",
    "                                 \n",
    "                               'f_uds_j','ferr_uds_j','m_uds_j','merr_uds_j','flag_uds_j',\n",
    "                               'f_uds_k','ferr_uds_k','m_uds_k','merr_uds_k','flag_uds_k',\n",
    "                               'f_ap_uds_j','ferr_ap_uds_j','m_ap_uds_j','merr_ap_uds_j',\n",
    "                               'f_ap_uds_k','ferr_ap_uds_k','m_ap_uds_k','merr_ap_uds_k'])\n",
    "\n",
    "\n",
    "ukidss_origin = Table()\n",
    "ukidss_origin.add_column(master_catalogue['ukidss_intid'])\n",
    "\n",
    "origin = np.full(len(master_catalogue), '     ', dtype='<U5')\n",
    "origin[has_uds_j] = \"UDS\"\n",
    "origin[has_ukidss_j] = \"DXS\"\n",
    "origin[use_candels_j] = \"CANDELS\"\n",
    "ukidss_origin.add_column(Column(data=origin, name= 'f_ukidss_j' ))\n",
    "\n",
    "origin = np.full(len(master_catalogue), '     ', dtype='<U5')\n",
    "origin[has_uds_h] = \"UDS\"\n",
    "origin[use_candels_h] = \"CANDELS\"\n",
    "ukidss_origin.add_column(Column(data=origin, name= 'f_ukidss_h' ))\n",
    "\n",
    "origin = np.full(len(master_catalogue), '     ', dtype='<U5')\n",
    "origin[has_uds_k] = \"UDS\"\n",
    "origin[has_ukidss_k] = \"DXS\"\n",
    "origin[use_candels_k] = \"CANDELS\"\n",
    "ukidss_origin.add_column(Column(data=origin, name= 'f_ukidss_k' ))\n",
    "\n",
    "origin_ap = np.full(len(master_catalogue), '     ', dtype='<U5')\n",
    "origin_ap[has_ap_uds_k] = \"UDS\"\n",
    "origin_ap[has_ap_ukidss_k] = \"DXS\"\n",
    "ukidss_origin.add_column(Column(data=origin_ap, name= 'f_ap_ukidss_k' ))\n",
    "origin_ap = np.full(len(master_catalogue), '     ', dtype='<U5')\n",
    "origin_ap[has_ap_uds_j] = \"UDS\"\n",
    "origin_ap[has_ap_ukidss_j] = \"DXS\"\n",
    "ukidss_origin.add_column(Column(data=origin_ap, name= 'f_ap_ukidss_j' ))\n",
    "\n",
    "ukidss_origin.write(\"{}/xmm-lss_ukidss_fluxes_origins{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IX - Cross-identification table\n",
    "\n",
    "We are producing a table associating to each HELP identifier, the identifiers of the sources in the pristine catalogues. This can be used to easily get additional information from them.\n",
    "\n",
    "For convenience, we also cross-match the master list with the SDSS catalogue and add the objID associated with each source, if any. **TODO: should we correct the astrometry with respect to Gaia positions?**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XI - Saving the catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = [\"help_id\", \"field\", \"ra\", \"dec\", \"hp_idx\"]\n",
    "\n",
    "bands = [column[5:] for column in master_catalogue.colnames if 'f_ap' in column]\n",
    "for band in bands:\n",
    "    columns += [\"f_ap_{}\".format(band), \"ferr_ap_{}\".format(band),\n",
    "                \"m_ap_{}\".format(band), \"merr_ap_{}\".format(band),\n",
    "                \"f_{}\".format(band), \"ferr_{}\".format(band),\n",
    "                \"m_{}\".format(band), \"merr_{}\".format(band),\n",
    "                \"flag_{}\".format(band)]    \n",
    "    \n",
    "columns += [\"stellarity\", \"stellarity_origin\", \"flag_cleaned\", \"flag_merged\", \"flag_gaia\", \"flag_optnir_obs\", \"flag_optnir_det\", \n",
    "            \"zspec\", \"zspec_qual\", \"zspec_association_flag\", \"ebv\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing columns: {'dxs_id', 'ukidss_dec', 'uds_id', 'uds_flag_cleaned', 'ukidss_ra', 'candels_id', 'candels_stellarity', 'dxs_flag_gaia', 'dxs_flag_cleaned', 'dxs_stellarity', 'uds_flag_gaia', 'uds_stellarity', 'candels_flag_cleaned', 'ukidss_intid', 'ukidss_flag_merged', 'candels_flag_gaia'}\n"
     ]
    }
   ],
   "source": [
    "# We check for columns in the master catalogue that we will not save to disk.\n",
    "print(\"Missing columns: {}\".format(set(master_catalogue.colnames) - set(columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_catalogue.write(\"{}/ukidss_merged_catalogue_xmm-lss.fits\".format(TMP_DIR), overwrite = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
