GAMA-09 master catalogue

Preparation of KIDS/VST data

Kilo Degree Survey/VLT Survey Telescope catalogue: the catalogue comes from dmu0_KIDS.

In the catalogue, we keep:

  • The identifier (it's unique in the catalogue);
  • The position;
  • The stellarity;
  • The aperture corrected aperture magnitude in each band (10 pixels = 2")
  • The Petrosian magnitude to be used as total magnitude (no “auto” magnitude is provided).

We take 2014 as the observation year from a typical image header.

In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

from collections import OrderedDict
import os

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table
import numpy as np

from herschelhelp_internal.flagging import  gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates
from herschelhelp_internal.utils import astrometric_correction, mag_to_flux, flux_to_mag
In [3]:
OUT_DIR =  os.environ.get('TMP_DIR', "./data_tmp")
try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass

RA_COL  = "kids_ra"
DEC_COL = "kids_dec"

I - Column selection

In [4]:
imported_columns = OrderedDict({
        'ID': "kids_id",
        'RAJ2000': "kids_ra",
        'DECJ2000': "kids_dec",
        'CLASS_STAR':  "kids_stellarity",
        'MAG_AUTO_U': "m_kids_u", 
        'MAGERR_AUTO_U': "merr_kids_u", 
        'MAG_AUTO_G': "m_kids_g", 
        'MAGERR_AUTO_G': "merr_kids_g", 
        'MAG_AUTO_R': "m_kids_r", 
        'MAGERR_AUTO_R': "merr_kids_r", 
        'MAG_AUTO_I': "m_kids_i", 
        'MAGERR_AUTO_I': "merr_kids_i", 
        'FLUX_APERCOR_10_U': "f_ap_kids_u",
        'FLUXERR_APERCOR_10_U': "ferr_ap_kids_u",
        'FLUX_APERCOR_10_G': "f_ap_kids_g",
        'FLUXERR_APERCOR_10_G': "ferr_ap_kids_g",
        'FLUX_APERCOR_10_R': "f_ap_kids_r",
        'FLUXERR_APERCOR_10_R': "ferr_ap_kids_r",
        'FLUX_APERCOR_10_I': "f_ap_kids_i",
        'FLUXERR_APERCOR_10_I': "ferr_ap_kids_i"

    })


catalogue = Table.read("../../dmu0/dmu0_KIDS/data/KIDS-DR3_GAMA-09.fits")[list(imported_columns)]
for column in imported_columns:
    catalogue[column].name = imported_columns[column]

epoch = 2014 #A range of observation dates from 2011 to 2015.

# Clean table metadata
catalogue.meta = None
In [5]:
# Adding flux and band-flag columns
for col in catalogue.colnames:
    if col.startswith('m_'):
        
        errcol = "merr{}".format(col[1:])

        flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))
        
        # Fluxes are added in µJy
        catalogue.add_column(Column(flux * 1.e6, name="f{}".format(col[1:])))
        catalogue.add_column(Column(error * 1.e6, name="f{}".format(errcol[1:])))
        
        # Band-flag column
        if "ap" not in col:
            catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
    if col.startswith('f_'):
        
        errcol = "ferr{}".format(col[1:])
        
        #Convert fluxes in maggies to uJy
        catalogue[col] *= 3631. * 1.e6
        catalogue[col].unit = 'uJy'
        catalogue[errcol] *= 3631. * 1.e6
        catalogue[errcol].unit = 'uJy'

        mag, mag_error = flux_to_mag(np.array(catalogue[col]) * 1.e-6, 
                                     np.array(catalogue[errcol]) * 1.e-6)
        
        # Magnitudes are added
        catalogue.add_column(Column(mag, name="m{}".format(col[1:])))
        catalogue.add_column(Column(mag_error, name="m{}".format(errcol[1:])))
        
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: divide by zero encountered in log10
  magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: invalid value encountered in log10
  magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:80: RuntimeWarning: invalid value encountered in true_divide
  errors = 2.5 / np.log(10) * errors_on_fluxes / fluxes
In [6]:
catalogue[:10].show_in_notebook()
Out[6]:
<Table masked=True length=10>
idxkids_idkids_rakids_deckids_stellaritym_kids_umerr_kids_um_kids_gmerr_kids_gm_kids_rmerr_kids_rm_kids_imerr_kids_if_ap_kids_uferr_ap_kids_uf_ap_kids_gferr_ap_kids_gf_ap_kids_rferr_ap_kids_rf_ap_kids_iferr_ap_kids_if_kids_uferr_kids_uflag_kids_uf_kids_gferr_kids_gflag_kids_gf_kids_rferr_kids_rflag_kids_rf_kids_iferr_kids_iflag_kids_im_ap_kids_umerr_ap_kids_um_ap_kids_gmerr_ap_kids_gm_ap_kids_rmerr_ap_kids_rm_ap_kids_imerr_ap_kids_i
degdegmagmagmagmagmagmagmagmaguJyuJyuJyuJyuJyuJyuJyuJy
0KIDS J083701.35-005729.70129.255631095-0.9582497379710.92383720.50730.10640518.51650.00669415nannan17.57120.006740920.7381520.2025853.187990.0937371nannan25.46440.21759222.75552.2301False142.3650.877755FalsenannanFalse340.0232.11107False24.22960.29797922.64120.0319241nannan20.38520.00927753
1KIDS J083756.52-005753.00129.485504309-0.9647209380050.036291416.36440.00148657nannannannannannan223.8570.763833nannannannannannan1033.311.41479FalsenannanFalsenannanFalsenannanFalse18.02510.00370469nannannannannannan
2KIDS J083727.72-005850.71129.365505624-0.980753829940.9669216.19430.00174228nannannannannannan1100.141.3669nannannannannannan1208.631.93948FalsenannanFalsenannanFalsenannanFalse16.29640.001349nannannannannannan
3KIDS J083446.46-005843.09128.693563518-0.9786351057810.95870115.90470.00121472nannannannannannan1474.721.43213nannannannannannan1578.041.76551FalsenannanFalsenannanFalsenannanFalse15.97820.00105438nannannannannannan
4KIDS J083646.89-005841.28129.195390856-0.9781346124710.97365916.42660.0018003nannannannannannan913.7741.21871nannannannannannan975.7671.61796FalsenannanFalsenannanFalsenannanFalse16.49790.00144806nannannannannannan
5KIDS J083738.56-005812.65129.410663157-0.9701815770730.97920715.4690.000879895nannannannannannan2151.481.70168nannannannannannan2357.131.91025FalsenannanFalsenannanFalsenannanFalse15.56820.000858743nannannannannannan
6KIDS J083440.84-005838.44128.670151767-0.9773436859230.99797616.19820.00144626nannannannannannan1101.21.29568nannannannannannan1204.291.60417FalsenannanFalsenannanFalsenannanFalse16.29530.00127748nannannannannannan
7KIDS J083516.52-005757.12128.818833104-0.9658667654690.96521915.90550.00127789nannannannannannan1464.311.47278nannannannannannan1576.941.85603FalsenannanFalsenannanFalsenannanFalse15.98590.00109202nannannannannannan
8KIDS J083440.70-005833.32128.669598302-0.9759218047790.9133319.28620.011607817.76360.001477617.13030.00086215616.90060.0014259648.37940.330695225.3480.267692391.0860.260295483.480.46052770.06660.749091False284.8060.387598False510.3470.405254False630.5830.828183False19.68840.007421518.01790.0012897517.41930.00072263417.1890.00103419
9KIDS J083431.07-005816.64128.629439495-0.9712877449490.97081116.21790.0016896nannannannannannan1102.011.55786nannannannannannan1182.571.8403FalsenannanFalsenannanFalsenannanFalse16.29450.00153484nannannannannannan

II - Removal of duplicated sources

We remove duplicated objects from the input catalogues.

In [7]:
SORT_COLS = ['merr_ap_kids_u', 
             'merr_ap_kids_g', 
             'merr_ap_kids_r', 
             'merr_ap_kids_i']
FLAG_NAME = 'kids_flag_cleaned'

nb_orig_sources = len(catalogue)

catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)

nb_sources = len(catalogue)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.
Check the NumPy 1.11 release notes for more information.
  ma.MaskedArray.__setitem__(self, index, value)
The initial catalogue had 6002233 sources.
The cleaned catalogue has 6002146 sources (87 removed).
The cleaned catalogue has 87 sources flagged as having been cleaned

III - Astrometry correction

We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.

In [8]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_GAMA-09.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [9]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)
In [10]:
delta_ra, delta_dec =  astrometric_correction(
    SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
    gaia_coords
)

print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))
RA correction: -0.04423632481120876 arcsec
Dec correction: -0.06355399180684174 arcsec
In [11]:
catalogue[RA_COL] +=  delta_ra.to(u.deg)
catalogue[DEC_COL] += delta_dec.to(u.deg)
In [12]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)

IV - Flagging Gaia objects

In [13]:
catalogue.add_column(
    gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [14]:
GAIA_FLAG_NAME = "kids_flag_gaia"

catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))
276907 sources flagged.

V - Flagging objects near bright stars

VI - Saving to disk

In [15]:
catalogue.write("{}/KIDS.fits".format(OUT_DIR), overwrite=True)