EGS master catalogue

Preparation of Canada France Hawaii Telescope Legacy Survey (CFHTLS) data

The catalogue is in dmu0_CFHTLS.

In the catalogue, we keep:

  • The position;
  • The stellarity (g band stellarity);
  • The aperture magnitude (3 arcsec).
  • The total magnitude (Kron like aperture magnitude).

We use the 2007 release, which we take as the date.

In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

from collections import OrderedDict
import os

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table
import numpy as np

from herschelhelp_internal.flagging import  gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates
from herschelhelp_internal.utils import astrometric_correction, mag_to_flux
In [3]:
OUT_DIR =  os.environ.get('TMP_DIR', "./data_tmp")
try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass

RA_COL = "cfhtls-deep_ra"
DEC_COL = "cfhtls-deep_dec"

I - Column selection

In [4]:
imported_columns = OrderedDict({
        'cfhtls': "cfhtls-deep_id",
        'raj2000': "cfhtls-deep_ra",
        'dej2000': "cfhtls-deep_dec",
        'gcl':  "cfhtls-deep_stellarity",
        'umaga': "m_cfhtls-deep_u",
        'e_umaga': "merr_cfhtls-deep_u",
        'gmaga': "m_cfhtls-deep_g",
        'e_gmaga': "merr_cfhtls-deep_g",
        'rmaga': "m_cfhtls-deep_r",
        'e_rmaga': "merr_cfhtls-deep_r",
        'imaga': "m_cfhtls-deep_i",
        'e_imaga': "merr_cfhtls-deep_i",
        'zmaga': "m_cfhtls-deep_z",
        'e_zmaga': "merr_cfhtls-deep_z",
        'umag': "m_ap_cfhtls-deep_u",
        'e_umag': "merr_ap_cfhtls-deep_u",
        'gmag': "m_ap_cfhtls-deep_g",
        'e_gmag': "merr_ap_cfhtls-deep_g",
        'rmag': "m_ap_cfhtls-deep_r",
        'e_rmag': "merr_ap_cfhtls-deep_r",
        'imag': "m_ap_cfhtls-deep_i",
        'e_imag': "merr_ap_cfhtls-deep_i",
        'zmag': "m_ap_cfhtls-deep_z",
        'e_zmag': "merr_ap_cfhtls-deep_z"
        
    })


catalogue = Table.read("../../dmu0/dmu0_CFHTLS/data/CFHTLS-DEEP_EGS.fits")[list(imported_columns)]
for column in imported_columns:
    catalogue[column].name = imported_columns[column]

epoch = 2007

# Clean table metadata
catalogue.meta = None
In [5]:
# Adding flux and band-flag columns
for col in catalogue.colnames:
    if col.startswith('m_'):
        
        errcol = "merr{}".format(col[1:])
        
        #catalogue[col][catalogue[col] <= 0] = np.nan
        #catalogue[errcol][catalogue[errcol] <= 0] = np.nan  
        

        flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))
        
        # Fluxes are added in µJy
        catalogue.add_column(Column(flux * 1.e6, name="f{}".format(col[1:])))
        catalogue.add_column(Column(error * 1.e6, name="f{}".format(errcol[1:])))
        
        # Band-flag column
        if "ap" not in col:
            catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
        
# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting.
In [6]:
catalogue[:10].show_in_notebook()
Out[6]:
<Table masked=True length=10>
idxcfhtls-deep_idcfhtls-deep_racfhtls-deep_deccfhtls-deep_stellaritym_cfhtls-deep_umerr_cfhtls-deep_um_cfhtls-deep_gmerr_cfhtls-deep_gm_cfhtls-deep_rmerr_cfhtls-deep_rm_cfhtls-deep_imerr_cfhtls-deep_im_cfhtls-deep_zmerr_cfhtls-deep_zm_ap_cfhtls-deep_umerr_ap_cfhtls-deep_um_ap_cfhtls-deep_gmerr_ap_cfhtls-deep_gm_ap_cfhtls-deep_rmerr_ap_cfhtls-deep_rm_ap_cfhtls-deep_imerr_ap_cfhtls-deep_im_ap_cfhtls-deep_zmerr_ap_cfhtls-deep_zf_cfhtls-deep_uferr_cfhtls-deep_uflag_cfhtls-deep_uf_cfhtls-deep_gferr_cfhtls-deep_gflag_cfhtls-deep_gf_cfhtls-deep_rferr_cfhtls-deep_rflag_cfhtls-deep_rf_cfhtls-deep_iferr_cfhtls-deep_iflag_cfhtls-deep_if_cfhtls-deep_zferr_cfhtls-deep_zflag_cfhtls-deep_zf_ap_cfhtls-deep_uferr_ap_cfhtls-deep_uf_ap_cfhtls-deep_gferr_ap_cfhtls-deep_gf_ap_cfhtls-deep_rferr_ap_cfhtls-deep_rf_ap_cfhtls-deep_iferr_ap_cfhtls-deep_if_ap_cfhtls-deep_zferr_ap_cfhtls-deep_z
degdegmagmagmagmagmagmagmagmagmagmagmagmagmagmagmagmagmagmagmagmag
00300_0574781215.69108253.1325570.0224.7950.134nannan26.672.90525.0770.61723.3760.38924.8340.137nannan26.5012.45224.9930.56223.4020.3920.438530.0541228FalsenannanFalse0.0779830.208652False0.3382210.192203False1.620320.580531False0.4230580.0533822nannan0.09111710.2057770.3654260.1891521.581980.571165
10300_0574432215.68950753.1320340.5826.0750.40626.0470.46325.3140.34624.8110.38723.1220.19426.1070.41725.9850.43625.2850.33424.8110.38623.1390.1970.1348960.050443False0.138420.0590278False0.2718940.0866467False0.4321150.154023False2.047390.365828False0.1309780.0503050.1465550.0588520.2792540.08590570.4321150.1536252.015580.365714
20300_0575128215.69101253.1309660.9820.4510.00419.1630.00518.320.00218.0770.00217.940.00420.4910.00319.1740.00418.3420.00218.1010.00117.9660.00323.96620.0882948False78.48740.361448False170.6080.314272False213.4030.393102False242.1030.891939False23.09940.063825977.69620.286243167.1860.307968208.7370.192254236.3740.653126
30300_0572283215.6891953.1286880.8627.1561.08726.3280.522nannan26.4171.48924.6720.73127.1721.09926.3250.517nannan26.2831.30924.7020.7470.04984250.0499005False0.1068560.0513743FalsenannanFalse0.09844640.135011False0.4911330.330668False0.04911330.04971330.1071520.051023nannan0.1113780.1342810.4777490.328698
40300_0571771215.6953.1278890.6427.2971.227nannannannannannannannan27.0761.018nannannannannannannannan0.04377230.0494675FalsenannanFalsenannanFalsenannanFalsenannanFalse0.05365370.0503064nannannannannannannannan
50300_0571613215.68727953.127260.9524.6550.06124.2150.03924.1330.0623.4460.05123.1890.09824.6930.05624.2350.03624.1290.05323.4690.04623.2260.0890.4988840.0280288False0.7481690.0268745False0.8068640.0445889False1.519150.0713585False1.924860.173741False0.4817250.02484640.7345130.02435440.8098420.03953231.48730.06301351.860370.152499
60300_0571954215.68634153.1281580.6226.4020.24726.0370.1626.2280.30826.0460.40927.2783.12226.4070.24826.0320.15926.1770.29326.0630.41527.072.5720.09981590.0227077False0.1397010.0205871False0.1171650.0332373False0.1385480.0521914False0.04454510.128088False0.09935740.02269490.1403460.02055290.12280.03313930.1363950.05213430.0539510.127805
70300_0572142215.68423353.1284970.4226.6080.30626.0260.15526.1950.2725.330.18225.6660.63326.6540.26226.0380.1326.1010.20425.4520.16725.6360.5110.08256580.02327False0.1411240.0201469False0.1207810.0300358False0.2679170.0449104False0.1966070.114625False0.07914080.01909750.1395720.01671160.1317040.0247460.2394420.03682920.2021160.0951255
80300_0571608215.68529153.1276680.7225.6950.12525.0060.05325.190.10124.5430.08724.0480.13725.7260.12925.0070.05425.190.10124.5620.08924.0460.1360.1914260.0220387False0.3610770.0176259False0.3047890.0283529False0.5530960.0443195False0.8725690.110102False0.1860370.02210370.3607450.01794190.3047890.02835290.54350.04455180.8741780.1095
90300_0571901215.68454353.1276050.726.9240.38625.1190.06223.9710.03322.1140.00921.3550.01126.8270.32325.1370.05823.9830.0322.1340.00821.370.0110.06171630.0219413False0.3253870.0185809False0.9366980.0284701False5.180840.0429455False10.42320.105601False0.06748380.0200760.3200370.01709640.9264030.02559745.086270.03747710.28020.104152

II - Removal of duplicated sources

We remove duplicated objects from the input catalogues.

In [7]:
SORT_COLS = ['merr_ap_cfhtls-deep_u',
            'merr_ap_cfhtls-deep_g',
            'merr_ap_cfhtls-deep_r',
            'merr_ap_cfhtls-deep_i',
            'merr_ap_cfhtls-deep_z',]
FLAG_NAME = 'cfhtls-deep_flag_cleaned'

nb_orig_sources = len(catalogue)

catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)

nb_sources = len(catalogue)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.
Check the NumPy 1.11 release notes for more information.
  ma.MaskedArray.__setitem__(self, index, value)
The initial catalogue had 566572 sources.
The cleaned catalogue has 566572 sources (0 removed).
The cleaned catalogue has 0 sources flagged as having been cleaned

III - Astrometry correction

We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.

In [8]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_EGS.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [9]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)
In [10]:
delta_ra, delta_dec =  astrometric_correction(
    SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
    gaia_coords
)

print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))
RA correction: -0.11242272060485448 arcsec
Dec correction: -0.033367632829595095 arcsec
In [11]:
catalogue[RA_COL] +=  delta_ra.to(u.deg)
catalogue[DEC_COL] += delta_dec.to(u.deg)
In [12]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)

IV - Flagging Gaia objects

In [13]:
catalogue.add_column(
    gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [14]:
GAIA_FLAG_NAME = "cfhtls-deep_flag_gaia"

catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))
2390 sources flagged.

V - Flagging objects near bright stars

VI - Saving to disk

In [15]:
catalogue.write("{}/CFHTLS-DEEP.fits".format(OUT_DIR), overwrite=True)