EGS master catalogue

Preparation of HST CANDELS-3D data

The catalogue comes from dmu0_CANDELS-3D-HST.

In the catalogue, we keep:

  • The identifier (it's unique in the catalogue);
  • The position;
  • The stellarity;
  • The kron magnitude, there doesn't appear to be aperture magnitudes. This may mean the survey is unusable.
In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

from collections import OrderedDict
import os

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table
import numpy as np

from herschelhelp_internal.flagging import  gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates
from herschelhelp_internal.utils import astrometric_correction, flux_to_mag
In [3]:
OUT_DIR =  os.environ.get('TMP_DIR', "./data_tmp")
try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass

RA_COL = "candels_ra"
DEC_COL = "candels_dec"

I - Column selection

In [4]:
imported_columns = OrderedDict({
        'ID': "candels_id",
        'RAJ2000': "candels_ra",
        'DEJ2000': "candels_dec",
        'S/G':  "candels_stellarity",
        'F140Wap': "f_ap_candels_f140w",
        'e_F140Wap': "ferr_ap_candels_f140w",
        'F140W': "f_candels_f140w",
        'e_F140W': "ferr_candels_f140w",
        'F160Wap': "f_ap_candels_f160w",
        'e_F160Wap': "ferr_ap_candels_f160w",
        'F160W': "f_candels_f160w",
        'e_F160W': "ferr_candels_f160w",
        'F606W': "f_candels_f606w",
        'e_F606W': "ferr_candels_f606w",
        'F814W': "f_candels_f814w",
        'e_F814W': "ferr_candels_f814w",
        'F125W': "f_candels_f125w",
        'e_F125W': "ferr_candels_f125w"

    })


catalogue = Table.read("../../dmu0/dmu0_CANDELS-3D-HST/data/CANDELS-3D-HST_EGS.fits")[list(imported_columns)]
for column in imported_columns:
    catalogue[column].name = imported_columns[column]

epoch = 2012 #Year of publication

# Clean table metadata
catalogue.meta = None
WARNING: UnitsWarning: '0.3631uJy' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]
WARNING: UnitsWarning: '[Msun]' did not parse as fits unit: Invalid character at col 0 [astropy.units.core]
In [5]:
# Adding flux and band-flag columns
for col in catalogue.colnames:
    if col.startswith('f_'):
        
        errcol = "ferr{}".format(col[1:])
        
        #Calculate mags, errors including the fact that fluxes are in units of 0.3631 uJy
        mag, error = flux_to_mag(np.array(catalogue[col]) * 0.3631e-6, np.array(catalogue[errcol] * 0.3631e-6))
        
        
        # magnitudes are added
        catalogue.add_column(Column(mag, name="m{}".format(col[1:])))
        catalogue.add_column(Column(error, name="m{}".format(errcol[1:])))
        
        
        #Correct flux units to uJy
        catalogue[col] = catalogue[col] * 0.3631
        catalogue[col].unit = u.microjansky
        catalogue[errcol] = catalogue[errcol] * 0.3631
        catalogue[errcol].unit = u.microjansky        

        if ('125' in col) or ('814' in col) or ('606' in col) :
            # We add nan filled aperture photometry for consistency
            catalogue.add_column(Column(np.full(len(catalogue), np.nan), name="m_ap{}".format(col[1:])))
            catalogue.add_column(Column(np.full(len(catalogue), np.nan), name="merr_ap{}".format(col[1:])))
            catalogue.add_column(Column(np.full(len(catalogue), np.nan), name="f_ap{}".format(col[1:])))
            catalogue.add_column(Column(np.full(len(catalogue), np.nan), name="ferr_ap{}".format(col[1:])))

        

        
        # Band-flag column
        if "ap" not in col:
            catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
        
# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting.
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: divide by zero encountered in log10
  magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:80: RuntimeWarning: invalid value encountered in true_divide
  errors = 2.5 / np.log(10) * errors_on_fluxes / fluxes
/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: invalid value encountered in log10
  magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6
In [6]:
catalogue[:10].show_in_notebook()
Out[6]:
<Table masked=True length=10>
idxcandels_idcandels_racandels_deccandels_stellarityf_ap_candels_f140wferr_ap_candels_f140wf_candels_f140wferr_candels_f140wf_ap_candels_f160wferr_ap_candels_f160wf_candels_f160wferr_candels_f160wf_candels_f606wferr_candels_f606wf_candels_f814wferr_candels_f814wf_candels_f125wferr_candels_f125wm_ap_candels_f140wmerr_ap_candels_f140wm_candels_f140wmerr_candels_f140wflag_candels_f140wm_ap_candels_f160wmerr_ap_candels_f160wm_candels_f160wmerr_candels_f160wflag_candels_f160wm_candels_f606wmerr_candels_f606wm_ap_candels_f606wmerr_ap_candels_f606wf_ap_candels_f606wferr_ap_candels_f606wflag_candels_f606wm_candels_f814wmerr_candels_f814wm_ap_candels_f814wmerr_ap_candels_f814wf_ap_candels_f814wferr_ap_candels_f814wflag_candels_f814wm_candels_f125wmerr_candels_f125wm_ap_candels_f125wmerr_ap_candels_f125wf_ap_candels_f125wferr_ap_candels_f125wflag_candels_f125w
degdeguJyuJyuJyuJyuJyuJyuJyuJyuJyuJyuJyuJyuJyuJy
01215.2223815953.004184721nannannannan8181.010.01426339871.960.119445650.1670.01379451018.460.02085399100.380.0221371nannannannanFalse14.1181.89294e-0613.9141.31368e-05False16.86742.3036e-05nannannannanFalse16.38012.22315e-05nannannannanFalse14.00242.64111e-06nannannannanFalse
12215.0965881352.9180526701.273680.02695494.25711.994651.448950.0135948107.2270.59061833.0160.64613653.03080.98367491.6611.0976923.63740.022976618.96420.0229762False23.49740.01018718.82420.00598035False20.10320.0212483nannannannanFalse19.58870.0201395nannannannanFalse18.99450.0130023nannannannanFalse
23215.1614685152.959461210nannannannan0.5014770.019645226.07130.5402939.576040.4638613.79090.68894624.011.17067nannannannanFalse24.64940.042533220.35960.0225004False21.4470.0525927nannannannanFalse21.0510.0542396nannannannanFalse20.4490.0529379nannannannanFalse
34215.3052978553.05292130nannannannan0.2997350.02727830.4102670.03487030.1613580.011910.1669610.01866660.407870.0460701nannannannanFalse25.20820.098810424.86730.0922813False25.88050.0801396nannannannanFalse25.84350.121388nannannannanFalse24.87370.122637nannannannanFalse
45215.041839652.871273042686.4410.0484012825.290.0581904nannannannan77.01350.0104097256.2650.0160425nannan16.80857.65557e-0516.60857.65542e-05FalsenannannannanFalse19.18360.000146756nannannannanFalse17.87836.79683e-05nannannannanFalsenannannannannannanFalse
56215.3038024953.052120210nannannannan0.6145470.03019390.9828750.05237720.6628030.01403670.9521570.0218351.012540.0746969nannannannanFalse24.42860.053344423.91880.0578586False24.34650.0229935nannannannanFalse23.95320.0248983nannannannanFalse23.88650.0800967nannannannanFalse
67215.2671203653.027584082nannannannan0.1569940.02710980.2177290.04202880.08910470.01300010.1545610.02014410.2425360.0484702nannannannanFalse25.91030.18748625.55520.209582False26.52520.158405nannannannanFalse25.92730.141505nannannannanFalse25.43810.216981nannannannanFalse
78215.303207453.052955632nannannannan0.1342340.01907220.208430.03331330.07661410.01352080.1222590.02138840.2078020.0320127nannannannanFalse26.08030.15426325.60260.173533False26.68920.191609nannannannanFalse26.18180.189942nannannannanFalse25.60590.167262nannannannanFalse
89215.3037872353.053443912nannannannan0.1754350.01923560.2033650.02476380.08249630.01015520.1098990.01579960.1581920.0235928nannannannanFalse25.78970.11904525.62930.13221False26.60890.133653nannannannanFalse26.29750.15609nannannannanFalse25.9020.161927nannannannanFalse
910215.2786865253.035926822nannannannan0.1258790.01761250.2524130.04236290.09615980.01904170.2063530.02907120.2357940.0426751nannannannanFalse26.15010.15191225.39470.182221False26.44250.214999nannannannanFalse25.61350.152959nannannannanFalse25.46870.196502nannannannanFalse

II - Removal of duplicated sources

We remove duplicated objects from the input catalogues.

In [7]:
SORT_COLS = ['merr_candels_f140w', 
             'merr_candels_f160w', 
             'merr_candels_f606w', 
             'merr_candels_f814w',
             'merr_candels_f125w']
FLAG_NAME = 'candels_flag_cleaned'

nb_orig_sources = len(catalogue)

catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)

nb_sources = len(catalogue)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.
Check the NumPy 1.11 release notes for more information.
  ma.MaskedArray.__setitem__(self, index, value)
The initial catalogue had 41200 sources.
The cleaned catalogue has 41067 sources (133 removed).
The cleaned catalogue has 133 sources flagged as having been cleaned

III - Astrometry correction

We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.

In [8]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_EGS.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [9]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)
In [10]:
delta_ra, delta_dec =  astrometric_correction(
    SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
    gaia_coords
)

print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))
RA correction: -0.06336958314250296 arcsec
Dec correction: -0.19814636340669267 arcsec
In [11]:
catalogue[RA_COL] +=  delta_ra.to(u.deg)
catalogue[DEC_COL] += delta_dec.to(u.deg)
In [12]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)

IV - Flagging Gaia objects

In [13]:
catalogue.add_column(
    gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [14]:
GAIA_FLAG_NAME = "candels_flag_gaia"

catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))
145 sources flagged.

V - Flagging objects near bright stars

VI - Saving to disk

In [15]:
catalogue.write("{}/CANDELS.fits".format(OUT_DIR), overwrite=True)