HATLAS-SGP master catalogue

Checks and diagnostics

In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
plt.style.use('ggplot')

import locale
locale.setlocale(locale.LC_ALL, 'en_GB')

import os
import time
import itertools

from astropy.coordinates import SkyCoord
from astropy.table import Table
from astropy import units as u
from astropy import visualization as vis
import numpy as np
from matplotlib_venn import venn3, venn2

from herschelhelp_internal.masterlist import nb_compare_mags, nb_ccplots, nb_histograms, quick_checks
In [3]:
OUT_DIR = os.environ.get('OUT_DIR', "./data")
#SUFFIX = os.environ.get('SUFFIX', time.strftime("_%Y%m%d"))
SUFFIX = "_20171122"

master_catalogue = Table.read("{}/master_catalogue_hatlas-sgp{}.fits".format(OUT_DIR, SUFFIX))

0 - Quick checks

In [4]:
quick_checks(master_catalogue).show_in_notebook()
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(128, False) will return an array of dtype('bool')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(128, 0) will return an array of dtype('int64')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
Table shows only problematic columns.
Out[4]:
<Table length=40>
idxColumnAll nan#Measurements#Zeros#NegativeMinimum value
0ferr_ap_gpc1_g0.01004211.0732.00.00.0
1m_ap_gpc1_g0.01004211.00.08.0-0.39348500967
2merr_ap_gpc1_g0.01004211.0732.00.00.0
3ferr_gpc1_g0.01016026.03139.00.00.0
4m_gpc1_g0.01016026.00.023.0-2.22170996666
5merr_gpc1_g0.01016026.03139.00.00.0
6ferr_ap_gpc1_r0.01082713.0998.00.00.0
7m_ap_gpc1_r0.01082713.00.01.0-0.190017998219
8merr_ap_gpc1_r0.01082713.0998.00.00.0
9ferr_gpc1_r0.01086218.01125.00.00.0
10m_gpc1_r0.01086218.00.01.0-0.590920984745
11merr_gpc1_r0.01086218.01125.00.00.0
12ferr_ap_gpc1_i0.01133782.0465.00.00.0
13merr_ap_gpc1_i0.01133782.0465.00.00.0
14ferr_gpc1_i0.01135481.0272.00.00.0
15m_gpc1_i0.01135481.00.07.0-2.24709010124
16merr_gpc1_i0.01135481.0272.00.00.0
17ferr_ap_gpc1_z0.01061778.03449.00.00.0
18m_ap_gpc1_z0.01061778.00.02.0-0.506438970566
19merr_ap_gpc1_z0.01061778.03449.00.00.0
20ferr_gpc1_z0.01067459.01616.00.00.0
21m_gpc1_z0.01067459.00.01.0-1.07805001736
22merr_gpc1_z0.01067459.01616.00.00.0
23ferr_ap_gpc1_y0.01019793.06661.00.00.0
24merr_ap_gpc1_y0.01019793.06661.00.00.0
25ferr_gpc1_y0.01019192.02697.00.00.0
26m_gpc1_y0.01019192.00.01.0-1.45949995518
27merr_gpc1_y0.01019192.02697.00.00.0
28f_ap_omegacam_u0.07467153.011.0262518.0-56.0997009277
29ferr_ap_omegacam_u0.07467153.011.00.00.0
30merr_ap_omegacam_u0.07467142.00.0262518.0-637016.5
31f_ap_omegacam_g0.011440462.05.077648.0-549.62310791
32ferr_ap_omegacam_g0.011440462.05.00.00.0
33merr_ap_omegacam_g0.011440457.00.077648.0-50232.65625
34f_ap_omegacam_r0.012959145.03.028675.0-1094.06738281
35ferr_ap_omegacam_r0.012959145.03.00.00.0
36merr_ap_omegacam_r0.012959142.00.028675.0-632775.5
37f_ap_omegacam_i0.011300674.099323.0141896.0-39.0862464905
38ferr_ap_omegacam_i0.011300674.099323.00.00.0
39merr_ap_omegacam_i0.011201351.00.0141896.0-1007319.875

I - Summary of wavelength domains

In [5]:
flag_obs = master_catalogue['flag_optnir_obs']
flag_det = master_catalogue['flag_optnir_det']
In [6]:
venn2(
    [
        np.sum(flag_obs == 1),
        np.sum(flag_obs == 2),
        np.sum(flag_obs == 3)
    ],
    set_labels=('Optical', 'near-IR'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/len(flag_obs)))
)
plt.title("Wavelength domain observations");
In [7]:
venn2(
    [
        np.sum(flag_det[flag_obs == 3] == 1),
        np.sum(flag_det[flag_obs == 3] == 2),
        np.sum(flag_det[flag_obs == 3] == 3)
    ],
    set_labels=('near-IR', 'Optical'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/np.sum(flag_det != 0)))
)
plt.title("Detection of the {} sources detected\n in any wavelength domains "
          "(among {} sources)".format(
              locale.format('%d', np.sum(flag_det != 0), grouping=True),
              locale.format('%d', len(flag_det), grouping=True)));

II - Comparing magnitudes in similar filters

The master list if composed of several catalogues containing magnitudes in similar filters on different instruments. We are comparing the magnitudes in these corresponding filters.

In [8]:
u_bands = ["OmegaCAM u"]
g_bands = ["OmegaCAM g", "GPC1 g"]
r_bands = ["OmegaCAM r", "GPC1 r"]
i_bands = ["OmegaCAM i", "GPC1 i"]
z_bands = [              "GPC1 z", "VISTA z"]
y_bands = [              "GPC1 y", "VISTA y"]

II.a - Comparing depths

We compare the histograms of the total aperture magnitudes of similar bands.

In [9]:
for bands in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands]:
    colnames = ["m_{}".format(band.replace(" ", "_").lower()) for band in bands]
    nb_histograms(master_catalogue, colnames, bands)

II.b - Comparing magnitudes

We compare one to one each magnitude in similar bands.

In [10]:
for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands]:
    for band1, band2 in itertools.combinations(band_of_a_kind, 2):
        
        basecol1, basecol2 = band1.replace(" ", "_").lower(), band2.replace(" ", "_").lower()
        
        col1, col2 = "m_ap_{}".format(basecol1), "m_ap_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (aperture)".format(band1), "{} (aperture)".format(band2)))
        
        col1, col2 = "m_{}".format(basecol1), "m_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (total)".format(band1), "{} (total)".format(band2)))
GPC1 g (aperture) - OmegaCAM g (aperture):
- Median: -0.31
- Median Absolute Deviation: 0.30
- 1% percentile: -1.6985355429382323
- 99% percentile: 1.8626549105468695
GPC1 g (total) - OmegaCAM g (total):
- Median: -0.05
- Median Absolute Deviation: 0.21
- 1% percentile: -1.6923736712036148
- 99% percentile: 1.458908375190431
GPC1 r (aperture) - OmegaCAM r (aperture):
- Median: -0.19
- Median Absolute Deviation: 0.24
- 1% percentile: -1.1908159821044926
- 99% percentile: 1.3593329190185526
GPC1 r (total) - OmegaCAM r (total):
- Median: 0.09
- Median Absolute Deviation: 0.17
- 1% percentile: -1.208804918706055
- 99% percentile: 1.2806788397802757
GPC1 i (aperture) - OmegaCAM i (aperture):
- Median: -0.19
- Median Absolute Deviation: 0.20
- 1% percentile: -1.0411629385571295
- 99% percentile: 0.684708058386228
GPC1 i (total) - OmegaCAM i (total):
- Median: 0.06
- Median Absolute Deviation: 0.14
- 1% percentile: -0.8757926416992193
- 99% percentile: 0.8245664915612798