COSMOS master catalogue¶

Checks and diagnostics¶

In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
33f5ec7 (Wed Dec 6 16:56:17 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
plt.style.use('ggplot')

import locale
locale.setlocale(locale.LC_ALL, 'en_GB')

import os
import time
import itertools

from astropy.coordinates import SkyCoord
from astropy.table import Table
from astropy import units as u
from astropy import visualization as vis
import numpy as np
from matplotlib_venn import venn3, venn2

from herschelhelp_internal.masterlist import nb_compare_mags, nb_ccplots, nb_histograms, quick_checks, find_last_ml_suffix
In [3]:
OUT_DIR = os.environ.get('OUT_DIR', "./data")
SUFFIX = find_last_ml_suffix()
#SUFFIX = "_20170815"

master_catalogue = Table.read("{}/master_catalogue_cosmos_{}.fits".format(OUT_DIR, SUFFIX))

0 - Quick checks¶

In [4]:
quick_checks(master_catalogue).show_in_notebook()
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(268, False) will return an array of dtype('bool')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(268, 0) will return an array of dtype('int64')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
Table shows only problematic columns.
Out[4]:
<Table length=73>
idxColumnAll nan#Measurements#Zeros#NegativeMinimum value
0ferr_ap_gpc1_gFalse124592900.0
1merr_ap_gpc1_gFalse124592900.0
2ferr_gpc1_gFalse1244023100.0
3merr_gpc1_gFalse1244023100.0
4ferr_ap_gpc1_rFalse1311071200.0
5merr_ap_gpc1_rFalse1311071200.0
6ferr_gpc1_rFalse1308491400.0
7merr_gpc1_rFalse1308491400.0
8ferr_ap_gpc1_iFalse1310105300.0
9merr_ap_gpc1_iFalse1310105300.0
10ferr_gpc1_iFalse1308252300.0
11merr_gpc1_iFalse1308252300.0
12ferr_ap_gpc1_zFalse1297086000.0
13merr_ap_gpc1_zFalse1297086000.0
14ferr_gpc1_zFalse1291394700.0
15merr_gpc1_zFalse1291394700.0
16ferr_ap_gpc1_yFalse12701731400.0
17merr_ap_gpc1_yFalse12701731400.0
18ferr_gpc1_yFalse12455210300.0
19merr_gpc1_yFalse12455210300.0
20f_ap_candels_f140wFalse22049800.0
21ferr_ap_candels_f140wFalse22219100.0
22f_candels_f140wFalse222177170-6.97878249412e-07
23merr_candels_f140wFalse222170170-8571863.0
24f_ap_candels_f160wFalse33729010-8.35347861994e-08
25merr_ap_candels_f160wFalse33729010-51.3008613586
26f_candels_f160wFalse33729010-1.01014421716e-07
27merr_candels_f160wFalse33729010-39.8191490173
28merr_ap_decam_gFalse374647033289-14408.2753906
29f_decam_gFalse374831023706-402896.53125
30merr_decam_gFalse374831023706-35817.1914062
31merr_ap_decam_rFalse374062014425-9953.43359375
32f_decam_rFalse37431409329-13168483.0
33merr_decam_rFalse37431409329-2156.7097168
34f_ap_decam_zTrue0000.0
35ferr_ap_decam_zTrue0000.0
36m_ap_decam_zTrue0000.0
37merr_ap_decam_zFalse376562016539-17089.4492188
38f_decam_zFalse37657703655-6126.95214844
39merr_decam_zFalse37657703655-5242.02246094
40f_ap_kids_uFalse10727404470-0.577037513256
41merr_ap_kids_uFalse10727404470-17416.1914062
42f_ap_kids_gFalse13288201896-0.409766614437
43merr_ap_kids_gFalse13288201896-4520.70751953
44f_ap_kids_rFalse1515940439-0.278113663197
45merr_ap_kids_rFalse1515940439-6335.54638672
46f_ap_kids_iFalse12259502925-1.63008785248
47merr_ap_kids_iFalse12259502925-2953.06762695
48ferr_ap_wirds_ksFalse1709156500.0
49merr_ap_wirds_ksFalse1709156500.0
50ferr_wirds_ksFalse1709154500.0
51merr_wirds_ksFalse1709154500.0
52ferr_ap_wirds_jFalse3614915600.0
53merr_ap_wirds_jFalse3614915600.0
54ferr_wirds_jFalse3614912700.0
55merr_wirds_jFalse3614912700.0
56f_ap_suprime_gFalse230209826800.0
57f_suprime_gFalse210242755300.0
58f_ap_suprime_rFalse2371460126600.0
59f_suprime_rFalse2146220241300.0
60f_suprime_iFalse21477183100.0
61f_ap_suprime_zFalse234209312700.0
62f_suprime_zFalse211787731700.0
63f_ap_suprime_yFalse208419136100.0
64f_suprime_yFalse188595681000.0
65f_ap_suprime_n921False175358884200.0
66f_suprime_n921False1583174177700.0
67f_candels_f125wFalse3372108-2.04955426852e-07
68merr_candels_f125wFalse3372108-9.78206348419
69f_candels_f606wFalse2781816337-3.11445404577e-07
70merr_candels_f606wFalse278180337-9.90975298969e+20
71f_candels_f814wFalse2782511144-2.57528682823e-07
72merr_candels_f814wFalse278250144-4.70703566679e+21

I - Summary of wavelength domains¶

In [5]:
flag_obs = master_catalogue['flag_optnir_obs']
flag_det = master_catalogue['flag_optnir_det']
In [6]:
venn2(
    [
        np.sum(flag_obs == 1),
        np.sum(flag_obs == 2),
        np.sum(flag_obs == 3)
    ],
    set_labels=('Optical', 'near-IR'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/len(flag_obs)))
)
plt.title("Wavelength domain observations");
In [7]:
venn2(
    [
        np.sum(flag_det[flag_obs == 3] == 1),
        np.sum(flag_det[flag_obs == 3] == 2),
        np.sum(flag_det[flag_obs == 3] == 3)
    ],
    set_labels=('Optical', 'near-IR'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/np.sum(flag_det != 0)))
)
plt.title("Detection of the {} sources detected\n in any wavelength domains "
          "(among {} sources)".format(
              locale.format('%d', np.sum(flag_det != 0), grouping=True),
              locale.format('%d', len(flag_det), grouping=True)));

II - Comparing magnitudes in similar filters¶

The master list if composed of several catalogues containing magnitudes in similar filters on different instruments. We are comparing the magnitudes in these corresponding filters.

In [8]:
u_bands = ["Megacam u",            "KIDS u"]
g_bands = ["Megacam g", "DECam g", "KIDS g", "SUPRIME g", "GPC1 g"]
r_bands = ["Megacam r", "DECam r", "KIDS r", "SUPRIME r", "GPC1 r"]
i_bands = ["Megacam i",            "KIDS i", "SUPRIME i", "GPC1 i"]
z_bands = ["Megacam z", "DECam z",           "SUPRIME z", "GPC1 z"]
y_bands = [                                      "SUPRIME y", "GPC1 y"]

II.a - Comparing depths¶

We compare the histograms of the total aperture magnitudes of similar bands.

In [9]:
for bands in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands]:
    colnames = ["m_{}".format(band.replace(" ", "_").lower()) for band in bands]
    nb_histograms(master_catalogue, colnames, bands)

II.b - Comparing magnitudes¶

We compare one to one each magnitude in similar bands.

In [10]:
for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands]:
    for band1, band2 in itertools.combinations(band_of_a_kind, 2):
        
        basecol1, basecol2 = band1.replace(" ", "_").lower(), band2.replace(" ", "_").lower()
        
        col1, col2 = "m_ap_{}".format(basecol1), "m_ap_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (aperture)".format(band1), "{} (aperture)".format(band2)))
        
        col1, col2 = "m_{}".format(basecol1), "m_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (total)".format(band1), "{} (total)".format(band2)))
KIDS u (aperture) - Megacam u (aperture):
- Median: 0.15
- Median Absolute Deviation: 0.40
- 1% percentile: -2.289331703186035
- 99% percentile: 2.664651012420655
KIDS u (total) - Megacam u (total):
- Median: 0.30
- Median Absolute Deviation: 0.43
- 1% percentile: -2.038863296508789
- 99% percentile: 3.542579879760737
DECam g (aperture) - Megacam g (aperture):
- Median: -0.05
- Median Absolute Deviation: 0.24
- 1% percentile: -1.2481439590454102
- 99% percentile: 2.7848606109619167
DECam g (total) - Megacam g (total):
- Median: 0.26
- Median Absolute Deviation: 0.21
- 1% percentile: -0.5215093994140625
- 99% percentile: 2.622013511657716
KIDS g (aperture) - Megacam g (aperture):
- Median: 0.21
- Median Absolute Deviation: 0.20
- 1% percentile: -0.8616609191894531
- 99% percentile: 2.0629606628417836
KIDS g (total) - Megacam g (total):
- Median: 0.49
- Median Absolute Deviation: 0.30
- 1% percentile: -0.48593807220458984
- 99% percentile: 3.219382667541507
SUPRIME g (aperture) - Megacam g (aperture):
- Median: -0.00
- Median Absolute Deviation: 0.17
- 1% percentile: -1.3727685356140138
- 99% percentile: 2.1900751876831044