# Functions to check which vulnerabilities exists in cve-search database for 
# the products in database
#
# Imported in fixdb in the collection named vulns.
#
# The format of the collection is the following
#
# Made by Gaetan Egger
#
# Inspired from cve-search
#
# Software is free software released under the "Modified BSD license"
#
# Copyright (c) 2012         Wim Remes
# Copyright (c) 2012-2014     Alexandre Dulaunoy - a@foo.be
# Copyright (c) 2014-2016     Pieter-Jan Moreels - pieterjan.moreels@gmail.com

# Imports
import re, datetime, subprocess

# To import debsecan conf
from ivulncheck_api.lib.config_api import Configuration as conf

import logging
logging.basicConfig(filename=conf.getLogfile(), level=logging.INFO)


from ivulncheck_api.lib import Vulns
from ivulncheck_api.lib import functions as f

from ivulncheck_api.lib import ubuntu_cve_tracker_layer as uctl
from ivulncheck_api.lib import cvesearch_api_layer as cveapi
from ivulncheck_api.lib import debsecan_layer as dsal

import apt_pkg
apt_pkg.init()

# 0 => no update found
# 1 => update found

def check_versions(orig_vers, other_vers, type):
    
    vc = apt_pkg.version_compare(orig_vers, other_vers)
    
    if type == "fix":
        if vc < 0:
            return True
        else:
            return False
    elif type == "vuln":
        if vc <= 0:
            return True
        else:
            return False
    else:
        logging.error('[%s] (%s)\tType for version compare missing.', datetime.datetime.today().isoformat(), __name__)
        return None
        
    
    
# Check if package name has a CPE ID
def pkg2cpe(db):
    nb_mod_pkg2cpe = 0
    nb_err_pkg2cpe = 0
    no_matched_packages = db.getPackages(query={'$or': [{'cpe_id': None}, {'cpe_srcid': None}]})
    nb_cpe = 0
    for no_matched_package in no_matched_packages:
        pkg_name = no_matched_package['id']
        pkg_vers = no_matched_package['version']
        src_name = no_matched_package['src_id']
        src_vers = no_matched_package['src_version']
        #~ logging.info('[%s] (%s)\t%s : %s\t%s : %s', datetime.datetime.today().isoformat(), __name__, pkg_name, pkg_vers, src_name, src_vers)
        # First search in index
        cpe_id = db.getPkg2CPE(pkg_name)
        if cpe_id is None:
            # Then search CVESearch
            (editor, cpe_name, cpe_vers) = f.dpkg2cpe(pkg_name, pkg_vers)
            if editor is not None:
                (cvesearch_api_status, cpe_id) = cveapi.findCPEid(editor + ":" + cpe_name + ":" + cpe_vers)
            else:
                (cvesearch_api_status, cpe_id) = cveapi.findCPEid(cpe_name + ":" + cpe_vers)
            if cvesearch_api_status == True and cpe_id is None:
                # Insert index with "empty" for faster search
                result_pkg2cpe = db.insertPkg2CPE({'id': pkg_name, 'cpeid': "empty"})
                if result_pkg2cpe == True:
                    nb_mod_pkg2cpe += 1
                else:
                    nb_err_pkg2cpe += 1
            elif cvesearch_api_status == False:
                logging.error('[%s] (%s)\tUnable to search CPE via CVE-Search API.', datetime.datetime.today().isoformat(), __name__)
                return -1,-1
            else:
                # Insert index with cpeid
                result_pkg2cpe = db.insertPkg2CPE({'id': pkg_name, 'cpeid': cpe_id})
                if result_pkg2cpe == True:
                    nb_mod_pkg2cpe += 1
                else:
                    nb_err_pkg2cpe += 1
        elif cpe_id == "empty":
            cpe_id = None
            
            
        cpe_src_id = None
        if pkg_name != src_name:
            # First search in index
            cpe_src_id = db.getPkg2CPE(src_name)
            if cpe_src_id is None:
                # Then search CVESearch
                (editor, cpe_name, cpe_vers) = f.dpkg2cpe(src_name, src_vers)
                if editor is not None:
                    (cvesearch_api_status, cpe_src_id) = cveapi.findCPEid(editor + ":" + cpe_name + ":" + cpe_vers)
                else:
                    (cvesearch_api_status, cpe_src_id) = cveapi.findCPEid(cpe_name + ":" + cpe_vers)
                if cvesearch_api_status == True and cpe_src_id is None:
                    # Insert index with "empty" for faster search
                    result_pkg2cpe = db.insertPkg2CPE({'id': src_name, 'cpeid': "empty"})
                    if result_pkg2cpe == True:
                        nb_mod_pkg2cpe += 1
                    else:
                        nb_err_pkg2cpe += 1
                elif cvesearch_api_status == False:
                    logging.error('[%s] (%s)\tUnable to search CPE via CVE-Search API.', datetime.datetime.today().isoformat(), __name__)
                    return -1, -1
                else:
                    # Insert index with cpeid
                    result_pkg2cpe = db.insertPkg2CPE({'id': src_name, 'cpeid': cpe_src_id})
                    if result_pkg2cpe == True:
                        nb_mod_pkg2cpe += 1
                    else:
                        nb_err_pkg2cpe += 1
            elif cpe_src_id == "empty":
                cpe_src_id = None 
                
        else:
            cpe_src_id = cpe_id
        db.updateCPEPackage(pkg_name, pkg_vers, cpe_id, cpe_src_id)
            
    logging.info('[%s] (%s)\t%d PKG2CPE indexes inserted in VulnDB and %d errors.', datetime.datetime.today().isoformat(), __name__, nb_mod_pkg2cpe, nb_err_pkg2cpe)

    return nb_mod_pkg2cpe, nb_err_pkg2cpe

# Check vuln based on packages (CPE) names
def cve_vulns(db):
    #IN PROGRESS
    logging.info('[%s] (%s)\tStarted check cve_vulns.', datetime.datetime.today().isoformat(), __name__)
    
    # Get existing packages
    pkgs = db.getPackages()
    vulns = []
    
    nb_err = 0
    nb_mod = 0
    nb_up = 0
    for pkg in pkgs:
        if pkg['cpe_id'] is not None:
            # To remove debian specific version patterns
            cpe_version = re.search(r'(?!\d\:)[\d\.]+', pkg['version']).group(0)
            # Search vuln for cpeid + version
            # to add version when cpe ends with : or not
            if re.match(r'.+\:$', pkg['cpe_id']):
                cpe_id = pkg['cpe_id'] + cpe_version
            else:
                cpe_id = pkg['cpe_id'] + ':' + cpe_version

            (cvesearch_api_status, pkg_cves) = cveapi.getCVEFor(cpe_id)
            if cvesearch_api_status == False:
                logging.error('[%s] (%s)\tUnable to search CVEs via CVE-Search API.', datetime.datetime.today().isoformat(), __name__)
                return -1, -1
            # if CVE exists for product
            if pkg_cves is not None and len(pkg_cves) > 0:
                for pkg_cve in pkg_cves:
                    # Convert CVEs to Ivulncheck vulns
                    versions = []
                    versions.append(pkg['version'])
                    src_versions = []
                    src_versions.append(pkg['src_version'])
                    vuln_versions = []
                    vuln_versions.append(cpe_version)
                    vulns.append(Vulns.Vuln(pkg_cve['id'], "cve", pkg['src_id'], versions, "", [], vuln_versions, [], [], None, f.CVSS2deb(pkg_cve['cvss']), "").to_json())
        
        if pkg['cpe_src_id'] is not None and pkg['cpe_src_id'] != pkg['cpe_id']:
            # To remove debian specific version patterns
            cpe_src_version = re.search(r'(?!\d\:)[\d\.]+', pkg['src_version']).group(0)
            # Search vuln for cpeid + version
            # to add versiopn when cpe ends with : or not
            if re.match(r'.+\:$', pkg['cpe_src_id']):
                cpe_src_id = pkg['cpe_src_id'] + cpe_src_version
            else:
                cpe_src_id = pkg['cpe_src_id'] + ':' + cpe_src_version

            (cvesearch_api_status, pkg_cves) = cveapi.getCVEFor(cpe_src_id)
            if cvesearch_api_status == False:
                logging.error('[%s] (%s)\tUnable to search CVEs via CVE-Search API.', datetime.datetime.today().isoformat(), __name__)
                return -1, -1
            # if CVE exists for product
            if pkg_cves is not None and len(pkg_cves) > 0:
                for pkg_cve in pkg_cves:
                    # Convert CVEs to Ivulncheck vulns
                    versions = []
                    versions.append(pkg['version'])
                    src_versions = []
                    src_versions.append(pkg['src_version'])
                    vuln_versions = []
                    vuln_versions.append(cpe_version)
                    vulns.append(Vulns.Vuln(pkg_cve['id'], "cve", pkg['src_id'], versions, "", [], vuln_versions, [], [], None, f.CVSS2deb(pkg_cve['cvss']), "").to_json())
        logging.info('[%s] (%s)\t%ss vulns checked.', datetime.datetime.today().isoformat(), __name__, pkg['id'])
                    
    for vuln in vulns:
        # insert vuln
        result = db.insertVuln(vuln)
        if result == True:
            nb_mod = nb_mod + 1
        else:
            nb_err = nb_err + 1
    logging.info('[%s] (%s)\t%d vulns added with %d errors.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    return nb_mod, nb_err

# Check vuln based on products only (CPE) names
def products_vulns(db):
    
    logging.info('[%s] (%s)\tStarted check product_vulns.', datetime.datetime.today().isoformat(), __name__)
    
    # Get existing packages
    prods = db.getProducts()
    #~ print(len(pkgs))
    vulns = []
    
    nb_err = 0
    nb_mod = 0
    for prod in prods:
    
        # Search vuln for cpeid + version
        # to add version when cpe ends with : or not
        if re.match(r'.+\:$', prod['cpe_id']):
            cpe_id = prod['cpe_id'] + prod['version']
        else:
            cpe_id = prod['cpe_id'] + ':' + prod['version']
        # ~ logging.info('[%s] (%s)\tTrying getCVEFor %s.', datetime.datetime.today().isoformat(), __name__, cpe_id)
        (cvesearch_api_status, prod_cves) = cveapi.getCVEFor(cpe_id)
        if cvesearch_api_status == False:
            logging.error('[%s] (%s)\tUnable to search CVEs via CVE-Search API.', datetime.datetime.today().isoformat(), __name__)
            return -1, -1
        # ~ logging.info('[%s] (%s)\t%d.', datetime.datetime.today().isoformat(), __name__, len(prod_cves))
        # if CVE exists for product
        if len(prod_cves) > 0:
            for prod_cve in prod_cves:
                # Convert CVEs to ivulncheck vulns
                versions = []
                versions.append(prod['version'])
                vuln_versions = []
                # ~ logging.info('[%s] (%s)\tCVEID : %s \tVulnerable configuration : %s.', datetime.datetime.today().isoformat(), __name__, prod_cve['id'], prod_cve['vulnerable_configuration'][0]['id'])
                for vuln_conf in prod_cve['vulnerable_configuration']:
                    # ~ logging.info('[%s] (%s)\tVulnerable configuration : %s.', datetime.datetime.today().isoformat(), __name__, vuln_conf['id'])
                    if re.match(r'^' + cpe_id, vuln_conf['id']):
                        vuln_versions.append(vuln_conf['id'].split(':',6)[5])
                vulns.append(Vulns.Vuln(prod_cve['id'], "cve", prod['id'], versions, "", [], vuln_versions, [], [], None, f.CVSS2deb(prod_cve['cvss']), "").to_json())
                  
    for vuln in vulns:
        # insert vuln
        result = db.insertVuln(vuln)
        if result == True:
            nb_mod = nb_mod + 1
        else:
            nb_err = nb_err + 1
    logging.info('[%s] (%s)\t%d vulns added with %d errors.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    return nb_mod, nb_err

# Check vulns based on debsecan tool return on hosts
def debsecan_vulns(db):

    vulns = []

    (url, backup_file_path) = conf.getDebsecanConf()
    
    suites = db.getSuites()
    
    logging.info('[%s] (%s)\tStarted check debsecan_vulns.', datetime.datetime.today().isoformat(), __name__)
    
    for suite in suites:
        suite_pkgs = db.getPackages(query={"os_type": "Debian", "os_version": suite})
        
        # To inject directly a package list
        tmpvulns = dsal.rate_system(suite_pkgs, dsal.fetch_data(suite), display=False)        
        # To parse a "status" like file
        #~ tmpvulns = dsal.rate_system(options, dsal.fetch_data(url, options['suite']), display=True)        
        
        
        for tmpvuln in tmpvulns:
            vulns.append(tmpvuln)

    nb_mod = 0
    nb_err = 0
    for vuln in vulns:
        result = db.insertVuln(vuln)
        if result == True:
            nb_mod = nb_mod + 1
        else:
            nb_err = nb_err + 1
    logging.info('[%s] (%s)\t%d vulns added with %d errors.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    return nb_mod, nb_err

def ubuntu_vulns(db):
    
    logging.info('[%s] (%s)\tStarted check ubuntu_vulns.', datetime.datetime.today().isoformat(), __name__)
    
    vulns = []
    nb_mod = 0
    nb_err = 0

    
    releases = db.getReleases()
    for release in releases:
        tables = uctl.ubuntuTable(release)
        pkgs = db.getPackages(query={'os_type': "Ubuntu", 'os_version': release})

        for table in tables:
            # ~ print(table['cve_id'] + " " + table['urgency'])
            for pkg in pkgs:
                if table['src_package'] == pkg['src_id']:
                    # ~ print("Vuln found")
                    # ~ print(pkg['id'])
                    versions = []
                    versions.append(pkg['version'])
                    src_versions = []
                    src_versions.append(pkg['src_version'])
                    fixed = []
                    if table['pkg_status'] == "released":
                        fixed.append(release[0:2])
                    # Discard vuln if already fixed
                    if len(table['fixed_versions']) > 0:
                        if check_versions(pkg['src_version'], table['fixed_versions'][0], "fix") == False:
                            continue
                        
                    vulns.append(Vulns.Vuln(table['cve_id'], "ubuntu", pkg['id'], versions, pkg['src_id'], src_versions, [], fixed, table['fixed_versions'], None, table['urgency'], "").to_json())
    
        for vuln in vulns:
            # ~ print(vuln['id'] + " " + vuln['urgency'])
            result = db.insertVuln(vuln)
            if result == True:
                nb_mod = nb_mod + 1
            else:
                nb_err = nb_err + 1
                
    logging.info('[%s] (%s)\t%d vulns from ubuntu inserted with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    return nb_mod, nb_err
    
   
## A refaire en comparant avec les numéros de version
def debsecan_fixes(db):

    # ~ pkgs = db.getPackages()
    nb_mod = 0
    nb_err = 0
    
    # ~ lock = RLock()
    fixes = []
    
    
    logging.info('[%s] (%s)\tStarted check debsecan_fixes.', datetime.datetime.today().isoformat(), __name__)
    
    pkgs = db.getPackages(query={'os_type': "Debian"})
    pkg_list = []
    
    pkg_vulns_index = {}
   
    for pkg in pkgs:
        query = []
        query.append({'bin_package': pkg['id']})
        query.append({'bin_versions': pkg['version']})
        pkg_vulns = db.getVulns(query=query)
        if len(pkg_vulns) > 0:
            pkg_list.append(pkg)
            pkg_vulns_index[pkg['id'] + ":" + pkg['version']] = pkg_vulns
            
    for pkg in pkgs:
        try:
            pkg_vulns = pkg_vulns_index[pkg['id'] + ":" + pkg['version']]
        except KeyError as ke:
            continue
            
        suite = pkg['os_version']
        
        for pkg_vuln in pkg_vulns:
            # If vuln is tagged for the current suite
            if f.suite2tag(suite) in pkg_vuln['fixed']:
                #~ print("it should never get here")
            
                cve_ids = []
                cve_ids.append(pkg_vuln['id'])
        
                #~ # To only keep newer fixed version
                for fixed_version in pkg_vuln['fixed_versions']:
                    # Check if debXuY correspong to current suite
                    if bool(re.match(r'[\w\.\+\-\~]+deb' + f.suite2tag(suite) + r'u[\d]{1,2}', fixed_version)) == True:
                        if check_versions(pkg['version'], fixed_version, "fix") == True:
                            fixes.append(Vulns.Fix(pkg['hosts'], pkg['id'], pkg['version'], pkg['src_id'], pkg['src_version'], fixed_version, pkg_vuln['remotely_exp'], pkg_vuln['urgency'], cve_ids, pkg['os_type'], pkg['os_version'], pkg_vuln['src'], False, "", "").to_json())
                            # To make the check faster and indicate the first version fixed
                            break
            # Check backport (if bpoX correspond to current suite)
            elif bool(re.match(r'[\w\.\+\-\~]+\~bpo' + f.suite2tag(suite) + r'\d?\+[\w\+\-\~]+', pkg['src_version'])) == True:
                
                cve_ids = []
                cve_ids.append(pkg_vuln['id'])
            
                #~ # To only keep newer fixed version
                for fixed_version in pkg_vuln['fixed_versions']:
                    #~ logging.info('[%s] (%s)\tBPO %s : %s\t%s.', datetime.datetime.today().isoformat(), __name__,pkg['id'], pkg['version'], fixed_version)
                    if check_versions(pkg['version'], fixed_version, "fix") == True:
                        fixes.append(Vulns.Fix(pkg['hosts'], pkg['id'], pkg['version'], pkg['src_id'], pkg['src_version'], fixed_version, pkg_vuln['remotely_exp'], pkg_vuln['urgency'], cve_ids, pkg['os_type'], pkg['os_version'], pkg_vuln['src'], False, "", "").to_json())
                        # To make the check faster and indicate the first version fixed
                        break
                                
    # ~ logging.info('[%s] (%s)\t%d fixed_vuln read.', datetime.datetime.today().isoformat(), __name__, len(fixes))
    
    for fix in fixes:
        result = db.insertFix(fix)
        if result == True:
            nb_mod += 1
        else:
            nb_err += 1
            

    logging.info('[%s] (%s)\t%d fixes from debsecan inserted with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)

    return nb_mod, nb_err
    
def ubuntu_fixes(db):

    # ~ pkgs = db.getPackages()
    nb_mod = 0
    nb_err = 0
    
    # ~ lock = RLock()
    fixes = []
    
    
    logging.info('[%s] (%s)\tStarted check ubuntu_fixes.', datetime.datetime.today().isoformat(), __name__)
    
    pkgs = db.getPackages(query={'os_type': "Ubuntu"})
    pkg_list = []
    
    pkg_vulns_index = {}
   
    # Creating index of vulns
    for pkg in pkgs:
        query = []
        query.append({'bin_package': pkg['id']})
        query.append({'bin_versions': pkg['version']})
        pkg_vulns = db.getVulns(query=query)
        if len(pkg_vulns) > 0:
            pkg_list.append(pkg)
            pkg_vulns_index[pkg['id'] + ":" + pkg['version']] = pkg_vulns
            
    for pkg in pkgs:
        try:
            pkg_vulns = pkg_vulns_index[pkg['id'] + ":" + pkg['version']]
        except KeyError as ke:
            continue
            
        release = pkg['os_version']
        
        for pkg_vuln in pkg_vulns:
            # If vuln is tagged for the current suite
            if release[0:2] in pkg_vuln['fixed']:
            
                cve_ids = []
                cve_ids.append(pkg_vuln['id'])
        
                # To only keep newer fixed version
                correct_fixed_version = pkg_vuln['fixed_versions'][0]
                for fixed_version in pkg_vuln['fixed_versions']:
                    if check_versions(pkg['version'], fixed_version, "fix") == True:
                        # Get the lower version of fix available (xenial vs artful)
                        if check_versions(correct_fixed_version, fixed_version, "fix") == True:
                            correct_fixed_version = fixed_version
                        fixes.append(Vulns.Fix(pkg['hosts'], pkg['id'], pkg['version'], pkg['src_id'], pkg['src_version'], correct_fixed_version, pkg_vuln['remotely_exp'], pkg_vuln['urgency'], cve_ids, pkg['os_type'], pkg['os_version'], pkg_vuln['src'], False, "", "").to_json())
                                
    # ~ logging.info('[%s] (%s)\t%d fixed_vuln read.', datetime.datetime.today().isoformat(), __name__, len(fixes))
    
    for fix in fixes:
        result = db.insertFix(fix)
        if result == True:
            nb_mod += 1
        else:
            nb_err += 1
            

    logging.info('[%s] (%s)\t%d fixes from ubuntu inserted with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)

    return nb_mod, nb_err
    
    
def madison_fixes(db):
    
    logging.info('[%s] (%s)\tStarted check madison_fixes.', datetime.datetime.today().isoformat(), __name__)
    
    FAKE_REPOS_PATH=conf.getFakeReposPath()
    
    nb_deb = 0
    nb_ub = 0
    nb_mod = 0
    nb_err = 0
    fixes = []
    
    # ~ print(ubuntu_host['id'])
    suites = db.getSuites()
    releases = db.getReleases()
    
    for suite in suites:
        # ~ logging.info('[%s] (%s)\tRelease found: %s', datetime.datetime.today().isoformat(), __name__, release)
        pkgs = db.getPackages(query={'os_type': "Debian", 'os_version': suite})
        for pkg in pkgs:
            available_version = None
            try:
                p = None
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'])
                if re.match(r'^.+\:amd64$', pkg['id']):
                    p = subprocess.Popen(["madison-lite", "--mirror", FAKE_REPOS_PATH + "/debian-security", "-s", suite, pkg['id'].split(':',2)[0]], stdout=subprocess.PIPE)
                # ~ print(pkg['id'])
                else:
                    p = subprocess.Popen(["madison-lite", "--mirror", FAKE_REPOS_PATH + "/debian-security", "-s", suite, pkg['id']], stdout=subprocess.PIPE)
                (out, err) = p.communicate()
                
                available_version = out.decode("utf-8").split(" | ")[1]
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
            except:
                # ~ print("ERROR ==> " + pkg['id'])
                pass
            # ~ print(pkg['id'] + "\t\t" + pkg['version'] + "\t\t" + available_version)
            if available_version is not None:
                if check_versions(pkg['version'], available_version, "fix"):
                    # ~ logging.info('[%s] (%s)\tFIXED\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
                    # ~ pkg_vulns = db.getVulns(query={'src_package': pkg['src_id'], 'src_versions': pkg['src_version']})
                    # ~ urgency = ""
                    # ~ for pkg_vuln in pkg_vulns:
                        # ~ # Get max urgency
                        # ~ if pkg_vuln['urgency'] == "high":
                            # ~ urgency = "high"
                        # ~ elif pkg_vuln['urgency'] == "medium" and urgency != "high":
                            # ~ urgency = "medium"
                        # ~ elif pkg_vuln['urgency'] == "low" and urgency == "":
                            # ~ urgency = "low"
                    fixes.append(Vulns.Fix(pkg['hosts'], pkg['id'], pkg['version'], pkg['src_id'], pkg['src_version'], available_version, None, "", [], pkg['os_type'], pkg['os_version'], "madison", False, "", "").to_json())
                    nb_deb += 1
    
    logging.info('[%s] (%s)\t%d fixes for debian found by madison.', datetime.datetime.today().isoformat(), __name__, nb_deb)
    
    for release in releases:
        # ~ logging.info('[%s] (%s)\tRelease found: %s', datetime.datetime.today().isoformat(), __name__, release)
        pkgs = db.getPackages(query={'os_type': "Ubuntu", 'os_version': release})
        for pkg in pkgs:
            available_version = None
            try:
                p = None
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'])
                if re.match(r'^.+\:amd64$', pkg['id']):
                    p = subprocess.Popen(["madison-lite", "--mirror", FAKE_REPOS_PATH + "/ubuntu", "-s", release + "-security", pkg['id'].split(':',2)[0]], stdout=subprocess.PIPE)
                # ~ print(pkg['id'])
                else:
                    p = subprocess.Popen(["madison-lite", "--mirror", FAKE_REPOS_PATH + "/ubuntu", "-s", release + "-security", pkg['id']], stdout=subprocess.PIPE)
                (out, err) = p.communicate()
                
                available_version = out.decode("utf-8").split(" | ")[1]
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
                # ~ logging.info('[%s] (%s)\tPackage : %s\tInstalled: %s\tAvailable', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
            except:
                # ~ print("ERROR ==> " + pkg['id'])
                pass
            # ~ print(pkg['id'] + "\t\t" + pkg['version'] + "\t\t" + available_version)
            if available_version is not None:
                if check_versions(pkg['version'], available_version, "fix"):
                    # ~ logging.info('[%s] (%s)\tFIXED\tPackage : %s\tInstalled: %s\tAvailable: %s', datetime.datetime.today().isoformat(), __name__, pkg['id'], pkg['version'], available_version)
                    # ~ pkg_vulns = db.getVulns(query={'src_package': pkg['src_id'], 'src_versions': pkg['src_version']})
                    # ~ urgency = ""
                    # ~ for pkg_vuln in pkg_vulns:
                        # ~ # Get max urgency
                        # ~ if pkg_vuln['urgency'] == "critical":
                            # ~ urgency = "critical"
                        # ~ elif pkg_vuln['urgency'] == "high" and urgency != "critical":
                            # ~ urgency = "high"
                        # ~ elif pkg_vuln['urgency'] == "medium" and (urgency != "critical" and urgency != "high"):
                            # ~ urgency = "medium"
                        # ~ elif pkg_vuln['urgency'] == "low" and urgency == "":
                            # ~ urgency = "low"
                    fixes.append(Vulns.Fix(pkg['hosts'], pkg['id'], pkg['version'], pkg['src_id'], pkg['src_version'], available_version, None, "", [], pkg['os_type'], pkg['os_version'], "madison", False, "", "").to_json())
                    nb_ub += 1
    logging.info('[%s] (%s)\t%d fixes for ubuntu found by madison.', datetime.datetime.today().isoformat(), __name__, nb_ub)
    
    for fix in fixes:
        result = db.insertFix(fix)
        if result == True:
            nb_mod += 1
        else:
            nb_err += 1
            
    logging.info('[%s] (%s)\t%d fixes from madison inserted with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    return nb_deb, nb_ub, nb_mod, nb_err
    
def cve_published_dates(db):
    #IN PROGRESS
    vulns = db.getVulns()
    nb_mod = 0
    nb_err = 0
    
    for vuln in vulns:
        if vuln['published_date'] == "":
            (cvesearch_api_status, cve) = cveapi.getCVE(vuln['id'])
            if cvesearch_api_status == False:
                break
            if cve is not None:
                vuln['published_date'] = cve['Published']
                result = db.updateVuln(vuln)
                if result == True:
                    nb_mod += 1
                else:
                    nb_err += 1
            
    logging.info('[%s] (%s)\t%d vulns completed with CVE published date with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    
    return nb_mod, nb_err 
    

def debfix_published_dates(db):
    #IN PROGRESS
    fixes = db.getFixes(query={'src': "debian"})
    nb_mod = 0
    nb_err = 0
    
    for fix in fixess:
        if fix['fix_published_date'] == "":
            fix_date = "TODO"
            if fix_date is not None:
                fix['fix_published_date'] = fix_date
                result = db.updateFix(fix)
                if result == True:
                    nb_mod += 1
                else:
                    nb_err += 1
            
    logging.info('[%s] (%s)\t%d Debian fixes completed with published date with %d problems.', datetime.datetime.today().isoformat(), __name__, nb_mod, nb_err)
    
    return nb_mod, nb_err
                
