#!/usr/bin/python
# -*- coding: utf-8 -*-
# Author: Kees Cook <kees@ubuntu.com>
# Author: Jamie Strandboge <jamie@ubuntu.com>
# Copyright (C) 2005-2017 Canonical Ltd.
#
# This script is distributed under the terms and conditions of the GNU General
# Public License, Version 3 or later. See http://www.gnu.org/copyleft/gpl.html
# for details.
from __future__ import print_function

import codecs
import datetime
import glob
import os
import re
import signal
import subprocess
import sys
import time
from ivulncheck_api.lib.uct import cache_urllib

from ivulncheck_api.lib.config_api import Configuration as conf

# Releases with '/' indicate a release using a ppa overlay of the form '<base
# releases>/<ppa>'. This is a superset of 'releases' since not all scripts
# support ppa overlays (eg, USN publication, various reports, etc)
all_releases = [
    'dapper', 'edgy', 'feisty', 'gutsy', 'hardy', 'intrepid', 'jaunty',
    'karmic', 'lucid', 'maverick', 'natty', 'oneiric', 'precise',
    'precise/esm', 'quantal', 'raring', 'saucy', 'trusty', 'utopic',
    'vivid', 'vivid/stable-phone-overlay', 'vivid/ubuntu-core',
    'wily', 'xenial', 'yakkety', 'zesty', 'artful', 'bionic', 'cosmic'
]

# common to all scripts
releases = [r for r in all_releases if '/' not in r]

eol_releases = [
    'dapper', 'edgy', 'feisty', 'gutsy', 'hardy', 'intrepid',
    'jaunty', 'karmic', 'lucid', 'maverick', 'natty', 'oneiric',
    'precise', 'quantal', 'raring', 'saucy', 'utopic', 'vivid',
    'vivid/stable-phone-overlay', 'vivid/ubuntu-core', 'wily',
    'yakkety', 'zesty'
]

devel_release = 'cosmic'

# releases to display for flavors
flavor_releases = [
    'lucid', 'precise', 'trusty', 'utopic', 'vivid', 'wily', 'xenial',
    'yakkety', 'zesty', 'artful', 'bionic', 'cosmic'
]

# primary name of extended support maintenance (esm) releases
esm_releases = [x[0:-len('/esm')] for x in all_releases if x.endswith('/esm')]

# you can generate a reasonably accurate release stamp via
# $ echo "(($(date -r  /path/to/ubuntu/archive/dists/$release/Release.gpg +%s) / 3600 ) + 1) * 3600" | /usr/bin/bc
release_stamps = {
    'warty':    1098748800,
    'hoary':    1112918400,
    'breezy':   1129075200,
    'dapper':   1149120000,
    'edgy':     1161864000,
    'feisty':   1176984000,
    'gutsy':    1192708800,
    'hardy':    1209038400,
    'intrepid': 1225368000,
    'jaunty':   1240488000,
    'karmic':   1256817600,
    'lucid':    1272565800,
    'maverick': 1286706600,
    'natty':    1303822800,
    'oneiric':  1318446000,
    'precise':  1335423600,
    'quantal':  1350547200,
    'raring':   1366891200,
    'saucy':    1381993200,
    'trusty':   1397826000,
    'utopic':   1414083600,
    'vivid':    1429027200,
    'wily':     1445518800,
    'xenial':   1461279600,
    'yakkety':  1476518400,
    'zesty':    1492153200,
    'artful':   1508418000,
    'bionic':   1524870000,
}

release_names = {
    'warty': 'Ubuntu 4.10 (Warty Warthog)',
    'hoary': 'Ubuntu 5.04 (Hoary Hedgehog)',
    'breezy': 'Ubuntu 5.10 (Breezy Badger)',
    'dapper': 'Ubuntu 6.06 LTS (Dapper Drake)',
    'edgy': 'Ubuntu 6.10 (Edgy Eft)',
    'feisty': 'Ubuntu 7.04 (Feisty Fawn)',
    'gutsy': 'Ubuntu 7.10 (Gutsy Gibbon)',
    'hardy': 'Ubuntu 8.04 LTS (Hardy Heron)',
    'intrepid': 'Ubuntu 8.10 (Intrepid Ibex)',
    'jaunty': 'Ubuntu 9.04 (Jaunty Jackalope)',
    'karmic': 'Ubuntu 9.10 (Karmic Koala)',
    'lucid': 'Ubuntu 10.04 LTS (Lucid Lynx)',
    'maverick': 'Ubuntu 10.10 (Maverick Meerkat)',
    'natty': 'Ubuntu 11.04 (Natty Narwhal)',
    'oneiric': 'Ubuntu 11.10 (Oneiric Ocelot)',
    'precise': 'Ubuntu 12.04 LTS (Precise Pangolin)',
    'precise/esm': 'Ubuntu 12.04 ESM (Precise Pangolin)',
    'quantal': 'Ubuntu 12.10 (Quantal Quetzal)',
    'raring': 'Ubuntu 13.04 (Raring Ringtail)',
    'saucy': 'Ubuntu 13.10 (Saucy Salamander)',
    'trusty': 'Ubuntu 14.04 LTS (Trusty Tahr)',
    'utopic': 'Ubuntu 14.10 (Utopic Unicorn)',
    'vivid': 'Ubuntu 15.04 (Vivid Vervet)',
    'vivid/stable-phone-overlay': 'Ubuntu Touch 15.04',
    'vivid/ubuntu-core': 'Ubuntu Core 15.04',
    'wily': 'Ubuntu 15.10 (Wily Werewolf)',
    'xenial': 'Ubuntu 16.04 LTS (Xenial Xerus)',
    'yakkety': 'Ubuntu 16.10 (Yakkety Yak)',
    'zesty': 'Ubuntu 17.04 (Zesty Zapus)',
    'artful': 'Ubuntu 17.10 (Artful Aardvark)',
    'bionic': 'Ubuntu 18.04 LTS (Bionic Beaver)',
    'cosmic': 'Ubuntu 18.10 (Cosmic Cuttlefish)',
}

valid_tags = [
    'universe-binary', 'not-ue', 'apparmor', 'stack-protector',
    'fortify-source', 'symlink-restriction', 'hardlink-restriction',
    'heap-protector', 'pie'
]

# eol and unsupported kernel_srcs
#                   'linux-source-2.6.15',
#                   'linux-ti-omap',
#                   'linux-linaro',
#                   'linux-qcm-msm',
#                   'linux-ec2',
#                   'linux-fsl-imx51',
#                   'linux-mvl-dove',
#                    'linux-lts-backport-maverick',
#                    'linux-lts-backport-natty',
#                    'linux-lts-backport-oneiric',
kernel_srcs = set(['linux',
                   'linux-ti-omap4',
                   'linux-armadaxp',
                   'linux-mako',
                   'linux-manta',
                   'linux-flo',
                   'linux-goldfish',
                   'linux-joule',
                   'linux-raspi2',
                   'linux-snapdragon',
                   'linux-aws',
                   'linux-azure',
                   'linux-gcp',
                   'linux-gke',
                   'linux-kvm',
                   'linux-oem',
                   'linux-euclid',
                   'linux-lts-quantal',
                   'linux-lts-raring',
                   'linux-lts-saucy',
                   'linux-lts-trusty',
                   'linux-lts-utopic',
                   'linux-lts-vivid',
                   'linux-lts-wily',
                   'linux-lts-xenial',
                   'linux-hwe',
                   'linux-hwe-edge'])
kernel_topic_branches = kernel_srcs.difference(['linux'])

# '<name>: (<git url>, <LP project for bugs>)
product_kernels = {
    'linux-krillin': ('https://github.com/bq/aquaris-E4.5',
                      'https://launchpad.net/...'),
    'linux-vegetahd': ('https://github.com/bq/aquaris-E5',
                       'https://launchpad.net/...')
}

# for sanity, try to keep these in alphabetical order
description_overrides = {
    'acpid': 'Advanced Configuration and Power Interface daemon',
    'apache2': 'Apache HTTP server',
    'apparmor': 'Linux security system',
    'apt': 'Advanced front-end for dpkg',
    'audiofile': 'Open-source version of the SGI audiofile library',
    'augeas': 'Configuration editing tool',
    'batik': 'SVG Library',
    'bogofilter': 'a fast Bayesian spam filter',
    'boost1.49': 'C++ utility libraries',
    'boost1.50': 'C++ utility libraries',
    'bsh': 'Java scripting environment',
    'c-ares': 'library for asynchronous name resolution',
    'ceilometer': 'OpenStack Telemetry service',
    'cinder': 'OpenStack storage service',
    'clamav': 'Anti-virus utility for Unix',
    'click': 'Click package manager',
    'colord': 'Service to manage device colour profiles',
    'cpio': 'a tool to manage archives of files',
    'cups': 'Common UNIX Printing System(tm)',
    'cupsys': 'Common UNIX Printing System(tm)',
    'cups-filters': 'OpenPrinting CUPS Filters',
    'curl': 'HTTP, HTTPS, and FTP client and client libraries',
    'cyrus-sasl2': 'Cyrus Simple Authentication and Security Layer',
    'db': 'Berkeley DB Utilities',
    'db4.8': 'Berkeley DB Utilities',
    'db5.3': 'Berkeley DB Utilities',
    'dbus': 'simple interprocess messaging system',
    'dbus-glib': 'simple interprocess messaging system',
    'dhcp3': 'DHCP server and client',
    'djvulibre': 'DjVu image format library and tools',
    'dovecot': 'IMAP and POP3 email server',
    'dpdk': 'set of libraries for fast packet processing',
    'ecryptfs-utils': 'eCryptfs cryptographic filesystem utilities',
    'eglibc': 'GNU C Library',
    'emacs24': 'GNU Emacs editor',
    'eucalyptus': 'Elastic Utility Computing Architecture',
    'evince': 'Document viewer',
    'evolution-data-server': 'Evolution suite data server',
    'exempi': 'library to parse XMP metadata',
    'exim4': 'Exim is a mail transport agent',
    'expat': 'XML parsing C library',
    'file': 'Tool to determine file types',
    'firefox': 'Mozilla Open Source web browser',
    'flac': 'Free Lossless Audio Codec',
    'fontconfig': 'generic font configuration library',
    'fop': 'XML formatter',
    'freerdp': 'RDP client for Windows Terminal Services',
    'freetype': 'FreeType 2 is a font engine library',
    'gdk-pixbuf': 'GDK Pixbuf library',
    'ghostscript': 'PostScript and PDF interpreter',
    'glance': 'OpenStack Image Registry and Delivery Service',
    'glibc': 'GNU C Library',
    'gnupg': 'GNU privacy guard - a free PGP replacement',
    'gnupg2': 'GNU privacy guard - a free PGP replacement',
    'gnutls13': 'GNU TLS library',
    'gnutls26': 'GNU TLS library',
    'gnutls28': 'GNU TLS library',
    'graphite2': 'Font rendering engine for Complex Scripts',
    'grub2': 'GRand Unified Bootloader',
    'gst-plugins-base0.10': 'GStreamer plugins',
    'gst-plugins-base1.0': 'GStreamer plugins',
    'gst-plugins-good0.10': 'GStreamer plugins',
    'gst-plugins-good1.0': 'GStreamer plugins',
    'gtk-vnc': 'VNC viewer widget',
    'gtk+2.0': 'GTK+ graphical user interface library',
    'gtk+3.0': 'GTK+ graphical user interface library',
    'harfbuzz': 'OpenType text shaping engine',
    'heimdal': 'Heimdal Kerberos Network Authentication Protocol',
    'heat': 'OpenStack Orchestration Service',
    'horizon': 'Web interface for OpenStack cloud infrastructure',
    'icedtea-web': 'A web browser plugin to execute Java applets',
    'icu': 'International Components for Unicode library',
    'imagemagick': 'Image manipulation programs and library',
    'imlib2': 'Image manipulation and rendering library',
    'intel-microcode': 'Processor microcode for Intel CPUs',
    'isc-dhcp': 'DHCP server and client',
    'jasper': 'Library for manipulating JPEG-2000 files',
    'jbig2dec': 'JBIG2 decoder library',
    'jbigkit': 'JBIG1 data compression library',
    'jockey': 'user interface and desktop integration for driver management',
    'json-c': 'JSON manipulation library',
    'kde4libs': 'KDE 4 core applications and libraries',
    'kdepim': 'Personal Information Management apps',
    'kdepimlibs': 'the KDE PIM libraries',
    'kdeutils': 'KDE general-purpose utilities',
    'keystone': 'OpenStack identity service',
    'krb5': 'MIT Kerberos Network Authentication Protocol',
    'konversation': 'Internet Relay Chat (IRC) client for KDE',
    'language-selector': 'Language selector for Ubuntu',
    'lcms2': 'Little CMS color management library',
    'libarchive': 'Library to read/write archive files',
    'libav': 'Multimedia player, server, encoder and transcoder',
    'libexif': 'library to parse EXIF files',
    'libcommons-fileupload-java': 'File upload capability for servlets and web applications',
    'libconfig-inifiles-perl': 'Perl module for working with INI configuration files',
    'libdmx': 'X11 Distributed Multihead extension library',
    'libfs': 'X11 Font Services library',
    'libgadu': 'Gadu-Gadu protocol library',
    'libgc': "Boehm-Demers-Weiser garbage collecting storage allocator library",
    'libgcrypt11': 'LGPL Crypto library',
    'libgcrypt20': 'LGPL Crypto library',
    'libgd2': 'GD Graphics Library',
    'libgdata': 'Library to access GData services',
    'libidn': 'implementation of IETF IDN specifications',
    'libjpeg6b': 'library for handling JPEG files',
    'libjpeg-turbo': 'library for handling JPEG files',
    'libkdcraw': 'RAW picture decoding library',
    'libksba': 'X.509 and CMS support library',
    'libmodplug': 'Library for mod music based on ModPlug',
    'libmspack': 'library for Microsoft compression formats',
    'libndp': 'Library for Neighbor Discovery Protocol',
    'libpng': 'PNG (Portable Network Graphics) file library',
    'libproxy': 'automatic proxy configuration management library',
    'libraw': 'raw image decoder library',
    'libreoffice': 'Office productivity suite',
    'libreoffice-l10n': 'Office productivity suite help',
    'librsvg': 'renderer library for SVG files',
    'libsoup2.4': 'HTTP client/server library for GNOME',
    'libssh': 'A tiny C SSH library',
    'libtasn1-3': 'Library to manage ASN.1 structures',
    'libtasn1-6': 'Library to manage ASN.1 structures',
    'libunity-webapps': 'UnityWebapps library',
    'libvdpau': 'Video Decode and Presentation API for Unix',
    'libvirt': 'Libvirt virtualization toolkit',
    'libvncserver': 'vnc server library',
    'libvorbis': 'The Vorbis General Audio Compression Codec',
    'libxcb': 'X11 C Binding',
    'libxcursor': 'X11 cursor management library',
    'libxext': 'X11 miscellaneous extensions library',
    'libxfixes': 'X11 miscellaneous fixes extension library',
    'libxfont': 'X11 font rasterisation library',
    'libxi': 'X11 Input extension library',
    'libxinerama': 'X11 Xinerama extension library',
    'libxp': 'X11 Printing Extension (Xprint) client library',
    'libxpm': 'X11 pixmap library',
    'libxrandr': 'X11 RandR extension library',
    'libxrandr-lts-quantal': 'X11 RandR extension library',
    'libxrender': 'X11 Rendering Extension client library',
    'libxres': 'X11 Resource extension library',
    'libxslt': 'XSLT processing library',
    'libxt': 'X11 toolkit intrinsics library',
    'libxtst': 'X11 Record extension library',
    'libxv': 'X11 Video extension library',
    'libxvmc': 'X11 Video extension library',
    'libxxf86dga': 'X11 Direct Graphics Access extension library',
    'libxxf86vm': 'X11 XFree86 video mode extension library',
    'linux': 'Linux kernel',
    'linux-ec2': 'Linux kernel for EC2',
    'linux-aws': 'Linux kernel for Amazon Web Services (AWS) systems',
    'linux-azure': 'Linux kernel for Microsoft Azure Cloud systems',
    'linux-gcp': 'Linux kernel for Google Cloud Platform (GCP) systems',
    'linux-gke': 'Linux kernel for Google Container Engine (GKE) systems',
    'linux-kvm': 'Linux kernel for cloud environments',
    'linux-oem': 'Linux kernel for OEM processors',
    'linux-euclid': 'Linux kernel for Intel Euclid systems',
    'linux-fsl-imx51': 'Linux kernel for IMX51',
    'linux-linaro': 'Linux kernel for ARM',
    'linux-lts-backport-maverick': 'Linux kernel backport from Maverick',
    'linux-lts-backport-natty': 'Linux kernel backport from Natty',
    'linux-lts-backport-oneiric': 'Linux kernel backport from Oneiric',
    'linux-lts-quantal': 'Linux hardware enablement kernel from Quantal',
    'linux-lts-raring': 'Linux hardware enablement kernel from Raring',
    'linux-lts-saucy': 'Linux hardware enablement kernel from Saucy',
    'linux-lts-trusty': 'Linux hardware enablement kernel from Trusty for Precise ESM',
    'linux-lts-utopic': 'Linux hardware enablement kernel from Utopic for Trusty',
    'linux-lts-vivid': 'Linux hardware enablement kernel from Vivid for Trusty',
    'linux-lts-wily': 'Linux hardware enablement kernel from Wily for Trusty',
    'linux-lts-xenial': 'Linux hardware enablement kernel from Xenial for Trusty',
    'linux-hwe': 'Linux hardware enablement (HWE) kernel',
    'linux-hwe-edge': 'Linux hardware enablement (HWE) testing kernel',
    'linux-mvl-dove': 'Linux kernel for DOVE',
    'linux-qcm-msm': 'Linux kernel for MSM',
    'linux-source-2.6.15': 'Linux kernel',
    'linux-ti-omap': 'Linux kernel for OMAP',
    'linux-ti-omap4': 'Linux kernel for OMAP4',
    'linux-armadaxp': 'Linux kernel for Armada XP',
    'linux-raspi2': 'Linux kernel for Raspberry Pi 2',
    'linux-snapdragon': 'Linux kernel for Snapdragon processors',
    'linux-joule': 'Linux kernel for Joule systems',
    'lxc': 'Linux Containers userspace tools',
    'lxd': 'Container hypervisor based on LXC',
    'lynx-cur': 'Text-mode WWW Browser with NLS support',
    'maas': 'Ubuntu MAAS Server',
    'mesa': 'free implementation of the EGL API',
    'mesa-lts-quantal': 'free implementation of the EGL API',
    'mime-support': 'MIME support programs',
    'miniupnpc': 'UPnP IGD client lightweight library',
    'moin': 'Collaborative hypertext environment',
    'mono': 'Mono is a platform for running and developing applications',
    'munin': 'Network-wide graphing framework',
    'mysql-dfsg-5.0': 'MySQL database',
    'mysql-dfsg-5.1': 'MySQL database',
    'mysql-5.1': 'MySQL database',
    'mysql-5.5': 'MySQL database',
    'mysql-5.6': 'MySQL database',
    'mysql-5.7': 'MySQL database',
    'nas': 'Network Audio System',
    'nbd': 'Network Block Device protocol',
    'net-snmp': 'SNMP (Simple Network Management Protocol) server and applications',
    'network-manager': 'Network connection manager',
    'network-manager-applet': 'GNOME frontend for NetworkManager',
    'neutron': 'OpenStack Virtual Network Service',
    'nova': 'OpenStack Compute cloud infrastructure',
    'nss': 'Network Security Service library',
    'nut': 'Network UPS tools',
    'nvidia-graphics-drivers': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-173': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-173-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-304': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-304-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-331': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-331-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-340': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-340-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-352': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-352-updates': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-367': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-375': 'NVIDIA binary X.Org driver',
    'nvidia-graphics-drivers-384': 'NVIDIA binary X.Org driver',
    'nvidia-settings': 'Tool for configuring the NVIDIA graphics driver',
    'nvidia-settings-updates': 'Tool for configuring the NVIDIA graphics driver',
    'open-iscsi': 'Open Source iSCSI implementation',
    'openjdk-6': 'Open Source Java implementation',
    'openjdk-6b18': 'Open Source Java implementation',
    'openjdk-7': 'Open Source Java implementation',
    'openjdk-8': 'Open Source Java implementation',
    'openoffice.org': 'Office productivity suite',
    'openssh': 'secure shell (SSH) for secure access to remote machines',
    'openssl': 'Secure Socket Layer (SSL) cryptographic library and tools',
    'openvpn': 'virtual private network software',
    'openvswitch': 'Ethernet virtual switch',
    'pacemaker': 'Cluster resource manager',
    'pam': 'Pluggable Authentication Modules',
    'paramiko': 'Python SSH2 library',
    'pcre3': 'Perl 5 Compatible Regular Expression Library',
    'pcsc-lite': 'Middleware to access a smart card using PC/SC',
    'perl': 'Practical Extraction and Report Language',
    'php5': 'HTML-embedded scripting language interpreter',
    'php7.0': 'HTML-embedded scripting language interpreter',
    'php7.1': 'HTML-embedded scripting language interpreter',
    'pillow': 'Python Imaging Library',
    'poppler': 'PDF rendering library',
    'postgresql-8.3': 'Object-relational SQL database',
    'postgresql-8.4': 'Object-relational SQL database',
    'postgresql-9.1': 'Object-relational SQL database',
    'postgresql-9.3': 'Object-relational SQL database',
    'postgresql-9.4': 'Object-relational SQL database',
    'postgresql-9.5': 'Object-relational SQL database',
    'postgresql-9.6': 'Object-relational SQL database',
    'ppp': 'Point-to-Point Protocol (PPP)',
    'puppet': 'Centralized configuration management',
    'pymongo': 'Python interface to the MongoDB document-oriented database',
    'pyopenssl': 'Python wrapper around the OpenSSL library',
    'python2.6': 'An interactive high-level object-oriented language',
    'python2.7': 'An interactive high-level object-oriented language',
    'python3.2': 'An interactive high-level object-oriented language',
    'python3.3': 'An interactive high-level object-oriented language',
    'python3.4': 'An interactive high-level object-oriented language',
    'python3.5': 'An interactive high-level object-oriented language',
    'python-cryptography': 'Cryptography Python library',
    'python-dbusmock': 'mock D-Bus objects for tests',
    'python-django': 'High-level Python web development framework',
    'python-imaging': 'Python Imaging Library',
    'python-pysaml2': 'Pure python implementation of SAML2',
    'qemu': 'Machine emulator and virtualizer',
    'qemu-kvm': 'Machine emulator and virtualizer',
    'qt4-x11': 'Qt 4 libraries',
    'qtbase-opensource-src': 'Qt 5 libraries',
    'quagga': 'BGP/OSPF/RIP routing daemon',
    'quantum': 'OpenStack Virtual Network Service',
    'rampart': 'Apache web services security engine',
    'requests': 'elegant and simple HTTP library for Python',
    'rhythmbox': 'Music player and organizer',
    'rsyslog': 'Enhanced syslogd',
    'ruby1.8': 'Object-oriented scripting language',
    'ruby1.9.1': 'Object-oriented scripting language',
    'ruby2.0': 'Object-oriented scripting language',
    'ruby2.1': 'Object-oriented scripting language',
    'ruby2.3': 'Object-oriented scripting language',
    'samba': 'SMB/CIFS file, print, and login server for Unix',
    'serf': 'high-performance asynchronous HTTP client library',
    'spice': 'SPICE protocol client and server library',
    'sqlite3': 'C library that implements an SQL database engine',
    'squid': 'Web proxy cache server',
    'squid3': 'Web proxy cache server',
    'strongswan': 'IPsec VPN solution',
    'swift': 'OpenStack distributed virtual object store',
    'thunderbird': 'Mozilla Open Source mail and newsgroup client',
    'tiff': 'Tag Image File Format (TIFF) library',
    'tomcat8': 'Servlet and JSP engine',
    'twisted': 'Event-based framework for internet applications',
    'ubufox': 'Ubuntu Firefox specific configuration defaults and apt support',
    'ubuntu-core-launcher': 'Snap application launcher',
    'udisks': 'service to access and manipulate storage devices',
    'udisks2': 'service to access and manipulate storage devices',
    'unity-firefox-extension': 'Unity Integration for Firefox',
    'usb-creator': 'create a startup disk using a CD or disc image',
    'vsftpd': 'FTP server written for security',
    'wayland': 'Wayland compositor infrastructure',
    'webkit': 'Web content engine library for GTK+',
    'webkitgtk': 'Web content engine library for GTK+',
    'webkit2gtk': 'Web content engine library for GTK+',
    'wpa': 'client support for WPA and WPA2',
    'wpasupplicant': 'client support for WPA and WPA2',
    'xmlrpc-c': 'Lightweight RPC library based on XML and HTTP',
    'xmlrpc-epi': 'a XML-RPC request library',
    'xorg-server': 'X.Org X11 server',
    'xorg-server-lts-quantal': 'X.Org X11 server',
    'xorg-server-lts-raring': 'X.Org X11 server',
    'xorg-server-lts-trusty': 'X.Org X11 server',
    'xorg-server-lts-utopic': 'X.Org X11 server',
    'xorg-server-lts-vivid': 'X.Org X11 server',
    'xorg-server-lts-xenial': 'X.Org X11 server',
    'xorg-server-hwe-16.04': 'X.Org X11 server',
    'xulrunner-1.9.2': 'Mozilla Gecko runtime environment',
}

# "arch_list" is all the physical architectures buildable
# "official_architectures" includes everything that should be reported on
official_architectures = ['amd64', 'armel', 'armhf', 'arm64', 'i386', 'lpia', 'powerpc', 'ppc64el', 's390x', 'sparc']
arch_list = official_architectures + ['hppa', 'ia64']
official_architectures = ['source', 'all'] + official_architectures

# The build expectations per release, per arch
release_expectations = {
    'dapper': {
        'required': ['amd64', 'i386', 'sparc', 'powerpc'],
        'expected': ['ia64', 'hppa'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'edgy': {
        'required': ['amd64', 'i386', 'sparc', 'powerpc'],
        'expected': [],
        'bonus': ['ia64', 'hppa'],
        'arch_all': 'i386',
    },
    'feisty': {
        'required': ['amd64', 'i386', 'sparc'],
        'expected': ['powerpc'],
        'bonus': ['hppa'],
        'arch_all': 'i386',
    },
    'gutsy': {
        'required': ['amd64', 'i386', 'sparc'],
        'expected': ['powerpc', 'hppa', 'lpia'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'hardy': {
        'required': ['amd64', 'i386', 'lpia'],
        'expected': ['powerpc', 'hppa', 'sparc'],
        'bonus': ['ia64'],
        'arch_all': 'i386',
    },
    'intrepid': {
        'required': ['amd64', 'i386', 'lpia'],
        'expected': ['powerpc', 'hppa', 'sparc'],
        'bonus': ['ia64'],
        'arch_all': 'i386',
    },
    'jaunty': {
        'required': ['amd64', 'i386'],
        'expected': ['lpia', 'powerpc', 'hppa', 'sparc', 'armel'],
        'bonus': ['ia64'],
        'arch_all': 'i386',
    },
    'karmic': {
        'required': ['amd64', 'i386', 'armel'],
        'expected': ['lpia', 'powerpc', 'sparc'],
        'bonus': ['ia64'],
        'arch_all': 'i386',
    },
    'lucid': {
        'required': ['amd64', 'i386', 'armel'],
        'expected': ['powerpc', 'sparc'],
        'bonus': ['ia64'],
        'arch_all': 'i386',
    },
    'maverick': {
        'required': ['amd64', 'i386', 'armel'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'natty': {
        'required': ['amd64', 'i386', 'armel'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'oneiric': {
        'required': ['amd64', 'i386', 'armel'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'precise': {
        'required': ['amd64', 'i386', 'armhf'],
        'expected': ['armel', 'powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'quantal': {
        'required': ['amd64', 'i386', 'armhf'],
        'expected': ['armel', 'powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'raring': {
        'required': ['amd64', 'i386', 'armhf'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'saucy': {
        'required': ['amd64', 'i386', 'armhf'],
        'expected': ['powerpc'],
        'bonus': ['arm64'],
        'arch_all': 'i386',
    },
    'trusty': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'utopic': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'i386',
    },
    'vivid': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'wily': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'xenial': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'yakkety': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': ['powerpc'],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'zesty': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': [],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'artful': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': [],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'bionic': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': [],
        'bonus': [],
        'arch_all': 'amd64',
    },
    'cosmic': {
        'required': ['amd64', 'i386', 'armhf', 'arm64', 'ppc64el', 's390x'],
        'expected': [],
        'bonus': [],
        'arch_all': 'amd64',
    },
}

# components in the archive
components = ['main', 'restricted', 'universe', 'multiverse']

# non-overlapping release package name changes, first-match wins
pkg_aliases = {
    'linux': ['linux-source-2.6.15'],
    'xen': ['xen-3.3', 'xen-3.2', 'xen-3.1'],
    'eglibc': ['glibc'],
    'qemu-kvm': ['kvm'],
}

# alternate names for packages in graphs
pkg_alternates = {
    'linux-source-2.6.15': 'linux',
    'linux-source-2.6.17': 'linux',
    'linux-source-2.6.20': 'linux',
    'linux-source-2.6.22': 'linux',
    'linux-restricted-modules-2.6.15': 'linux',
    'linux-backports-modules-2.6.15': 'linux',
    'linux-restricted-modules-2.6.17': 'linux',
    'linux-restricted-modules-2.6.20': 'linux',
    'linux-backports-modules-2.6.20': 'linux',
    'linux-restricted-modules-2.6.22': 'linux',
    'linux-backports-modules-2.6.22': 'linux',
    'linux-ubuntu-modules-2.6.22': 'linux',
    'linux-restricted-modules-2.6.24': 'linux',
    'linux-backports-modules-2.6.24': 'linux',
    'linux-ubuntu-modules-2.6.24': 'linux',
    'linux-restricted-modules': 'linux',
    'linux-backports-modules-2.6.27': 'linux',
    'linux-backports-modules-2.6.28': 'linux',
    'linux-backports-modules-2.6.31': 'linux',
    'xen-3.1': 'xen',
    'xen-3.2': 'xen',
    'xen-3.3': 'xen',
    'firefox-3.0': 'firefox',
    'firefox-3.5': 'firefox',
    'xulrunner-1.9': 'firefox',
    'xulrunner-1.9.1': 'firefox',
    'xulrunner-1.9.2': 'firefox',
    'ruby1.8': 'ruby',
    'ruby1.9': 'ruby',
    'python2.4': 'python',
    'python2.5': 'python',
    'python2.6': 'python',
    'openoffice.org-amd64': 'openoffice.org',
    'gnutls12': 'gnutls',
    'gnutls13': 'gnutls',
    'gnutls26': 'gnutls',
    'postgresql-8.1': 'postgresql',
    'postgresql-8.2': 'postgresql',
    'postgresql-8.3': 'postgresql',
    'compiz-fusion-plugins-main': 'compiz',
    'mysql-dfsg-5.0': 'mysql',
    'mysql-dfsg-5.1': 'mysql',
    'mysql-5.1': 'mysql',
    'gst-plugins-base0.10': 'gstreamer',
    'gst-plugins-good0.10': 'gstreamer',
    'mozilla-thunderbird': 'thunderbird',
    'openjdk-6b18': 'openjdk-6',
}


# update whenever an LTS moves from fully supported to partially
# supported
lts_partial_supported_releases = ['hardy', 'lucid']

# The CVE states considered "closed"
status_closed = set(['released', 'not-affected', 'ignored', 'DNE'])
# Possible CVE priorities
priorities = ['negligible', 'low', 'medium', 'high', 'critical']

CVE_RE = re.compile(r'^CVE-\d\d\d\d-[N\d]{4,7}$')


def release_sort(release_list):
    '''takes a list of release names and sorts them in release order'''

    # turn list into a tuples of (name, release index)
    rels = [(x, all_releases.index(x)) for x in release_list]
    # sort list by release index, then pull out just the names
    return [x[0] for x in sorted(rels, key=lambda x: x[1])]


class MetaKernelTable(object):
    def __init__(self):
        self.table = dict()

    # sources is expected to a be a list, the primary kernel first
    # e.g. add_new_kernel('precise', ['linux', 'lbm-3.2'], 'linux-meta', '-3.2.0')
    def add_new_kernel(self, release, sources, meta, suffix):
        if release not in self.table:
            self.table[release] = dict()
        (primary, subordinates) = (sources[0], sources[1:])
        self.table[release][primary] = dict()
        self.table[release][primary]['suffix'] = suffix
        self.table[release][primary]['meta'] = meta
        self.table[release][primary]['subordinates'] = subordinates

    def get_meta(self, release, kernel):
        if release in self.table and kernel in self.table[release]:
            return self.table[release][kernel]['meta'][0]
        return None

    def get_next_kernel(self):
        for release in self.table:
            for kernel in self.table[release]:
                srcs = [kernel]
                if len(self.table[release][kernel]['subordinates']) > 0:
                    srcs.extend(self.table[release][kernel]['subordinates'])
                meta = self.table[release][kernel]['meta']
                suffix = self.table[release][kernel]['suffix']
                yield (release, srcs, meta, suffix)


meta_kernels = MetaKernelTable()
meta_kernels.add_new_kernel('precise', ['linux', 'linux-backports-modules-3.2.0'], ['linux-meta'], '-3.2.0')
#meta_kernels.add_new_kernel('precise',['linux-ti-omap4'],['linux-meta-ti-omap4'],'-3.2.0')
#meta_kernels.add_new_kernel('precise',['linux-lts-quantal'],['linux-meta-lts-quantal'],'-3.5.0')
#meta_kernels.add_new_kernel('precise',['linux-lts-raring'],['linux-meta-lts-raring'],'-3.8.0')
#meta_kernels.add_new_kernel('precise',['linux-lts-saucy'],['linux-meta-lts-saucy'],'-3.11.0')
meta_kernels.add_new_kernel('precise', ['linux-lts-trusty'], ['linux-meta-lts-trusty'], '-3.13.0')
meta_kernels.add_new_kernel('trusty', ['linux'], ['linux-meta'], '-3.13.0')
meta_kernels.add_new_kernel('trusty', ['linux-aws'], ['linux-meta-aws'], '-4.4.0')
#meta_kernels.add_new_kernel('trusty', ['linux-exynos5'], ['linux-meta-exynos5'], '-3.13.0')
#meta_kernels.add_new_kernel('trusty', ['linux-keystone'], ['linux-meta-keystone'], '-3.13.0')
#meta_kernels.add_new_kernel('trusty', ['linux-lts-utopic'], ['linux-meta-lts-utopic'], '-3.16.0')
#meta_kernels.add_new_kernel('trusty', ['linux-lts-vivid'], ['linux-meta-lts-vivid'], '-3.19.0')
#meta_kernels.add_new_kernel('trusty', ['linux-lts-wily'], ['linux-meta-lts-wily'], '-4.2.0')
meta_kernels.add_new_kernel('trusty', ['linux-lts-xenial'], ['linux-meta-lts-xenial'], '-4.4.0')
#meta_kernels.add_new_kernel('vivid', ['linux'], ['linux-meta'], '-3.19.0')
meta_kernels.add_new_kernel('xenial', ['linux'], ['linux-meta'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-raspi2'], ['linux-meta-raspi2'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-aws'], ['linux-meta-aws'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-azure'], ['linux-meta-azure'], '-4.11.0')   # suffix may need to change, but it looks like it is ignored
meta_kernels.add_new_kernel('xenial', ['linux-azure-edge'], ['linux-meta-azure-edge'], '-4.11.0')   # suffix may need to change, but it looks like it is ignored
meta_kernels.add_new_kernel('xenial', ['linux-gcp'], ['linux-meta-gcp'], '-4.8.0')  # suffix may need to change, but it looks like it is ignored
meta_kernels.add_new_kernel('xenial', ['linux-gke'], ['linux-meta-gke'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-kvm'], ['linux-meta-kvm'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-oem'], ['linux-meta-oem'], '-4.13.0')
meta_kernels.add_new_kernel('xenial', ['linux-euclid'], ['linux-meta-euclid'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-joule'], ['linux-meta-joule'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-snapdragon'], ['linux-meta-snapdragon'], '-4.4.0')
meta_kernels.add_new_kernel('xenial', ['linux-hwe'], ['linux-meta-hwe'], '-4.8.0')
meta_kernels.add_new_kernel('xenial', ['linux-hwe-edge'], ['linux-meta-hwe-edge'], '-4.8.0')
#meta_kernels.add_new_kernel('yakkety', ['linux'], ['linux-meta'], '-4.8.0')
#meta_kernels.add_new_kernel('yakkety', ['linux-raspi2'], ['linux-meta-raspi2'], '-4.8.0')
#meta_kernels.add_new_kernel('yakkety', ['linux-snapdragon'], ['linux-meta-snapdragon'], '-4.4.0')
#meta_kernels.add_new_kernel('zesty', ['linux'], ['linux-meta'], '-4.10.0')
#meta_kernels.add_new_kernel('zesty', ['linux-raspi2'], ['linux-meta-raspi2'], '-4.10.0')
#meta_kernels.add_new_kernel('zesty', ['linux-snapdragon'], ['linux-meta-snapdragon'], '-4.4.0')
meta_kernels.add_new_kernel('artful', ['linux'], ['linux-meta'], '-4.13.0')
meta_kernels.add_new_kernel('artful', ['linux-raspi2'], ['linux-meta-raspi2'], '-4.13.0')
meta_kernels.add_new_kernel('artful', ['linux-snapdragon'], ['linux-meta-snapdragon'], '-4.4.0')
meta_kernels.add_new_kernel('bionic', ['linux'], ['linux-meta'], '-4.15.0')
meta_kernels.add_new_kernel('bionic', ['linux-raspi2'], ['linux-meta-raspi2'], '-4.15.0')
meta_kernels.add_new_kernel('bionic', ['linux-oem'], ['linux-meta-oem'], '-4.15.0')
meta_kernels.add_new_kernel('bionic', ['linux-aws'], ['linux-meta-aws'], '-4.15.0')
meta_kernels.add_new_kernel('bionic', ['linux-azure'], ['linux-meta-azure'], '-4.15.0')   # suffix may need to change, but it looks like it is ignored
meta_kernels.add_new_kernel('bionic', ['linux-gcp'], ['linux-meta-gcp'], '-4.15.0')  # suffix may need to change, but it looks like it is ignored
meta_kernels.add_new_kernel('bionic', ['linux-kvm'], ['linux-meta-kvm'], '-4.15.0')


# list of kernel versions to masquerade as when things end up in the
# wrong pockets or otherwise should not have a USN published for it.
kernel_glitches = {
    'linux': {
        'maverick': {
            '2.6.35-28.49': '2.6.35-28.50'
        },
        'precise': {
            '3.2.0-105.146': '3.2.0-106.147'
        },
        'trusty': {
            '3.13.0-49.81': '3.13.0-49.83'
        },
        'utopic': {
            '~': '3.16.0-44.59'
        },
        'xenial': {
            '4.4.0-28.47': '4.4.0-31.50'
        },
        'zesty': {
            '~': '4.10.0-20.22'
        },
        'artful': {  # artful update to disable spi driver
            '4.13.0-19.22': '4.13.0-21.24'
        },
    },
    'linux-aws': {  # linux-aws
        'trusty': {
            '4.4.0-1009.9': '4.4.0-1010.10'
        },
        'xenial': {
            '4.4.0-1047.56': '4.4.0-1048.57'
        },
    },
    'linux-euclid': {
        'xenial': {
            '4.4.0-9027.29': '4.4.0-9028.30'
        },
    },
    'linux-gcp': {  # linux-gcp
        'xenial': {
            '~': '4.10.0-1006.6'  # initial publication
        },
    },
    'linux-gke': {  # linux-gke
        'xenial': {
            '~': '4.4.0-1003.3'
        },
    },
    'linux-oem': {
        'xenial': {
            '4.13.0-1019.20': '4.13.0-1020.21'
        },
    },
    'linux-exynos5': {  # oem linux-exynos5 accidentally miscopied to security
        'trusty': {
            '~': '3.13.0-5.6'
        }
    },
    'linux-raspi2': {  # meltdown updates did not apply to linux-raspi2
        'xenial': {
            '4.4.0-1080.88': '4.4.0-1082.90'
        },
    },
    'linux-ti-omap4': {
        'precise': {
            '3.2.0-1483.110': '3.2.0-1484.111'
        },
    },
    'linux-keystone': {  # oem linux-keystone added post-release and
                         #  no USNs, so use '~', '<version in security>
        'trusty': {
            # '~': '3.13.0-43.68'
            '~': '3.13.0-68.96',
        },
    },
    'linux-snapdragon': {  # meltdown updates did not apply to linux-snapdragon
        'xenial': {
            '4.4.0-1081.86': '4.4.0-1084.89'
        },
        'yakkety': {
            '~': '4.4.0-1063.68'
        },
        'zesty': {
            '~': '4.4.0-1081.86'
        },
        'artful': {
            '~': '4.4.0-1093.98'
        },
    },
    'linux-lts-utopic': {  # first release of utopic backport kernel
        'trusty': {
            # '~': '3.16.0-25.33~14.04.2'
            '3.16.0-46.62~14.04.1': '3.16.0-49.65~14.04.1'
        }
    },
    'linux-lts-vivid': {  # zombie vivid lives on
        'trusty': {
            '3.19.0-66.74~14.04.1': '3.19.0-80.88~14.04.1',
        }
    },
    'linux-lts-wily': {  # wily lts accidentally miscopied to security
        'trusty': {
            '4.2.0-18.22~14.04.1': '4.2.0-19.23~14.04.1',
        }
    },
    'linux-lts-xenial': {
        'trusty': {
            # only change in 4.4.0-112.135~14.04.1 is NOBP config for s390x,
            # which is not a supported arch for Ubuntu 14.04 LTS.
            '4.4.0-111.134~14.04.1': '4.4.0-112.135~14.04.1',
        }
    },
    'linux-hwe': {  # hwe kernel initial publication
        'xenial': {
            '~': '4.8.0-39.42~16.04.1',
        }
    },
    'linux-hwe-edge': {  # hwe-edge is a pre-hwe testing kernel,
                         # should not issue USNs for it
        'xenial': {
            '~': '4.15.0-22.24~16.04.1',
        }
    },
    'linux-azure-edge': {  # azure-edge is a pre-azure testing kernel,
                           # should not issue USNs for it
        'xenial': {
            '~': '4.15.0-1012.12~16.04.2',
        }
    },
}


def lookup_glitch_version(src, release, version):
    if src in kernel_glitches and release in kernel_glitches[src] and \
           version in kernel_glitches[src][release]:
        return kernel_glitches[src][release][version]
    # No glitch found
    return None


# list of kernel meta abi versions to ignore abi mismatches when
# published in the archive.
kernel_mabi_glitches = {
    'linux-meta-hwe-edge': {
        'xenial': [
            '4.13.0.31.33',  # points to linux-hwe abi 31 instead of linux-hwe-edge
            '4.13.0.32.34',  # points to linux-hwe abi 32 instead of linux-hwe-edge
            '4.13.0.36.37',  # points to linux-hwe abi 36 instead of linux-hwe-edge
        ]
    },
}


def ignore_kernel_mabi(meta_src, release, version):
    return (meta_src in kernel_mabi_glitches
            and release in kernel_mabi_glitches[meta_src]
            and version in kernel_mabi_glitches[meta_src][release])


def set_cve_dir(path):
    '''Return a path with CVEs in it. Specifically:
       - if 'path' has CVEs in it, return path
       - if 'path' is a relative directory with no CVEs, see if UCT is defined
         and if so, see if 'UCT/path' has CVEs in it and return path
    '''
    p = path
    found = False
    if len(glob.glob("%s/CVE-*" % path)) > 0:
        found = True
    elif not path.startswith('/') and 'UCT' in os.environ:
        tmp = os.path.join(os.environ['UCT'], path)
        if len(glob.glob("%s/CVE-*" % tmp)) > 0:
            found = True
            p = tmp
            #print("INFO: using '%s'" % p, file=sys.stderr)

    if not found:
        print("WARN: could not find CVEs in '%s' (or relative to UCT)" % path, file=sys.stderr)
    return p


active_dir = set_cve_dir(os.path.join(conf.getUCTPath(), "active"))
retired_dir = set_cve_dir(os.path.join(conf.getUCTPath(), "retired"))
ignored_dir = set_cve_dir(os.path.join(conf.getUCTPath(), "ignored"))
embargoed_dir = os.path.join(conf.getUCTPath(), "embargoed")     # Intentionally not using set_cve_dir()

cve_dirs = [active_dir, retired_dir, ignored_dir]
if os.path.islink(embargoed_dir):
    cve_dirs.append(embargoed_dir)
supported_pkgs = dict()

EXIT_FAIL = 1
EXIT_OKAY = 0

config = {}


def parse_CVEs_from_uri(url):
    """Return a list of all CVE numbers mentioned in the given URL."""

    list = []
    cvere = re.compile("((?:CAN|can|CVE|cve)-\d\d\d\d-(\d|N){3,6}\d)")
    try:
        text = cache_urllib.urlopen(url).read().splitlines()
        for line in text:
            comment = line.find('#')
            if comment != -1:
                line = line[:comment]
            for cve in cvere.finditer(line):
                list.append(cve.group().upper().replace('CAN', 'CVE', 1))
    except IOError:
        print("Could not open", url, file=sys.stderr)

    return list


def read_config():
    '''Read in and do basic validation on config file'''
    try:
        from configobj import ConfigObj
    except ImportError:
        # Dapper lacks this class, so reimplement it quickly
        class ConfigObj(dict):
            def __init__(self, filepath):
                for line in open(filepath).readlines():
                    line = line.strip()
                    if line.startswith('#') or len(line) == 0:
                        continue
                    name, stuff = line.strip().split('=', 1)
                    self[name] = eval(stuff)

            def __attr__(self, name):
                return self.stuff[name]

    config_file = os.path.join(conf.getUCTConfFile())

    if not os.path.exists(config_file):
        raise ValueError("Could not find '%s'" % (config_file))

    # FIXME: Why does this need to be defined as "global" when other globals
    # like "releases" and "EXIT_OKAY" don't need it??
    global config
    config = ConfigObj(config_file)

    # Validate required arguments
    if "plb_authentication" not in config:
        raise ValueError("Could not find 'plb_authentication' entry in %s." % (config_file))
    if not os.path.exists(config["plb_authentication"]):
        raise ValueError("Could not find file specified by 'plb_authentication' in %s." % (config_file))
    return config


def drop_dup_release(cve, rel):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    saw = set()
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if line.startswith('%s_' % (rel)):
            pkg = line.split('_')[1].split(':')[0]
            if pkg not in saw:
                output.write(line)
                saw.add(pkg)
        else:
            output.write(line)
    output.close()
    os.rename(cve + '.new', cve)


def clone_release(cve, pkg, oldrel, newrel):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if line.startswith('%s_%s:' % (oldrel, pkg)):
            newline = line.replace('%s_%s:' % (oldrel, pkg), '%s_%s:' % (newrel, pkg), 1)
            output.write(newline)
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)


def update_state(cve, pkg, rel, state, notes):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if line.startswith('%s_%s:' % (rel, pkg)):
            line = '%s_%s: %s' % (rel, pkg, state)
            if notes:
                line += ' (%s)' % (notes)
            line += '\n'
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)


def add_state(cve, pkg, rel, state, notes, after_rel):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if line.startswith('%s_%s:' % (after_rel, pkg)):
            output.write(line)
            line = '%s_%s: %s' % (rel, pkg, state)
            if notes:
                line += ' (%s)' % (notes)
            line += '\n'
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)


def prepend_field(cve, field, value):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    output.write('%s: %s\n' % (field, value))
    output.write(codecs.open(cve, encoding="utf-8").read())
    output.close()
    os.rename(cve + '.new', cve)


def update_field(cve, field, value=None):
    found = False
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if line.startswith('%s:' % (field)):
            found = True
            if value is None:
                continue
            else:
                output.write('%s: %s\n' % (field, value))
        else:
            output.write(line)
    output.close()
    os.rename(cve + '.new', cve)
    # Do we actually need to add it instead?
    if not found and value:
        prepend_field(cve, field, value)


def drop_field(cve, field):
    update_field(cve, field)


def add_reference(cve, url):
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    in_references = False
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if in_references and not line.startswith(' '):
            output.write(' ' + url + '\n')
            in_references = False
        elif in_references and url in line:
            # skip if already there
            print("Skipped adding reference for '%s' (already present)" % (cve), file=sys.stderr)
            output.close()
            os.unlink(cve + '.new')
            return False
        elif not in_references and line.startswith('References:'):
            in_references = True
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)

    return True


def add_patch(cve, pkg, url, type="patch"):
    patch_header = "Patches_%s:" % (pkg)
    in_patch = False

    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if in_patch and not line.startswith(' '):
            output.write(' ' + type + ': ' + url + '\n')
            in_patch = False
        elif in_patch and url in line:
            # skip if already there
            print("Skipped adding debdiff for '%s' (already present)" % (cve), file=sys.stderr)
            output.close()
            os.unlink(cve + '.new')
            return False
        elif not in_patch and line.startswith(patch_header):
            in_patch = True
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)

    return True


def update_multiline_field(cve, field, text):
    update = ""
    text = text.rstrip()
    # this is a multi-line entry -- it must start with a newline
    if not text.startswith('\n'):
        text = '\n' + text
    output = codecs.open(cve + ".new", 'w', encoding="utf-8")
    skip = 0
    for line in codecs.open(cve, encoding="utf-8").readlines():
        if skip and line.startswith(' '):
            continue
        skip = 0
        if line.startswith('%s:' % (field)):
            prefix = '%s:' % (field)
            for textline in text.split('\n'):
                wanted = '%s%s\n' % (prefix, textline)
                output.write(wanted)
                prefix = ' '
                update += wanted
            skip = 1
            continue
        output.write(line)
    output.close()
    os.rename(cve + '.new', cve)
    return update


# This returns the list of open CVEs and embargoed CVEs (which are included
# in the first list).
def get_cve_list():
    cves = [elem for elem in os.listdir(active_dir)
            if re.match('^CVE-\d+-(\d|N)+$', elem)]
    cves_retired = [elem for elem in os.listdir(retired_dir)
            if re.match('^CVE-\d+-(\d|N)+$', elem)]

    uems = []
    if os.path.islink(embargoed_dir):
        uems = [elem for elem in os.listdir(embargoed_dir)
                if re.match('^[\w-]*$', elem)]
        for cve in uems:
            if cve in cves:
                print("Duplicated CVE (in embargoed): %s" % (cve), file=sys.stderr)
        cves = cves + uems

    return (cves, cves_retired, uems)


def contextual_priority(cveinfo, pkg=None, rel=None):
    '''Return the priority based on release, then package, then global'''
    if pkg:
        pkg_p = 'Priority_%s' % (pkg)
        if rel:
            rel_p = '%s_%s' % (pkg_p, rel)
            if rel_p in cveinfo:
                return 2, cveinfo[rel_p]
        if pkg_p in cveinfo:
            return 1, cveinfo[pkg_p]
    return 0, cveinfo.get('Priority', 'untriaged')


def find_cve(cve):
    '''Return filepath for a given CVE'''
    for dir in cve_dirs:
        filename = os.path.join(dir, cve)
        if os.path.exists(filename):
            return filename
    raise ValueError("Cannot locate path for '%s'" % (cve))


def load_cve(cve, strict=False):
    '''Loads a given CVE into:
       dict( fields...
             'pkgs' -> dict(  pkg -> dict(  release ->  (state, notes)   ) )
           )
    '''

    msg = ''
    code = EXIT_OKAY

    data = dict()
    data.setdefault('tags', dict())
    affected = dict()
    lastfield = None
    fields_seen = []
    if not os.path.exists(cve):
        raise ValueError("File does not exist: '%s'" % (cve))
    for line in codecs.open(cve, encoding="utf-8").readlines():
        line = line.rstrip()

        # Ignore blank/commented lines
        if len(line) == 0 or line.startswith('#'):
            continue
        if line.startswith(' '):
            try:
                data[lastfield] += '\n%s' % (line[1:])
            except KeyError as e:
                msg += "%s: bad line '%s' (%s)\n" % (cve, line, e)
                code = EXIT_FAIL
            continue

        try:
            field, value = line.split(':', 1)
        except ValueError as e:
            msg += "%s: bad line '%s' (%s)\n" % (cve, line, e)
            code = EXIT_FAIL
            continue

        lastfield = field = field.strip()
        if field in fields_seen:
            msg += "%s: repeated field '%s'\n" % (cve, field)
            code = EXIT_FAIL
        else:
            fields_seen.append(field)
        value = value.strip()
        if field == 'Candidate':
            data.setdefault(field, value)
            if value != "" and not value.startswith('CVE-') and not value.startswith('UEM-') and not value.startswith('EMB-'):
                msg += "%s: unknown Candidate '%s' (must be /(CVE|UEM|EMB)-/)\n" % (cve, value)
                code = EXIT_FAIL
        elif 'Priority' in field:
            # For now, throw away comments on Priority fields
            if ' ' in value:
                value = value.split()[0]
            if 'Priority_' in field:
                try:
                    foo, pkg = field.split('_', 1)
                except ValueError:
                    msg += "%s: bad field with 'Priority_': '%s'\n" % (cve, field)
                    code = EXIT_FAIL
                    continue
            data.setdefault(field, value)
            if value not in ['untriaged'] + priorities:
                msg += "%s: unknown Priority '%s'\n" % (cve, value)
                code = EXIT_FAIL
        elif 'Patches_' in field:
            '''These are raw fields'''
            try:
                foo, pkg = field.split('_', 1)
            except ValueError:
                msg += "%s: bad field with 'Patches_': '%s'\n" % (cve, field)
                code = EXIT_FAIL
                continue
            data.setdefault(field, value)
        elif 'Tags_' in field:
            '''These are processed into the "tags" hash'''
            try:
                foo, pkg = field.split('_', 1)
            except ValueError:
                msg += "%s: bad field with 'Tags_': '%s'\n" % (cve, field)
                code = EXIT_FAIL
                continue
            data['tags'].setdefault(pkg, set())
            for word in value.strip().split(' '):
                if word not in valid_tags:
                    msg += "%s: bad '%s': '%s'\n" % (cve, word, field)
                    code = EXIT_FAIL
                    continue
                data['tags'][pkg].add(word)
        elif '_' in field:
            try:
                release, pkg = field.split('_', 1)
            except ValueError:
                msg += "%s: bad field with '_': '%s'\n" % (cve, field)
                code = EXIT_FAIL
                continue
            if release not in all_releases + ['upstream', 'devel', 'product'] and release not in eol_releases:
                msg += "%s: bad release '%s'\n" % (cve, release)
                code = EXIT_FAIL
                continue
            try:
                info = value.split(' ', 1)
            except ValueError:
                msg += "%s: missing state for '%s': '%s'\n" % (cve, field, value)
                code = EXIT_FAIL
                continue
            state = info[0]
            if state == '':
                state = 'needs-triage'

            if len(info) < 2:
                notes = ""
            else:
                notes = info[1].strip()
            if notes.startswith('('):
                notes = notes[1:]
            if notes.endswith(')'):
                notes = notes[:-1]

            # Work-around for old-style of only recording released versions
            if notes == '' and state[0] in ('0123456789'):
                notes = state
                state = 'released'

            if state not in ['needs-triage', 'needed', 'active', 'pending', 'released', 'deferred', 'DNE', 'ignored', 'not-affected']:
                msg += "%s: %s_%s has unknown state: '%s'\n" % (cve, release, pkg, state)
                code = EXIT_FAIL

            # Verify "released" kernels have version notes
            #if state == 'released' and pkg in kernel_srcs and notes == '':
            #    msg += "%s: %s_%s has state '%s' but lacks version note\n" % (cve, release, pkg, state)
            #    code = EXIT_FAIL

            # Verify "active" states have an Assignee
            if state == 'active' and data['Assigned-to'].strip() == "":
                msg += "%s: %s_%s has state '%s' but lacks 'Assigned-to'\n" % (cve, release, pkg, state)
                code = EXIT_FAIL

            affected.setdefault(pkg, dict())
            affected[pkg].setdefault(release, [state, notes])
        elif field not in ['References', 'Description', 'Ubuntu-Description', 'Notes', 'Bugs', 'Assigned-to', 'Approved-by', 'PublicDate', 'PublicDateAtUSN', 'CRD', 'Discovered-by']:
            msg += "%s: unknown field '%s'\n" % (cve, field)
            code = EXIT_FAIL
        else:
            data.setdefault(field, value)

    # Check for required fields
    for field in ['Candidate', 'PublicDate', 'Description']:
        nonempty = ['Candidate']
        if strict:
            nonempty += ['PublicDate']

        if field not in data:
            msg += "%s: missing field '%s'\n" % (cve, field)
            code = EXIT_FAIL
        elif field in nonempty and data[field].strip() == "":
            msg += "%s: required field '%s' is empty\n" % (cve, field)
            code = EXIT_FAIL

    # Fill in defaults for missing fields
    if 'Priority' not in data:
        data.setdefault('Priority', 'untriaged')
    # Perform override fields
    if 'PublicDateAtUSN' in data:
        data['PublicDate'] = data['PublicDateAtUSN']
    if 'CRD' in data and data['PublicDate'] != data['CRD']:
        if cve.startswith("embargoed"):
            print("%s: adjusting PublicDate to use CRD: %s" % (cve, data['CRD']), file=sys.stderr)
        data['PublicDate'] = data['CRD']

    pkgs = sorted(affected.keys())
    # this check isn't safe due to the "ignored" cves.
    if False and len(pkgs) == 0:
            msg += "%s: no packages affected?!\n" % (cve)
            code = EXIT_FAIL
    # check for missing upstreams
    append = ''
    for pkg in pkgs:
        if 'product' not in affected[pkg].keys() and \
           'upstream' not in affected[pkg].keys():
            append += 'upstream_%s: \n' % (pkg)
    if append != '':
        print("%s: adding missing 'upstream' for: %s" % (cve, ", ".join(pkgs)), file=sys.stderr)
        open(cve, 'a').write(append)

    data['pkgs'] = affected

    if code != EXIT_OKAY:
        raise ValueError(msg.strip())
    return data


def load_all(cves, uems):
    table = dict()
    priority = dict()
    for cve in cves:
        priority.setdefault(cve, dict())
        cvedir = active_dir
        if cve in uems:
            cvedir = embargoed_dir
        cvefile = os.path.join(cvedir, cve)
        info = load_cve(cvefile)
        table.setdefault(cve, info)
    return table


# supported options
#  pkgfamily = rename linux-source-* packages to "linux", or "xen-*" to "xen"
#  packages = list of packages to pay attention to
#  debug = bool, display debug information
def load_table(cves, cves_retired, uems, opt=None):
    table = dict()
    priority = dict()
    listcves = []
    cveinfo = dict()
    namemap = dict()
    for cve in cves:
        table.setdefault(cve, dict())
        priority.setdefault(cve, dict())
        cvedir = active_dir
        if cve in uems:
            cvedir = embargoed_dir
        if cves in cves_retired:
            cvedir = retired_dir
        cvefile = os.path.join(cvedir, cve)
        info = load_cve(cvefile)
        cveinfo[cve] = info

        # Allow for Priority overrides
        priority[cve]['default'] = 'untriaged'
        try:
            priority[cve]['default'] = info['Priority']
        except KeyError:
            priority[cve]['default'] = 'untriaged'

        for package in info['pkgs']:
            pkg = package
            realpkg = pkg
            # special-case the kernel, since it is per-release
            if opt and 'linux' in opt.pkgfamily:
                if pkg in ['linux-source-2.6.15', 'linux-source-2.6.20', 'linux-source-2.6.22']:
                    pkg = 'linux'
            # special-case xen, since it is per-release
            if opt and 'xen' in opt.pkgfamily:
                if pkg in ['xen-3.0', 'xen-3.1', 'xen-3.2', 'xen-3.3']:
                    pkg = 'xen'
            if opt and opt.packages and pkg not in opt.packages:
                continue
            table[cve].setdefault(pkg, dict())
            namemap.setdefault(pkg, dict())
            for release in info['pkgs'][package]:
                rel = release
                if rel == 'devel':
                    rel = devel_release
                status = info['pkgs'][package][release][0]

                if opt and 'linux' in opt.pkgfamily and status == 'DNE':
                    continue
                if opt and 'xen' in opt.pkgfamily:
                    if status == 'DNE':
                        continue
                    # Skip xen-3.1 for non-gutsy when using pkgfamily override
                    if realpkg == 'xen-3.1' and rel != 'gutsy':
                        continue
                table[cve][pkg].setdefault(rel, status)
                namemap[pkg].setdefault(rel, realpkg)

                # Add status comments only if they exist
                if len(info['pkgs'][package][release]) > 1:
                    status_comment = " ".join(info['pkgs'][package][release][1:]).strip()
                    if status_comment != "":
                        table[cve][pkg].setdefault("%s_comment" % rel, " ".join(info['pkgs'][package][release][1:]))

            field = 'Priority_' + pkg
            if field in info:
                priority[cve][pkg] = info[field]
            if opt and opt.debug:
                print("Loaded '%s'" % (pkg), file=sys.stderr)

        # Ignore CVEs that have no packages we're interested in
        if len(table[cve]) != 0:
            listcves.append(cve)
    updated_cves = listcves
    return (table, priority, updated_cves, namemap, cveinfo)


def is_overlay_ppa(rel):
    return '/' in rel


def split_overlay_ppa_from_release(rel):
    if not is_overlay_ppa(rel):
        return (rel, None)

    return rel.split('/')


def is_active_release(rel):
    return rel not in eol_releases


def is_active_esm_release(rel):
    return rel in esm_releases and not get_esm_name(rel) in eol_releases


def get_esm_name(rel):
    return rel + '/esm'


def is_supported(map, pkg, rel, cvedata=None):
    if rel not in supported_pkgs:
        supported_pkgs.setdefault(rel, set())
        # check if partially supported
        if rel in lts_partial_supported_releases or is_overlay_ppa(rel):
            bn = '%s-supported.txt' % (rel)
            if is_overlay_ppa(rel):
                (base, ppa) = split_overlay_ppa_from_release(rel)
                bn = '%s-%s-supported.txt' % (base, ppa)

            supported_fn = os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), bn)
            # Fallback to UCT if possible
            if not os.path.isfile(supported_fn) and 'UCT' in os.environ:
                supported_fn = os.path.join(os.environ['UCT'], os.path.basename(supported_fn))
            if os.path.exists(supported_fn):
                for p in open(supported_fn).readlines():
                    if p.startswith('#'):
                        continue
                    supported_pkgs[rel].add(p.strip())

    # Allow for a tagged override to declare a pkg (from the perspective of
    # a given CVE item) to be unsupported.
    if cvedata and pkg in cvedata['tags'] and \
       ('universe-binary' in cvedata['tags'][pkg] or
        'not-ue' in cvedata['tags'][pkg]):
        return False
    # Look for component or if we have a partially supported release, and if
    # partially supported only support those packages that are supported
    if pkg in map[rel] and \
       (map[rel][pkg]['section'] == 'main' or
        map[rel][pkg]['section'] == 'restricted') and \
       (len(supported_pkgs[rel]) == 0 or pkg in supported_pkgs[rel]):
        return True
    return False


def any_supported(map, pkg, releases, cvedata):
    for rel in releases:
        if is_supported(map, pkg, rel, cvedata):
            return True
    return False


def is_partner(map, pkg, rel):
    if pkg in map[rel] and \
       (map[rel][pkg]['pocket'] == 'commercial' or
        map[rel][pkg]['section'] == 'partner'):
        return True
    return False


def any_partner(map, pkg, releases):
    for rel in releases:
        if is_partner(map, pkg, rel):
            return True
    return False


def is_universe(map, pkg, rel, cvedata):
    if is_supported(map, pkg, rel, cvedata) or is_partner(map, pkg, rel):
        return False
    return True


def any_universe(map, pkg, releases, cvedata):
    for rel in releases:
        if is_universe(map, pkg, rel, cvedata):
            return True
    return False


def load_debian_dsas(filename, verbose=True):
    dsa = None
    debian = dict()

    dsalist = open(filename)
    if verbose:
        print("Loading %s ..." % (filename))
    count = 0
    for line in dsalist:
        count += 1
        line = line.rstrip()
        try:
            if line == "":
                continue
            if line.startswith('\t'):
                if not dsa:
                    continue
                line = line.lstrip()
                if line.startswith('{'):
                    debian[dsa]['cves'] = line.strip(r'[{}]').split()
            elif line.startswith('['):
                # [DD Mon YYYY] <dsa> <pkg1> <pkg2> ... - <description>
                dsa = line.split()[3]
                date = datetime.datetime.strptime(line.split(r']')[0].lstrip('['), "%d %b %Y")
                desc = " ".join(" ".join(line.split()[4:]).split(' - ')[1:]).strip()
                debian.setdefault(dsa, {'date': date, 'desc': desc, 'cves': []})
        except:
            print("Error parsing line %d: '%s'" % (count, line), file=sys.stderr)
            raise
    return debian


def load_debian_cves(filename, verbose=True):
    cve = None
    debian = dict()

    cvelist = open(filename)
    if verbose:
        print("Loading %s ..." % (filename))
    count = 0
    for line in cvelist:
        count += 1
        line = line.rstrip()
        try:
            if line == "":
                continue
            if line.startswith('\t'):
                if not cve:
                    continue
                line = line.lstrip()
                if line.startswith('['):
                    continue
                if line.startswith('{'):
                    continue
                if line.startswith('-'):
                    info = line[1:].lstrip().split(' ', 1)
                    pkg = info[0]
                    line = ""
                    if len(info) > 1:
                        line = info[1]

                    info = line.lstrip().split(' ', 1)
                    state = info[0]
                    if state == "":
                        state = "<unfixed>"
                    line = ""
                    if len(info) > 1:
                        line = info[1]

                    priority = "needs-triage"
                    bug = None
                    note = None
                    if '(' in line and ')' in line:
                        info = line.split('(')[1].split(')')[0]
                        bits = info.split(';')
                        for bit in bits:
                            bit = bit.strip()
                            if bit.startswith('#'):
                                bug = bit[1:]
                            elif bit.startswith('bug #'):
                                bug = bit[5:]
                            else:
                                priority = bit
                    else:
                        note = line
                    if priority == 'unimportant':
                        priority = 'negligible'

                    debian[cve]['pkgs'].setdefault(pkg, {'priority': priority, 'bug': bug, 'note': note, 'state': state})

                    debian[cve]['state'] = 'FOUND'
                if line.startswith('RESERVED'):
                    debian[cve]['state'] = 'RESERVED'
                if line.startswith('REJECTED'):
                    debian[cve]['state'] = 'REJECTED'
                if line.startswith('NOT-FOR-US'):
                    debian[cve]['state'] = line
                if line.startswith('NOTE'):
                    debian[cve]['note'] += [line]
                if line.startswith('TODO'):
                    if not line.endswith('TODO: check'):
                        debian[cve]['note'] += [line]
            else:
                #if cve:
                #    print("Previous CVE: %s: %s" % (cve, debian[cve]))
                cve = line.split().pop(0)
                debian.setdefault(cve, {'pkgs': dict(), 'state': None, 'note': [], 'desc': " ".join(line.split()[1:])})
        except:
            print("Error parsing line %d: '%s'" % (count, line), file=sys.stderr)
            raise
    return debian


def load_ignored_reasons(filename):
    '''Load CVEs from a list of form "CVE-YYYY-NNNN # Reason"'''

    ignored = dict()

    for line in open(filename):
        line = line.strip()
        if len(line) == 0 or line.startswith('#'):
            continue
        reason = "Ignored"
        if line.startswith('CVE') and '#' in line:
            line, reason = line.split('#', 1)
        reason = reason.strip()
        if reason.startswith('DNE -') or reason.startswith('NFU -'):
            reason = reason[5:].lstrip('-')
        reason = reason.strip()
        if ' ' in line:
            cves = line.split(' ')
        else:
            cves = [line]
        for cve in cves:
            if len(cve) == 0:
                continue
            ignored.setdefault(cve, reason)

    return ignored


def debian_truncate(desc):
    i = 0
    while i < len(desc) and (i < 60 or desc[i] != ' '):
        i += 1
    if i == len(desc):
        return desc
    return desc[:i] + " ..."


def prepend_debian_cve(filename, cve, desc):
    '''This is prefix the Debian CVE list with a new CVE and
       truncated description with a TODO: check marker'''

    input = open(filename)
    output = open(filename + ".new", 'w')

    print("Prepending %s ..." % (cve))
    output.write(cve)
    if len(desc) > 0:
        output.write(' (%s)' % (debian_truncate(desc)))
    output.write('\n\tTODO: check\n')
    output.write(input.read())
    input.close()
    output.close()
    os.rename(filename + ".new", filename)


def update_debian_todo_cves(ignored, known, filename, debian_sources, verbose=False, update=True):
    '''This will replace any "TODO: check" entries with
    knowledge from the Ubuntu CVE Tracker'''

    input = open(filename)
    if update:
        if verbose:
            print("Updating %s ..." % (filename))
        output = open(filename + ".new", 'w')
    else:
        if verbose:
            print("Dry run ...")
        output = open('/dev/null', 'w')
    cves = dict()

    count = 0
    cve = None
    reserved = False
    reserved_text = None
    todo = False
    for line in input:
        count += 1
        line = line.rstrip('\n')
        if line.startswith('CVE'):
            # finish up previous CVE processing
            if todo and reserved:
                if cve in ignored and reserved_text.rstrip('\n') == '\tRESERVED':
                    print("\tNOT-FOR-US: %s" % (ignored[cve]), file=output)
                else:
                    print(reserved_text.rstrip('\n'), file=output)

            # now start the new CVE processing
            cve = line.split().pop(0)
            todo = True
            reserved = False
            reserved_text = []
        elif line.startswith('\t'):
            if todo and (line == '\tTODO: check' or line == '\tRESERVED'):
                if cve in ignored:
                    if line == '\tRESERVED':
                        reserved_text = line + "\n"
                        reserved = True
                    elif line == '\tTODO: check':
                        print("\tNOT-FOR-US: %s" % (ignored[cve]), file=output)
                        todo = False
                        if verbose:
                            print("%s: NFU" % (cve))
                    continue
                if cve in known:
                    if cve not in cves:
                        cves[cve] = load_cve('%s/%s' % (active_dir, cve))
                    pkgs = cves[cve]['pkgs']
                    # HACK: Debian package name fix-ups
                    if 'linux' in pkgs:
                        pkgs = ['linux-2.6']
                    for src in pkgs:
                        # Skip packages not in Debian
                        if src not in debian_sources:
                            continue
                        print("\t- %s <unfixed>" % (src), file=output)
                        if verbose:
                            print("%s: %s" % (cve, src))
                        todo = False
                    # If the CVE is known to Ubuntu but doesn't hit anything, leave it alone
                    if todo and not reserved:
                        print(line, file=output)
                    continue
            elif reserved:
                if line.startswith('\tNOT-FOR-US: '):
                    print(reserved_text.rstrip('\n'), file=output)
                    todo = False
                else:
                    reserved_text += line + "\n"
                    continue
        elif line.startswith('begin') or line.startswith('end'):
            pass
        else:
            raise ValueError("Error parsing line %d: '%s'" % (count, line))
        print(line, file=output)
    input.close()
    output.close()
    if update:
        os.rename(filename + ".new", filename)


def save_debian_cves(debian, filename):
    raise ValueError("This code does not work yet")
    out = open(filename, 'w')

    for cve in reversed(sorted(debian.keys())):
        title = cve
        if debian[cve]['desc']:
            title += " " + debian[cve]['desc']
        print(title, file=out)
        pkgs = sorted(debian[cve]['pkgs'].keys())
        for pkg in pkgs:
            print("\t- %s %s" % (pkg, debian[cve]['pkgs'][pkg]['state']), file=out)
            items = []
            priority = debian[cve]['pkgs'][pkg]['priority']
            if priority == "needs-triage":
                priority = None
            if priority:
                items += [priority]
            if debian[cve]['pkgs'][pkg]['bug']:
                items += ['#' + debian[cve]['pkgs'][pkg]['bug']]
            if len(items):
                print("(%s)" % (";".join(items)), file=out)
            else:
                print("", file=out)
        if len(pkgs) == 0:
            if not debian[cve]['state']:
                print("\tTODO: check", file=out)
            else:
                print("\t%s\n" % (debian[cve]['state']), file=out)
        for line in debian[cve]['note']:
            print("\t%s" % (line), file=out)


def cve_age(cve, open_date, close_stamp, oldest=None):
    # 'oldest' is a timestamp that is used to add a lower bound to
    # dates in "open_date" and "close_stamp"
    if open_date == 'unknown' or len(open_date) == 0:
        raise ValueError("%s: empty PublicDate" % (cve))
    date = open_date
    # CRDs are traditionally 1400UTC, so use this unless something else
    # is specified.
    mytime = '14:00:00'
    if ' ' in date:
        tmp = date
        date = tmp.split()[0]
        mytime = tmp.split()[1]
    year, mon, day = [int(x) for x in date.split('-')]
    hour, minute, second = [int(x) for x in mytime.split(':')]
    open_obj = datetime.datetime(year, mon, day, hour, minute, second)
    close_obj = datetime.datetime.utcfromtimestamp(int(close_stamp))
    if oldest:
        oldest = datetime.datetime.utcfromtimestamp(oldest)
        if open_obj < oldest:
            open_obj = oldest
        if close_obj < oldest:
            close_obj = oldest
    delta = close_obj - open_obj
    return delta.days


def recursive_rm(dirPath):
    '''recursively remove directory'''
    names = os.listdir(dirPath)
    for name in names:
        path = os.path.join(dirPath, name)
        if not os.path.isdir(path):
            os.unlink(path)
        else:
            recursive_rm(path)
    os.rmdir(dirPath)


def lts_unsupported(src_map, cvepath, pkg, contents=""):
    '''check if package is EOL for a particular release, and if so, update the
       cve. Note, this is not used for EOLing an entire release, but rather
       making sure the CVEs represent reality for situations for LTS
       releases, where only part of a release is active (eg, server is
       supported for 5 years and desktop 3)'''

    updated = contents
    for rel in lts_partial_supported_releases:
        # if no package or not in main or restricted, we don't need to change
        # anything
        if rel not in src_map or pkg not in src_map[rel] or src_map[rel][pkg]['section'] in ['universe', 'multiverse'] or is_partner(src_map, pkg, rel):
            continue

        if contents == "":
            if os.path.exists(cvepath) and not is_supported(src_map, pkg, rel, None):
                try:
                    data = load_cve(cvepath)
                except ValueError as e:
                    print(e, file=sys.stderr)
                    return
                if rel not in data['pkgs'][pkg]:
                    continue
                state, notes = data['pkgs'][pkg][rel]
                if state not in ['released', 'DNE', 'not-affected', 'ignored', 'pending']:
                    if state in ['needed', 'needs-triage', 'deferred']:
                        state = 'ignored'
                    print("%s: ignoring EOL '%s' for %s" % (os.path.basename(cvepath), pkg, rel), file=sys.stderr)
                    update_state(cvepath, pkg, rel, state, 'reached end-of-life')
                    updated = codecs.open(cvepath, encoding="utf-8").read()
        else:
            tmp = ""
            line = ""
            for line in contents.splitlines():
                if line.startswith('%s_%s:' % (rel, pkg)) and not is_supported(src_map, pkg, rel, None):
                    state = line.split(':')[1].strip().split()[0]
                    if state not in ['released', 'DNE', 'not-affected', 'ignored', 'pending']:
                        if state in ['needed', 'needs-triage']:
                            state = 'ignored'
                        line = '%s_%s: %s (%s)' % (rel, pkg, state, 'reached end-of-life')
                tmp += line + "\n"
            updated = tmp

    return updated


# Usage:
# config = ConfigObj(os.path.expanduser("~/.ubuntu-cve-tracker.conf"))
# cve_lib.check_mirror_timestamp(config)
# cve_lib.check_mirror_timestamp(config, mirror='packages_mirror')
def check_mirror_timestamp(config, mirror=None):
    mirrors = ['packages_mirror', 'partner_mirror']
    if mirror is not None:
        mirrors = [mirror]
    for m in mirrors:
        if m not in config:
            continue
        a = config[m]

        secs = 86400
        if m == 'partner_mirror':
            secs = 86400 * 7

        if os.path.exists(a + ".timestamp") and time.mktime(time.localtime()) - os.stat(a + ".timestamp").st_mtime > secs:
            print("WARNING: '%s' is older than %d day(s). Please run '$UCT/scripts/packages-mirror -t'." % (a, secs / 86400), file=sys.stderr)


# return the arch that the arch 'all' packages are built on. For utopic
# and prior, it was i386, but vivid and later are built on amd64
def get_all_arch(release):
    return release_expectations[release]['arch_all']


def arch_is_valid_for_release(arch, release):
    return (arch in release_expectations[release]['required'] or
            arch in release_expectations[release]['expected'] or
            arch in release_expectations[release]['bonus'])


def oldest_supported_release():
    '''Get oldest non-eol release'''
    for r in all_releases:
        if r not in eol_releases:
            return r


def subprocess_setup():
    # Python installs a SIGPIPE handler by default. This is usually not what
    # non-Python subprocesses expect.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)


def cmd(command, input=None, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, stdin=None, timeout=None):
    '''Try to execute given command (array) and return its stdout, or return
    a textual error if it failed.'''

    try:
        sp = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=True, preexec_fn=subprocess_setup)
    except OSError as e:
        return [127, str(e)]

    out, outerr = sp.communicate(input)
    # Handle redirection of stdout
    if out is None:
        out = ''
    # Handle redirection of stderr
    if outerr is None:
        outerr = ''
    return [sp.returncode, out + outerr]


def check_editmoin():
    # Make sure editmoin would actually work
    if not os.path.exists(os.path.expanduser('~/.moin_ids')) and not os.path.exists(os.path.expanduser('~/.moin_users')):
        print("Error: Need to configure editmoin to use this option (usually ~/.moin_ids).\n", file=sys.stderr)
        return False

    return True

def cve_sort(a, b):

    # Strip any path elements before sorting
    a = a.split("/")[-1]
    b = b.split("/")[-1]

    a_year = int(a.split("-")[1])
    a_number = int(a.split("-")[2])
    b_year = int(b.split("-")[1])
    b_number = int(b.split("-")[2])

    if a_year > b_year:
        return 1
    elif a_year < b_year:
        return -1
    elif a_number > b_number:
        return 1
    elif a_number < b_number:
        return -1
    else:
        return 0

